Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/@types/C2D/C2D.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,9 +181,25 @@ export interface ComputeEnvironment extends ComputeEnvironmentBaseConfig {
consumerAddress: string // v1
queuedJobs: number
queuedFreeJobs: number
/**
* Seconds of remaining queue wait, **summed across every queued job** in this environment
* (not a per-job maximum, despite the name). Each job contributes
* `max(0, queueMaxWaitTime - elapsed since it was created)`.
*/
queMaxWaitTime: number
/** As `queMaxWaitTime`, summed over queued *free* jobs only. */
queMaxWaitTimeFree: number
/**
* Seconds of remaining runtime, **summed across every running job** in this environment
* (not a per-job maximum, despite the name). Each job contributes
* `max(0, maxJobDuration - elapsed since it started)`; a job that has been allocated but
* has not started executing yet contributes its full `maxJobDuration`.
*
* Because it is a sum, a client cannot derive "when does capacity free up" from this field:
* 15 jobs with 1 second left each report 15, the same as one job with 15 seconds left.
*/
runMaxWaitTime: number
/** As `runMaxWaitTime`, summed over running *free* jobs only. */
runMaxWaitTimeFree: number
}

Expand Down
100 changes: 83 additions & 17 deletions src/components/c2d/compute_engine_base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import type {
ComputeEnvFees
} from '../../@types/C2D/C2D.js'
import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js'
import { C2DClusterType } from '../../@types/C2D/C2D.js'
import { C2DClusterType, C2DStatusNumber } from '../../@types/C2D/C2D.js'
import { C2DDatabase } from '../database/C2DDatabase.js'
import { Escrow } from '../core/utils/escrow.js'
import { KeyManager } from '../KeyManager/index.js'
Expand All @@ -30,6 +30,24 @@ import { ValidateParams } from '../httpRoutes/validateCommands.js'
import { EncryptMethod } from '../../@types/fileObject.js'
import { CORE_LOGGER } from '../../utils/logging/common.js'
import { DockerRegistryAuthSchema } from '../../utils/config/schemas.js'

/**
* Parse a job timestamp stored as decimal seconds in a string column.
*
* Job timestamps are initialized to the *string* `'0'` (not null/empty) and only get a real
* value once the corresponding phase starts. `'0'` is truthy, so a truthiness check happily
* accepts it and `Number.parseFloat('0')` then places the job's start at the Unix epoch,
* producing ~56 years of "elapsed" time. Always go through this helper: it returns 0 for the
* `'0'` sentinel, for null/undefined/empty, and for anything non-finite or negative, so
* callers only have to test `> 0` to know whether a real timestamp is present.
*/
export function parseJobTimestamp(raw?: string): number {
if (!raw) return 0
const parsed = Number.parseFloat(raw)
if (!Number.isFinite(parsed) || parsed <= 0) return 0
return parsed
}

export abstract class C2DEngine {
private clusterConfig: C2DClusterInfo
public db: C2DDatabase
Expand Down Expand Up @@ -570,6 +588,36 @@ export abstract class C2DEngine {
}
}

/**
* Seconds of runtime budget a job still has, clamped to [0, maxJobDuration].
*
* Build time counts against `maxJobDuration`, consistent with the runtime-expiry check in
* the docker engine, so `buildStartTimestamp` wins when both are set. A job that has been
* allocated but has not started ticking yet (no valid timestamp on either field — e.g. it
* is still in PullImage/BuildImage/ConfiguringVolumes) still has its whole budget ahead of
* it, so report the full `maxJobDuration` rather than measuring from the epoch.
*/
protected getJobRemainingRuntimeSeconds(job: DBComputeJob, nowSec: number): number {
const budget = Number.isFinite(job?.maxJobDuration) ? job.maxJobDuration : 0
const buildStart = parseJobTimestamp(job?.buildStartTimestamp)
const start = buildStart > 0 ? buildStart : parseJobTimestamp(job?.algoStartTimestamp)
if (start === 0) return budget
return Math.max(0, budget - (nowSec - start))
}

/**
* Seconds a queued job may still wait before its queue request expires, clamped to
* [0, queueMaxWaitTime]. Mirrors the queue-expiry check in the docker engine. As with the
* runtime helper, a missing/sentinel `dateCreated` means "not measurable yet", so report the
* full requested wait instead of measuring from the epoch.
*/
protected getJobRemainingQueueWaitSeconds(job: DBComputeJob, nowSec: number): number {
const requested = Number.isFinite(job?.queueMaxWaitTime) ? job.queueMaxWaitTime : 0
const created = parseJobTimestamp(job?.dateCreated)
if (created === 0) return requested
return Math.max(0, requested - (nowSec - created))
}

public async getUsedResources(env: ComputeEnvironment): Promise<any> {
const usedResources: { [x: string]: any } = {}
const usedFreeResources: { [x: string]: any } = {}
Expand All @@ -591,32 +639,38 @@ export abstract class C2DEngine {
let maxRunningTime = 0
let maxRunningTimeFree = 0

// one instant for the whole response, so every job in it is measured against the same clock
const nowSec = Date.now() / 1000

for (const job of jobs) {
const isThisEnv = job.environment === env.id
const isRunning = job.queueMaxWaitTime === 0
// A job holds resources from the moment it leaves the queue. JobQueued is the only
// pre-allocation state; every later state means volumes/containers exist or are being
// created. queueMaxWaitTime is the caller's *requested* wait, not live state, and is
// never reset on release — do not use it as a liveness flag.
const isQueued = job.status === C2DStatusNumber.JobQueued

if (isThisEnv) {
if (isRunning) {
const timeElapsed = job.buildStartTimestamp
? new Date().getTime() / 1000 - Number.parseFloat(job?.buildStartTimestamp)
: new Date().getTime() / 1000 - Number.parseFloat(job?.algoStartTimestamp)
totalJobs++
maxRunningTime += job.maxJobDuration - timeElapsed
if (isQueued) {
const waitLeft = this.getJobRemainingQueueWaitSeconds(job, nowSec)
queuedJobs++
maxWaitTime += waitLeft
if (job.isFree) {
totalFreeJobs++
maxRunningTimeFree += job.maxJobDuration - timeElapsed
queuedFreeJobs++
maxWaitTimeFree += waitLeft
}
} else {
queuedJobs++
maxWaitTime += job.maxJobDuration
const runtimeLeft = this.getJobRemainingRuntimeSeconds(job, nowSec)
totalJobs++
maxRunningTime += runtimeLeft
if (job.isFree) {
queuedFreeJobs++
maxWaitTimeFree += job.maxJobDuration
totalFreeJobs++
maxRunningTimeFree += runtimeLeft
}
}
}

if (isRunning) {
if (!isQueued) {
for (const resource of job.resources) {
const envRes = envResourceMap.get(resource.id)
if (envRes) {
Expand Down Expand Up @@ -731,7 +785,15 @@ export abstract class C2DEngine {

// Gate 1 (per-env ceiling) — fungible resources only.
// envResource.total = env aggregate ceiling (from EnvironmentResourceRef.total).
if (isFungible && envResource.total - (envResource.inUse ?? 0) < request.amount)
// Both operands are defaulted: a resource object missing either field would make the
// subtraction NaN, and `NaN < amount` is false, i.e. the gate would silently admit the
// request. Missing must deny, not admit. (`total` is required by the type, but these
// objects also arrive from JSON config and older persisted shapes, which the compiler
// cannot vouch for — free-resource entries with no `total` have been seen in the field.)
if (
isFungible &&
(envResource.total ?? 0) - (envResource.inUse ?? 0) < request.amount
)
throw new Error(`Not enough available ${request.id} in this environment`)

// Gate 2 (engine-wide pool ceiling) — fungible + exclusive discrete.
Expand All @@ -744,7 +806,11 @@ export abstract class C2DEngine {
if (!env.free) throw new Error(`No free resources`)
envResource = this.getResource(env.free?.resources, request.id)
if (!envResource) throw new Error(`No such free resource ${request.id}`)
if (envResource.total - envResource.inUse < request.amount)
// Same NaN-admits hazard as gate 1, and more reachable here: `inUse` is optional on
// ComputeResource, and free-resource entries carrying `max`/`inUse` but no `total`
// have been observed on live nodes (builds predating free-resource pool resolution).
// Unguarded, that yields NaN and allows unlimited free allocation.
if ((envResource.total ?? 0) - (envResource.inUse ?? 0) < request.amount)
throw new Error(`Not enough available ${request.id} for free`)
}
}
Expand Down
23 changes: 13 additions & 10 deletions src/components/c2d/compute_engine_docker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import type {
EnvironmentResourceRef
} from '../../@types/C2D/C2D.js'
import { BASE_CHAIN_ID, USDC_TOKEN_ADDRESS_BASE } from '../../utils/config.js'
import { C2DEngine } from './compute_engine_base.js'
import { C2DEngine, parseJobTimestamp } from './compute_engine_base.js'
import { C2DDatabase } from '../database/C2DDatabase.js'
import { Escrow } from '../core/utils/escrow.js'
import { create256Hash } from '../../utils/crypt.js'
Expand Down Expand Up @@ -2407,10 +2407,15 @@ export class C2DEngineDocker extends C2DEngine {
let expiry

const buildDuration = this.getValidBuildDurationSeconds(job)
// Never measure from the '0' sentinel: parseFloat('0') would place the start at the
// Unix epoch and make an otherwise healthy container look instantly expired. If the
// algo start is not recorded yet, treat the clock as starting now — the next sweep
// will see the real timestamp.
const algoStart = parseJobTimestamp(job.algoStartTimestamp) || timeNow
if (buildDuration > 0) {
// if job has build time, reduce the remaining algorithm runtime budget
expiry = parseFloat(job.algoStartTimestamp) + job.maxJobDuration - buildDuration
} else expiry = parseFloat(job.algoStartTimestamp) + job.maxJobDuration
expiry = algoStart + job.maxJobDuration - buildDuration
} else expiry = algoStart + job.maxJobDuration
CORE_LOGGER.debug(
'container running since timeNow: ' + timeNow + ' , Expiry: ' + expiry
)
Expand Down Expand Up @@ -4741,13 +4746,11 @@ export class C2DEngineDocker extends C2DEngine {
}

private getValidBuildDurationSeconds(job: DBComputeJob): number {
const startRaw = job.buildStartTimestamp
const stopRaw = job.buildStopTimestamp
if (!startRaw || !stopRaw) return 0
const start = Number.parseFloat(startRaw)
const stop = Number.parseFloat(stopRaw)
if (!Number.isFinite(start) || !Number.isFinite(stop)) return 0
if (start <= 0) return 0
// parseJobTimestamp returns 0 for the '0' sentinel, empty/missing values and anything
// non-finite, so a job that never built an image reports no build duration.
const start = parseJobTimestamp(job.buildStartTimestamp)
const stop = parseJobTimestamp(job.buildStopTimestamp)
if (start === 0 || stop === 0) return 0
if (stop < start) return 0
return stop - start
}
Expand Down
26 changes: 21 additions & 5 deletions src/components/core/handler/getJobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,24 @@ import { CORE_LOGGER } from '../../../utils/logging/common.js'
import { buildInvalidRequestMessage } from '../../httpRoutes/validateCommands.js'
import { CommandHandler } from './handler.js'
import { P2PCommandResponse } from '../../../@types/OceanNode.js'
import { parseFromTimestampSeconds } from '../utils/timestamps.js'

export class GetJobsHandler extends CommandHandler {
validate(command: GetJobsCommand) {
if (command.fromTimestamp && typeof command.fromTimestamp !== 'string') {
return buildInvalidRequestMessage(
'Parameter : "fromTimestamp" is not a valid string'
)
// absent / empty means "no filter", as before
if (command.fromTimestamp) {
if (typeof command.fromTimestamp !== 'string') {
return buildInvalidRequestMessage(
'Parameter : "fromTimestamp" is not a valid string'
)
}
// Reject unparseable values instead of passing them to SQL, where they used to match
// nothing and return 200 + [] — indistinguishable from "no jobs in that window".
if (!Number.isFinite(parseFromTimestampSeconds(command.fromTimestamp))) {
return buildInvalidRequestMessage(
`Parameter "fromTimestamp" is not a valid date: "${command.fromTimestamp}" — use an ISO date or a Unix timestamp`
)
}
}
return { valid: true }
}
Expand All @@ -27,9 +38,14 @@ export class GetJobsHandler extends CommandHandler {
throw new Error('C2D database not initialized')
}

// The DB columns store decimal seconds; validate() has already rejected anything
// unparseable, so this is either a finite seconds value or undefined (no filter).
const fromTimestamp = task.fromTimestamp
? parseFromTimestampSeconds(task.fromTimestamp)
: undefined
const jobs = await c2d.getJobs(
task.environments,
task.fromTimestamp,
fromTimestamp ?? undefined,
task.consumerAddrs,
undefined,
task.runningJobs
Expand Down
17 changes: 4 additions & 13 deletions src/components/core/service/getServices.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,11 @@ import {
type ServiceJob
} from '../../../@types/C2D/ServiceOnDemand.js'
import { toListedServiceJob } from './utils.js'
import { parseFromTimestamp } from '../utils/timestamps.js'

// Parses the `fromTimestamp` filter into Unix milliseconds. Accepts an ISO date string
// or a Unix timestamp (seconds or milliseconds) given as a string / number-like string.
// Returns undefined for "no filter" and null for an unparseable value (caller → 400).
export function parseFromTimestamp(value?: string): number | undefined | null {
if (value === undefined || value === null || value === '') return undefined
if (/^\d+$/.test(String(value))) {
const n = Number(value)
// 1e12 ms ≈ Sep 2001; any plausible seconds value is far below it
return n > 1e12 ? n : n * 1000
}
const t = Date.parse(String(value))
return Number.isNaN(t) ? null : t
}
// Re-exported for backwards compatibility: the parser now lives in core/utils/timestamps.ts
// so the compute-jobs listing can share it.
export { parseFromTimestamp }

// SERVICE_LIST: the node-wide service listing, shaped like GetJobsHandler. Default (no
// filters) returns exactly what the engines count against the shared resource pools
Expand Down
29 changes: 29 additions & 0 deletions src/components/core/utils/timestamps.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Normalization for caller-supplied `fromTimestamp`-style query filters.
//
// This is for *request parameters*, not for stored job timestamps: use
// `parseJobTimestamp` (compute_engine_base.ts) to read a persisted timestamp column, which
// treats the `'0'` sentinel as "not set". Here `0`/garbage must be distinguishable from
// "no filter" so the handler can answer 400 instead of silently returning an empty list.

// Parses the `fromTimestamp` filter into Unix milliseconds. Accepts an ISO date string
// or a Unix timestamp (seconds or milliseconds) given as a string / number-like string.
// Returns undefined for "no filter" and null for an unparseable value (caller → 400).
export function parseFromTimestamp(value?: string): number | undefined | null {
if (value === undefined || value === null || value === '') return undefined
if (/^\d+$/.test(String(value))) {
const n = Number(value)
// 1e12 ms ≈ Sep 2001; any plausible seconds value is far below it
return n > 1e12 ? n : n * 1000
}
const t = Date.parse(String(value))
return Number.isNaN(t) ? null : t
}

// Same parse, expressed in Unix *seconds* — the unit the compute_jobs table stores its
// dateCreated/dateFinished columns in. Kept as an explicit wrapper so no call site has to
// guess whether it is holding seconds or milliseconds.
export function parseFromTimestampSeconds(value?: string): number | undefined | null {
const ms = parseFromTimestamp(value)
if (ms === undefined || ms === null) return ms
return ms / 1000
}
3 changes: 2 additions & 1 deletion src/components/database/C2DDatabase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,10 @@ export class C2DDatabase extends AbstractDatabase {
return await this.provider.getFinishedJobs(environments)
}

/** @param fromTimestamp lower bound in Unix seconds — see SQLiteCompute.getJobs. */
async getJobs(
environments?: string[],
fromTimestamp?: string,
fromTimestamp?: number,
consumerAddrs?: string[],
status?: C2DStatusNumber,
runningJobs?: boolean
Expand Down
Loading
Loading