From b1a6f1a678080a2fe3c6dcaf3e5c345533eb5c47 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:27:01 +0100 Subject: [PATCH 01/77] feat(webapp,run-engine,core,clickhouse): surface total concurrency in metrics and dashboard Queues with a totalConcurrencyLimit now report how they use it. The gauge pipeline emits total running and the stored cap, ClickHouse aggregates them into the queue metrics tiers, the queues list gets a Total column, the queue detail page charts total running against the cap, and the per-key table shows each key's effective limit including per-key overrides. Queue retrieve and list API responses include the same totals. --- .changeset/queue-total-concurrency-stats.md | 5 + .../v3/QueueListPresenter.server.ts | 25 +++- .../v3/QueueRetrievePresenter.server.ts | 20 ++++ .../route.tsx | 32 ++++- .../route.tsx | 69 ++++++++++- ...ueueParam.concurrency.combined.override.ts | 3 + ....$queueParam.concurrency.combined.reset.ts | 3 + ...queues.$queueParam.concurrency.override.ts | 3 + ...v1.queues.$queueParam.concurrency.reset.ts | 3 + .../resources.queues.concurrency-keys.ts | 10 +- apps/webapp/app/v3/querySchemas.ts | 24 ++++ apps/webapp/app/v3/queueMetricsMapping.ts | 2 + ...42_add_queue_metrics_total_concurrency.sql | 109 ++++++++++++++++++ .../clickhouse/src/queueMetrics.ts | 2 + internal-packages/metrics-pipeline/src/lua.ts | 18 ++- .../run-engine/src/engine/index.ts | 14 +++ .../run-engine/src/run-queue/index.ts | 49 +++++++- packages/core/src/v3/schemas/queues.ts | 15 +++ 18 files changed, 394 insertions(+), 12 deletions(-) create mode 100644 .changeset/queue-total-concurrency-stats.md create mode 100644 internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md new file mode 100644 index 00000000000..a70da24d1fb --- /dev/null +++ b/.changeset/queue-total-concurrency-stats.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 7db2a6d2e39..70b02f9a4da 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -9,7 +9,10 @@ import { engine } from "~/v3/runEngine.server"; import { BasePresenter } from "./basePresenter.server"; import { toQueueItem } from "./QueueRetrievePresenter.server"; -type QueueListEngine = Pick; +type QueueListEngine = Pick< + RunEngine, + "lengthOfQueues" | "currentConcurrencyOfQueues" | "totalConcurrencyOfQueues" +>; export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; const MAX_ITEMS_PER_PAGE = 100; @@ -34,6 +37,9 @@ const queueListSelect = { concurrencyLimitOverriddenAt: true, concurrencyLimitOverriddenBy: true, concurrencyLimitOverridePercent: true, + totalConcurrencyLimit: true, + totalConcurrencyLimitBase: true, + totalConcurrencyLimitOverriddenAt: true, type: true, paused: true, } satisfies Prisma.TaskQueueSelect; @@ -334,11 +340,15 @@ export class QueueListPresenter extends BasePresenter { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: string | null; concurrencyLimitOverridePercent: Prisma.Decimal | null; + totalConcurrencyLimit: number | null; + totalConcurrencyLimitBase: number | null; + totalConcurrencyLimitOverriddenAt: Date | null; type: TaskQueueType; paused: boolean; }[] ): Promise { - const [queuedByQueue, runningByQueue] = await Promise.all([ + const queuesWithTotalCap = queues.filter((q) => q.totalConcurrencyLimit !== null); + const [queuedByQueue, runningByQueue, totalRunningByQueue] = await Promise.all([ this.engineClient.lengthOfQueues( environment, queues.map((q) => q.name) @@ -347,6 +357,12 @@ export class QueueListPresenter extends BasePresenter { environment, queues.map((q) => q.name) ), + queuesWithTotalCap.length > 0 + ? this.engineClient.totalConcurrencyOfQueues( + environment, + queuesWithTotalCap.map((q) => q.name) + ) + : Promise.resolve({} as Record), ]); // Manually "join" the overridden users because there is no way to implement the relationship @@ -374,6 +390,11 @@ export class QueueListPresenter extends BasePresenter { ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) : null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, + totalRunning: + queue.totalConcurrencyLimit !== null ? (totalRunningByQueue[queue.name] ?? 0) : null, }), // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). concurrencyLimitOverridePercent: diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 6385b388d1f..d7231402f86 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -92,6 +92,7 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), + engine.totalConcurrencyOfQueues(environment, [queue.name]), ]); // Transform queues to include running and queued counts @@ -109,6 +110,11 @@ export class QueueRetrievePresenter extends BasePresenter { concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt ?? null, + totalRunning: + queue.totalConcurrencyLimit != null ? (results[2]?.[queue.name] ?? 0) : null, }), // The percent source-of-truth for percent-based overrides isn't part of the shared // `QueueItem` schema (that's a public contract), so we surface it as an extra field on @@ -150,6 +156,10 @@ export function toQueueItem(data: { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: User | null; paused: boolean; + totalConcurrencyLimit?: number | null; + totalConcurrencyLimitBase?: number | null; + totalConcurrencyLimitOverriddenAt?: Date | null; + totalRunning?: number | null; }): QueueItem & { releaseConcurrencyOnWaitpoint: boolean } { return { id: data.friendlyId, @@ -166,6 +176,16 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, + total: + data.totalConcurrencyLimit !== undefined + ? { + current: data.totalConcurrencyLimit, + base: data.totalConcurrencyLimitBase ?? null, + override: data.totalConcurrencyLimitOverriddenAt ? data.totalConcurrencyLimit : null, + overriddenAt: data.totalConcurrencyLimitOverriddenAt ?? null, + running: data.totalRunning ?? null, + } + : undefined, }, // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients releaseConcurrencyOnWaitpoint: true, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 475f25362dc..5ec0ada0371 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -705,6 +705,13 @@ function QueuesWithMetricsView() { Queued Running Limit + + Total + + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 7ddcdb1d607..ff4fc786bc5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -392,7 +392,12 @@ export default function Page() { ) ) : ( - + )} @@ -402,7 +407,13 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -436,10 +447,12 @@ function OverviewCharts({ ids, timeRange, queueName, + hasTotalLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; + hasTotalLimit: boolean; }) { const zoomToTimeFilter = useZoomToTimeFilter(); return ( @@ -479,6 +492,37 @@ function OverviewCharts({ // leading zeros so the reference line doesn't start with a false 0→limit step. carryBackfill={["limit"]} /> + {hasTotalLimit ? ( + + Runs in flight across ALL concurrency keys ( + ) versus the queue's total limit ( + + ). + + } + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "running", label: "Running", color: COLORS.running }, + ]} + thresholdStroke={{ + series: "running", + valueFromSeries: "cap", + aboveColor: "var(--color-warning)", + }} + carryBackfill={["cap"]} + /> + ) : null} Key Queued now Running now + + Limit + Oldest wait Started Peak backlog @@ -976,11 +1031,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -994,6 +1049,12 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} + + {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} + {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts index c643b77965a..77688a9fcc5 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts @@ -46,6 +46,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts index b2841f1efe6..0e588716658 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts @@ -45,6 +45,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 90f5772c5d3..42bb2008682 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -61,6 +61,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index 503d875e471..3f36e629f09 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -43,6 +43,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 67c2b9f500a..662013694ef 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,6 +43,8 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; + /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ + limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -151,8 +153,11 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts from Redis. - const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); + // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. + const [live, keyLimitOverrides] = await Promise.all([ + engine.concurrencyKeyLiveStats(environment, queueName, keys), + engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + ]); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -168,6 +173,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, + limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 690bbaf5396..8267a0f8020 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,6 +770,22 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, + max_total_running: { + name: "max_total_running", + ...column("UInt32", { + description: + "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", + fillMode: "carry", + }), + }, + max_total_limit: { + name: "max_total_limit", + ...column("UInt32", { + description: + "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + fillMode: "carry", + }), + }, max_ck_backlogged: { name: "max_ck_backlogged", ...column("UInt32", { @@ -1406,6 +1422,14 @@ const queueMetricsByKeySchema: TableSchema = { fillMode: "carry", }), }, + max_limit: { + name: "max_limit", + ...column("UInt32", { + description: + "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + fillMode: "carry", + }), + }, wait_ms_sum: { name: "wait_ms_sum", ...column("UInt64", { diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index 9433b361a88..d341f4a63cd 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,6 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), + total_running: num(f.tcc), + total_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql new file mode 100644 index 00000000000..7711effa6e3 --- /dev/null +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -0,0 +1,109 @@ +-- +goose Up + +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key +-- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. + +ALTER TABLE trigger_dev.queue_metrics_raw_v1 + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + +ALTER TABLE trigger_dev.queue_metrics_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_5m_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_ck_v1 + ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); + +-- Materialized views cannot be altered: recreate them with the new columns. The 5m +-- MV MUST keep reading raw, never cascade off queue_metrics_v1 (out-of-time-order +-- deltaSumTimestamp merges double-count bridging spans). + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + maxIf(queue_limit, op = 'gauge') AS max_limit, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; + +-- +goose Down +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index 39576b4a0a3..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,6 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts index 64f3b896c0d..701f608308a 100644 --- a/internal-packages/metrics-pipeline/src/lua.ts +++ b/internal-packages/metrics-pipeline/src/lua.ts @@ -17,6 +17,10 @@ export type GaugeComputeLuaParams = { // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9]. ckBacklogged?: string; ckMaxWaitMs?: string; + // Total-concurrency extras (both or neither, and only with the CK extras): appended as + // gauge[10]/gauge[11]. totalLimit is the RAW stored limit (0 = none); readers clamp. + totalRunning?: string; + totalLimit?: string; }; // Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat @@ -26,11 +30,21 @@ export type GaugeComputeLuaParams = { export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string { const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0"; const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null; - const gauge = hasCk + const hasTotal = params.totalRunning != null && params.totalLimit != null; + if (hasTotal && !hasCk) { + throw new Error("gauge totalRunning/totalLimit extras require the CK extras"); + } + const gauge = hasTotal ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 + local __tcc = tonumber(${params.totalRunning}) or 0 + local __tlim = tonumber(${params.totalLimit}) or 0 + __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw, __tcc, __tlim}` + : hasCk + ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 + local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}` - : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; + : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; return ` if ${params.enabledArg} then diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 40e3d7bc336..1909df7e9c6 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1748,6 +1748,20 @@ export class RunEngine { return this.runQueue.currentConcurrencyOfQueues(environment, queues); } + async totalConcurrencyOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyOfQueues(environment, queues); + } + + async totalConcurrencyLimitsOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyLimitsOfQueues(environment, queues); + } + async concurrencyKeyBreakdown( environment: MinimalAuthenticatedEnvironment, queue: string, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 1dcbcb33a88..33f2983e069 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -217,7 +217,8 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: + "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -248,6 +249,8 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -708,6 +711,46 @@ export class RunQueue { return limits; } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ + public async totalConcurrencyOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + pipeline.scard(this.keys.queueGroupConcurrencyKey(env, queue)); + }); + + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index]?.[1]; + acc[queue] = typeof value === "number" ? value : 0; + return acc; + }, + {} as Record + ); + } + + /** Batch read of the RAW stored total concurrency limits (undefined = no cap). */ + public async totalConcurrencyLimitsOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const keys = queues.map((queue) => this.keys.queueTotalConcurrencyLimitKey(env, queue)); + const values = keys.length > 0 ? await this.redis.mget(...keys) : []; + + return queues.reduce( + (acc, queue, index) => { + const value = values[index]; + acc[queue] = value != null ? Number(value) : undefined; + return acc; + }, + {} as Record + ); + } + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { await this.#callUpdateEnvironmentConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), @@ -2355,6 +2398,10 @@ export class RunQueue { fields.ckq = ckq; fields.ckw = ckw; } + if (gauge.length >= 11) { + fields.tcc = gauge[9]; + fields.tlim = gauge[10]; + } this.options.queueMetrics?.emitGauge(queue, fields); } diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 5ebc8258cf6..dc1b6e1de7c 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,6 +45,21 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), + /** The total concurrency cap across all concurrencyKey values of the queue */ + total: z + .object({ + /** The effective/current total concurrency limit (null = no cap) */ + current: z.number().nullable(), + /** The declared total limit an override reverts to on reset */ + base: z.number().nullable(), + /** The overridden total limit, when an override is active */ + override: z.number().nullable(), + /** When the total override was applied */ + overriddenAt: z.coerce.date().nullable(), + /** Runs currently in flight across all concurrencyKey values */ + running: z.number().nullable(), + }) + .optional(), }) .optional(), }); From c51afe211ebe2164d99f4823fcfeeab3d3f2aac7 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:38:22 +0100 Subject: [PATCH 02/77] fix(run-engine,clickhouse,webapp): total gauges on enqueue paths; restore views on rollback The CK enqueue gauges (fast path and queued path) now sample total running and the stored cap, so metric buckets fed only by enqueues no longer record zero totals. The migration's down section recreates the pre-existing materialized view definitions so ingestion keeps flowing after a rollback. The per-key table reads only the page's overrides with one HMGET instead of loading the queue's whole override hash. --- .../resources.queues.concurrency-keys.ts | 2 +- ...42_add_queue_metrics_total_concurrency.sql | 68 +++++++++++++++++++ .../run-engine/src/run-queue/index.ts | 36 +++++++++- 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 662013694ef..8c590554e51 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -156,7 +156,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. const [live, keyLimitOverrides] = await Promise.all([ engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), ]); const loadedAt = Date.now(); diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 7711effa6e3..8bb19faeef1 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -107,3 +107,71 @@ ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; + +-- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps +-- feeding every aggregate table after a rollback. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 33f2983e069..a1aa60664f7 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -212,6 +212,14 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { ckMaxWaitMs: "__ckwait", }; +// Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. +// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually +// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", +}; + // CK enqueue variants of the two gauges above, extended with the CK-health tail. const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", @@ -223,6 +231,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ @@ -234,6 +243,7 @@ const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua envRunning: "envCurrent", envLimit: "envLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already @@ -249,8 +259,7 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, - totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -711,6 +720,29 @@ export class RunQueue { return limits; } + /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ + public async getQueueConcurrencyKeyLimitsForKeys( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKeys: string[] + ): Promise> { + if (concurrencyKeys.length === 0) { + return {}; + } + + const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); + const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); + + const limits: Record = {}; + concurrencyKeys.forEach((key, index) => { + const value = values[index]; + if (value != null) { + limits[key] = Number(value); + } + }); + return limits; + } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, From 60c3bcc71f4ab80bb5b0c2d3fbbf658d06cd90bf Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 16:48:06 +0100 Subject: [PATCH 03/77] fix(clickhouse): keep migration comments semicolon-free The test harness splits a migration's up section on semicolons, so a semicolon inside a comment yields a comment-only statement that ClickHouse rejects as an empty query and every container-backed suite fails at setup. --- .../schema/042_add_queue_metrics_total_concurrency.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 8bb19faeef1..f45e6421ae3 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -2,7 +2,7 @@ -- Total-concurrency gauges: total_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), total_limit the --- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. From 3e7c719f1d53b090b7b282a1644f35873d439525 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 17:01:59 +0100 Subject: [PATCH 04/77] fix(webapp): skip the total concurrency read when the queue has no cap Queue retrieve only asks the engine for total running when a total limit is set, matching the list presenter and avoiding a pointless read for the common uncapped case. --- .../webapp/app/presenters/v3/QueueRetrievePresenter.server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index d7231402f86..04616a9cc0c 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -92,7 +92,9 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), - engine.totalConcurrencyOfQueues(environment, [queue.name]), + queue.totalConcurrencyLimit != null + ? engine.totalConcurrencyOfQueues(environment, [queue.name]) + : undefined, ]); // Transform queues to include running and queued counts From 1e10c985b4c9cffb0438ff2dc2c9bc3478116d43 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:21:57 +0100 Subject: [PATCH 05/77] feat(webapp): show the Total column in the non-metrics queues table too The total concurrency numbers come from live Redis, not the metrics pipeline, so the column belongs in both tables rather than only behind the queue metrics UI gate. --- .../route.tsx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 5ec0ada0371..5e830fd68a0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1813,6 +1813,12 @@ function ClassicQueuesView() { Queued Running Limit + + Total + {limit} + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From 3d6413fe3fe86834274b8fabad3a8b1480c353ea Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:33:02 +0100 Subject: [PATCH 06/77] feat(webapp): fold the total cap into the Limit column A separate Total column implied every queue should have one, and its dash read as a missing limit on queues that never use concurrency keys. Only queues that declare a totalConcurrencyLimit now change: their Limit cell reads as per-key plus total (e.g. 1 /key, 3 total) and Running turns warning-colored when the total cap is saturated. Plain queues are unchanged. --- .../route.tsx | 88 ++++++++----------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 5e830fd68a0..2caf4f95802 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -704,13 +704,12 @@ function QueuesWithMetricsView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright" + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -879,29 +885,16 @@ function QueuesWithMetricsView() { ) : ( limit )} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} @@ -1812,12 +1805,11 @@ function ClassicQueuesView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright", + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright", isAtConcurrencyLimit && "text-warning" )} > @@ -1940,27 +1939,16 @@ function ClassicQueuesView() { )} > {limit} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From d5b58669c34545bcb023ee7c345f7a49e0fc715c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:39:14 +0100 Subject: [PATCH 07/77] fix(webapp): saturate the total-cap warning on keyed runs only The total cap gates keyed admissions, so the Running cell now warns off the group count rather than the aggregate that also includes unkeyed runs. --- .../route.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 2caf4f95802..0364e9ca1b7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -854,7 +854,7 @@ function QueuesWithMetricsView() { "w-[1%]", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit @@ -1918,7 +1918,7 @@ function ClassicQueuesView() { "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit From a46c6e022095d8924cbfc209fbbcb6bb2a9f8c53 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:11:06 +0100 Subject: [PATCH 08/77] refactor(webapp,core,clickhouse): combined concurrency in responses, dashboard and metrics Queue API responses expose concurrency.combined, the dashboard says combined, and the new metrics columns are named combined_running and combined_limit. --- .../v3/QueueRetrievePresenter.server.ts | 2 +- .../route.tsx | 28 ++++++++--------- .../route.tsx | 10 +++---- apps/webapp/app/v3/querySchemas.ts | 10 +++---- apps/webapp/app/v3/queueMetricsMapping.ts | 4 +-- ...dd_queue_metrics_combined_concurrency.sql} | 30 +++++++++---------- .../clickhouse/src/queueMetrics.ts | 4 +-- packages/core/src/v3/schemas/queues.ts | 12 ++++---- 8 files changed, 50 insertions(+), 50 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_total_concurrency.sql => 042_add_queue_metrics_combined_concurrency.sql} (91%) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 04616a9cc0c..9d6e1d17712 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -178,7 +178,7 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, - total: + combined: data.totalConcurrencyLimit !== undefined ? { current: data.totalConcurrencyLimit, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 0364e9ca1b7..a8ba01515a0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -707,7 +707,7 @@ function QueuesWithMetricsView() { Limit @@ -853,10 +853,10 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -885,14 +885,14 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} @@ -1807,7 +1807,7 @@ function ClassicQueuesView() { Running Limit @@ -1917,10 +1917,10 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -1939,14 +1939,14 @@ function ClassicQueuesView() { )} > {limit} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ff4fc786bc5..8e445fa6111 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -396,7 +396,7 @@ export default function Page() { ids={ids} timeRange={timeRange} queueName={fullName} - hasTotalLimit={queue.concurrency?.total?.current != null} + hasTotalLimit={queue.concurrency?.combined?.current != null} /> )} @@ -494,25 +494,25 @@ function OverviewCharts({ /> {hasTotalLimit ? ( Runs in flight across ALL concurrency keys ( - ) versus the queue's total limit ( + ) versus the queue's combined limit ( ). } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_combined_running) AS running, least(max(max_combined_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} series={[ - { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "cap", label: "Combined limit", color: COLORS.limit }, { key: "running", label: "Running", color: COLORS.running }, ]} thresholdStroke={{ diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 8267a0f8020..05cd7f0b394 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,19 +770,19 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, - max_total_running: { - name: "max_total_running", + max_combined_running: { + name: "max_combined_running", ...column("UInt32", { description: "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", fillMode: "carry", }), }, - max_total_limit: { - name: "max_total_limit", + max_combined_limit: { + name: "max_combined_limit", ...column("UInt32", { description: - "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + "The queue's combined concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index d341f4a63cd..f093dc3f027 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,8 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), - total_running: num(f.tcc), - total_limit: num(f.tlim), + combined_running: num(f.tcc), + combined_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql similarity index 91% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql rename to internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index f45e6421ae3..03cb133799a 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -1,22 +1,22 @@ -- +goose Up --- Total-concurrency gauges: total_running is the in-flight count across ALL --- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- Total-concurrency gauges: combined_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 - ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, - ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS combined_limit UInt32 DEFAULT 0; ALTER TABLE trigger_dev.queue_metrics_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_5m_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_ck_v1 ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); @@ -45,8 +45,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -73,8 +73,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -104,9 +104,9 @@ DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; -ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS combined_running, DROP COLUMN IF EXISTS combined_limit; -- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps -- feeding every aggregate table after a rollback. diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index aa3cf5296d2..f3a6be695e4 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,8 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), - total_running: z.number().optional(), - total_limit: z.number().optional(), + combined_running: z.number().optional(), + combined_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index dc1b6e1de7c..4ae190e76a5 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,16 +45,16 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), - /** The total concurrency cap across all concurrencyKey values of the queue */ - total: z + /** The combined concurrency cap across all concurrencyKey values of the queue */ + combined: z .object({ - /** The effective/current total concurrency limit (null = no cap) */ + /** The effective/current combined concurrency limit (null = no cap) */ current: z.number().nullable(), - /** The declared total limit an override reverts to on reset */ + /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), - /** The overridden total limit, when an override is active */ + /** The overridden combined limit, when an override is active */ override: z.number().nullable(), - /** When the total override was applied */ + /** When the combined override was applied */ overriddenAt: z.coerce.date().nullable(), /** Runs currently in flight across all concurrencyKey values */ running: z.number().nullable(), From 451d4c92c7548d8030165b2b61d4f4d9e21cd9a9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:35:52 +0100 Subject: [PATCH 09/77] feat(webapp): bracketed combined limit in the Limit column Queues that set a combinedConcurrencyLimit show it bracketed next to the per-key limit with a fine dashed underline and an explanatory tooltip; the Limit header tooltip is width-capped. Queues without one are unchanged. --- .../route.tsx | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index a8ba01515a0..bc2304b9629 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -707,7 +707,8 @@ function QueuesWithMetricsView() { Limit @@ -886,14 +887,32 @@ function QueuesWithMetricsView() { limit )} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Running Limit @@ -1940,14 +1960,32 @@ function ClassicQueuesView() { > {limit} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Date: Sat, 29 Aug 2026 19:40:12 +0100 Subject: [PATCH 10/77] fix(webapp): combined-limit tooltip renders beside the cell link The tooltip trigger is a button, so nesting it in the Limit cell's link made clicking it navigate; it now renders as the cell's trailing adornment. --- .../route.tsx | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index bc2304b9629..ee2354ea1af 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -875,6 +875,39 @@ function QueuesWithMetricsView() { queue.paused ? "opacity-50" : undefined, queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} + // The combined-limit hint is a tooltip button, so it renders beside the + // link (trailing) rather than nested inside the ; the number stays the + // link. + trailingContent={ + queue.concurrency?.combined?.current != null ? ( + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> + ) : undefined + } > {queue.concurrencyLimitOverridePercent !== null ? ( <> @@ -886,34 +919,6 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.combined?.current != null ? ( - - ( - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )} - ) - - } - content={ - <> - Combined limit: at most{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. - - } - className="max-w-[260px]" - /> - ) : null} Date: Mon, 31 Aug 2026 10:30:33 +0100 Subject: [PATCH 11/77] Better tooltip message --- .../route.tsx | 550 +++++++++++++----- 1 file changed, 402 insertions(+), 148 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index ee2354ea1af..748be64dee1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,7 +8,10 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { + type ActionFunctionArgs, + type LoaderFunctionArgs, +} from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -22,10 +25,19 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTrigger, +} from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { + NavBar, + PageAccessories, + PageTitle, +} from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -55,7 +67,10 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { + redirectWithErrorMessage, + redirectWithSuccessMessage, +} from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -64,12 +79,18 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { + TimeFilter, + timeFilterFromTo, +} from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { + Chart, + type ChartConfig, +} from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -115,6 +136,7 @@ import { import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; import { isQueueAtCapacity } from "~/components/queues/queue-thresholds"; import { pageMeta } from "~/utils/pageTitle"; +import { InlineCode } from "~/components/code/InlineCode"; const SearchParamsSchema = z.object({ query: z.string().optional(), @@ -145,14 +167,19 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams) + Object.fromEntries(url.searchParams), ); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -181,7 +208,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays + maxPeriodDays, ); try { @@ -213,18 +240,23 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name + q.type === "task" ? `task/${q.name}` : q.name, ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ period: - resolveQueueMetricsPeriod({ period, from, to, defaultPeriod, maxPeriodDays }) ?? - undefined, + resolveQueueMetricsPeriod({ + period, + from, + to, + defaultPeriod, + maxPeriodDays, + }) ?? undefined, from: parseFiniteInt(from), to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays + maxPeriodDays, ); const queueMetrics = queueNames.length > 0 @@ -243,18 +275,25 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; } } catch (error) { - logger.warn("Queue list metrics unavailable, rendering without them", { error }); + logger.warn("Queue list metrics unavailable, rendering without them", { + error, + }); } } // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited> | null = null; + let allocation: Awaited< + ReturnType + > | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn("Queue allocation summary unavailable, rendering without it", { error }); + logger.warn( + "Queue allocation summary unavailable, rendering without it", + { error }, + ); } } @@ -272,7 +311,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: "Something went wrong, if this problem persists please contact support.", + statusText: + "Something went wrong, if this problem persists please contact support.", }); } }; @@ -283,13 +323,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method" + "Wrong method", ); } - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -312,7 +357,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); + return redirectWithErrorMessage( + redirectPath, + request, + "This branch is archived", + ); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -335,7 +384,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment paused", + ); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -343,10 +396,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment resumed", + ); } default: - return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); + return redirectWithErrorMessage( + redirectPath, + request, + "Something went wrong", + ); } }; @@ -359,14 +420,19 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === environment.concurrencyLimit * environment.burstFactor + environment.running === + environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; + limitStatus === "burst" + ? "text-warning" + : limitStatus === "limit" + ? "text-error" + : undefined; return { limitStatus, limitClassName }; } @@ -375,7 +441,11 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? : ; + return queueMetricsUiEnabled ? ( + + ) : ( + + ); } function QueuesWithMetricsView() { @@ -435,20 +505,23 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - } + }, ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; + const lastLiveBucketMs = lastLiveBlockRow + ? tileTimeToMs(lastLiveBlockRow.t) + : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS + LIVE_GAUGE_FRESH_MS, ); - const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = + lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -461,7 +534,8 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = + envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ @@ -522,7 +596,11 @@ function QueuesWithMetricsView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={ @@ -540,7 +618,9 @@ function QueuesWithMetricsView() { /> } - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} - + Including {envRunningLive - environment.concurrencyLimit}{" "} + burst runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -592,13 +675,21 @@ function QueuesWithMetricsView() { } value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} - suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} + suffix={ + allocation + ? `${allocationPct}% of the environment limit` + : undefined + } suffixClassName="text-text-dimmed" /> 1 ? `bursts up to ${burstLimit}` : undefined} + suffix={ + environment.burstFactor > 1 + ? `bursts up to ${burstLimit}` + : undefined + } suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -613,7 +704,10 @@ function QueuesWithMetricsView() { ) : ( Limit @@ -719,16 +814,17 @@ function QueuesWithMetricsView() { tooltip={

- Environment: uses the environment - limit of {environment.concurrencyLimit}. + Environment: + uses the environment limit of{" "} + {environment.concurrencyLimit}.

- User: a limit you set in your - code. + User: a limit + you set in your code.

- Override: a limit you set here or - via the API. + Override: a + limit you set here or via the API.

} @@ -748,8 +844,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time. marks - where the queue was throttled. + How many runs were waiting, over the selected time.{" "} + marks where the queue was throttled. } > @@ -763,16 +859,22 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath(organization, project, env, { - friendlyId: queue.id, - }); + const queueDetailPath = v3QueuePath( + organization, + project, + env, + { + friendlyId: queue.id, + }, + ); return ( ) : ( ) @@ -811,7 +913,9 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( } + button={ + + } content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -820,11 +924,16 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -842,7 +951,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -858,10 +967,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" - : queue.running > 0 && "text-text-bright" + : queue.running > 0 && "text-text-bright", )} > {queue.running} @@ -873,7 +982,8 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -888,7 +998,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -898,10 +1008,11 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -913,7 +1024,11 @@ function QueuesWithMetricsView() { <> {limit} - ({formatOverridePercent(queue.concurrencyLimitOverridePercent)}%) + ( + {formatOverridePercent( + queue.concurrencyLimitOverridePercent, + )} + %) ) : ( @@ -924,7 +1039,10 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + )} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -934,7 +1052,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent + queue.concurrencyLimitOverridePercent, )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -994,7 +1112,9 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 ? "time" : "times" + queueMetric.throttledTotal === 1 + ? "time" + : "times" } in this period` : "Peak queued in this period" } @@ -1002,8 +1122,16 @@ function QueuesWithMetricsView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -1056,7 +1184,9 @@ function QueuesWithMetricsView() { /> } @@ -1069,7 +1199,9 @@ function QueuesWithMetricsView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -1099,7 +1231,8 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + navigation.formData?.get("action") === + (env.paused ? "environment-resume" : "environment-pause"), ); return ( @@ -1114,7 +1247,9 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={env.paused ? "text-success" : "text-warning"} + leadingIconClassName={ + env.paused ? "text-success" : "text-warning" + } className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1142,13 +1277,15 @@ function EnvironmentPauseResumeButton({
- {env.paused ? "Resume environment?" : "Pause environment?"} + + {env.paused ? "Resume environment?" : "Pause environment?"} +
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env + env, )}. Any executing runs will continue to run.`}
setIsOpen(false)}> @@ -1164,7 +1301,13 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? : env.paused ? PlayIcon : PauseIcon + isLoading ? ( + + ) : env.paused ? ( + PlayIcon + ) : ( + PauseIcon + ) } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1188,7 +1331,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined + formData: FormData | undefined, ) { if (!formMethod || !formData) { return false; @@ -1202,7 +1345,13 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ; + return ( + + ); } type MetricTileRow = Record; @@ -1277,7 +1426,10 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); + return points.reduce( + (max, p) => (p.value === null ? max : Math.max(max, p.value)), + 0, + ); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1290,8 +1442,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns above 100%, - when it's into burst capacity. + How much of the environment's concurrency is in use. Turns{" "} + above 100%, when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1300,17 +1452,23 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), + formatValue: (v) => + v > 100 ? `${v}% — over the environment limit` : `${v}%`, formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: + limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); - return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v}% peak`, + }; }, }, { @@ -1324,7 +1482,11 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ bucket: tileTimeToMs(r.t), value: tileNumber(r.queued), })); - return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v.toLocaleString()} peak`, + }; }, }, { @@ -1332,8 +1494,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns {" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns{" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1362,8 +1524,9 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), - 0 + (max, r) => + tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, + 0, ); return { total: worst, @@ -1377,7 +1540,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: "The share of the selected window with at least one blocked dequeue.", + totalTooltip: + "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1398,7 +1562,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = + rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1464,10 +1629,13 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = + points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && + plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = + tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1486,11 +1654,12 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor] + [tile.id, tile.label, lineColor], ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = + data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1525,7 +1694,7 @@ function QueueEnvMetricChart({ {peak} @@ -1539,7 +1708,7 @@ function QueueEnvMetricChart({ {peak} @@ -1586,7 +1755,9 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} + yAxisProps={ + tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined + } tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1618,11 +1789,21 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; +type QueueHealthLabel = + | "Paused" + | "At capacity" + | "Backlogged" + | "Active" + | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ + paused, + running, + queued, + limit, +}: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1633,8 +1814,10 @@ function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): Queu // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": + "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: + "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1645,7 +1828,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1667,14 +1850,21 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) + ? percent.toString() + : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = - useTypedLoaderData(); + const { + environment, + queues, + pagination, + hasFilters, + autoReloadPollIntervalMs, + } = useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1683,7 +1873,8 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = + getEnvConcurrencyLimitStatus(environment); return ( @@ -1708,7 +1899,11 @@ function ClassicQueuesView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={
@@ -1730,7 +1925,9 @@ function ClassicQueuesView() { />
} - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {environment.running - environment.concurrencyLimit} burst runs{" "} - + Including{" "} + {environment.running - environment.concurrencyLimit} burst + runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1779,17 +1977,19 @@ function ClassicQueuesView() { - Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit{" "} + {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns + .canExceed ? ( ) : (
@@ -1831,7 +2040,7 @@ function ClassicQueuesView() { Running Limit @@ -1847,8 +2056,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's concurrency limit of{" "} - {environment.concurrencyLimit}. + This queue is limited by your environment's + concurrency limit of {environment.concurrencyLimit}.
@@ -1858,7 +2067,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in your code. + This queue is limited by a concurrency limit set in + your code.
@@ -1868,8 +2078,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually overridden from the - dashboard or API. + This queue's concurrency limit has been manually + overridden from the dashboard or API.
@@ -1885,7 +2095,8 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -1901,7 +2112,10 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -1911,17 +2125,26 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -1932,7 +2155,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -1946,11 +2169,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning" + isAtConcurrencyLimit && "text-warning", )} > {queue.running} @@ -1960,7 +2183,8 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {limit} @@ -1973,7 +2197,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -1983,10 +2207,11 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -1999,7 +2224,8 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {queue.concurrency?.overriddenAt ? ( @@ -2012,8 +2238,16 @@ function ClassicQueuesView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -2066,7 +2300,9 @@ function ClassicQueuesView() { /> } @@ -2079,7 +2315,9 @@ function ClassicQueuesView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -2110,3 +2348,19 @@ function BurstFactorTooltip({ /> ); } + +const limitTooltip = ( + <> + + How many runs can execute at once.{" "} + + + 1 (20) means 1 run + per concurrency key, but at most 20 runs across all keys. Set using{" "} + + combinedConcurrencyLimit + {" "} + in your code. + + +); From 8375efbca166f6148148873ccd162d83d366b739 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:37:07 +0100 Subject: [PATCH 12/77] docs(core): combined.current is the declared cap, clamped at admit time Also reflows an import to the formatter's current output. --- .../route.tsx | 512 +++++------------- 1 file changed, 144 insertions(+), 368 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 748be64dee1..12b29480751 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,10 +8,7 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { - type ActionFunctionArgs, - type LoaderFunctionArgs, -} from "@remix-run/server-runtime"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -25,19 +22,10 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { - Dialog, - DialogContent, - DialogHeader, - DialogTrigger, -} from "~/components/primitives/Dialog"; +import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { - NavBar, - PageAccessories, - PageTitle, -} from "~/components/primitives/PageHeader"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -67,10 +55,7 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { - redirectWithErrorMessage, - redirectWithSuccessMessage, -} from "~/models/message.server"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -79,18 +64,12 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { - TimeFilter, - timeFilterFromTo, -} from "~/components/runs/v3/SharedFilters"; +import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { - Chart, - type ChartConfig, -} from "~/components/primitives/charts/ChartCompound"; +import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -167,19 +146,14 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams), + Object.fromEntries(url.searchParams) ); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -208,7 +182,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays, + maxPeriodDays ); try { @@ -240,7 +214,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name, + q.type === "task" ? `task/${q.name}` : q.name ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ @@ -256,7 +230,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays, + maxPeriodDays ); const queueMetrics = queueNames.length > 0 @@ -283,17 +257,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited< - ReturnType - > | null = null; + let allocation: Awaited> | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn( - "Queue allocation summary unavailable, rendering without it", - { error }, - ); + logger.warn("Queue allocation summary unavailable, rendering without it", { error }); } } @@ -311,8 +280,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: - "Something went wrong, if this problem persists please contact support.", + statusText: "Something went wrong, if this problem persists please contact support.", }); } }; @@ -323,18 +291,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method", + "Wrong method" ); } - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -357,11 +320,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage( - redirectPath, - request, - "This branch is archived", - ); + return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -384,11 +343,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment paused", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -396,18 +351,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment resumed", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); } default: - return redirectWithErrorMessage( - redirectPath, - request, - "Something went wrong", - ); + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); } }; @@ -420,19 +367,14 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === - environment.concurrencyLimit * environment.burstFactor + environment.running === environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" - ? "text-warning" - : limitStatus === "limit" - ? "text-error" - : undefined; + limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; return { limitStatus, limitClassName }; } @@ -441,11 +383,7 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? ( - - ) : ( - - ); + return queueMetricsUiEnabled ? : ; } function QueuesWithMetricsView() { @@ -505,23 +443,20 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - }, + } ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow - ? tileTimeToMs(lastLiveBlockRow.t) - : NaN; + const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS, + LIVE_GAUGE_FRESH_MS ); - const freshLiveBlockRow = - lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -534,8 +469,7 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = - envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ @@ -596,11 +530,7 @@ function QueuesWithMetricsView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={ @@ -618,9 +548,7 @@ function QueuesWithMetricsView() { /> } - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit}{" "} - burst runs + Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -675,21 +600,13 @@ function QueuesWithMetricsView() { } value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} - suffix={ - allocation - ? `${allocationPct}% of the environment limit` - : undefined - } + suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} suffixClassName="text-text-dimmed" /> 1 - ? `bursts up to ${burstLimit}` - : undefined - } + suffix={environment.burstFactor > 1 ? `bursts up to ${burstLimit}` : undefined} suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -704,10 +621,7 @@ function QueuesWithMetricsView() { ) : (

- Environment: - uses the environment limit of{" "} - {environment.concurrencyLimit}. + Environment: uses the environment + limit of {environment.concurrencyLimit}.

- User: a limit - you set in your code. + User: a limit you set in your + code.

- Override: a - limit you set here or via the API. + Override: a limit you set here or + via the API.

} @@ -844,8 +756,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time.{" "} - marks where the queue was throttled. + How many runs were waiting, over the selected time. marks + where the queue was throttled. } > @@ -859,22 +771,16 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath( - organization, - project, - env, - { - friendlyId: queue.id, - }, - ); + const queueDetailPath = v3QueuePath(organization, project, env, { + friendlyId: queue.id, + }); return ( ) : ( ) @@ -913,9 +819,7 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( - } + button={} content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -924,16 +828,11 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -951,7 +850,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -967,10 +866,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" - : queue.running > 0 && "text-text-bright", + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -982,8 +881,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -998,7 +896,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -1008,11 +906,10 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -1024,10 +921,7 @@ function QueuesWithMetricsView() { <> {limit} - ( - {formatOverridePercent( - queue.concurrencyLimitOverridePercent, - )} + ({formatOverridePercent(queue.concurrencyLimitOverridePercent)} %) @@ -1039,10 +933,7 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn( - "w-[1%]", - queue.paused ? "opacity-50" : undefined, - )} + className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -1052,7 +943,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent, + queue.concurrencyLimitOverridePercent )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -1112,9 +1003,7 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 - ? "time" - : "times" + queueMetric.throttledTotal === 1 ? "time" : "times" } in this period` : "Peak queued in this period" } @@ -1122,16 +1011,8 @@ function QueuesWithMetricsView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -1184,9 +1065,7 @@ function QueuesWithMetricsView() { /> } @@ -1199,9 +1078,7 @@ function QueuesWithMetricsView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -1231,8 +1108,7 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === - (env.paused ? "environment-resume" : "environment-pause"), + navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") ); return ( @@ -1247,9 +1123,7 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={ - env.paused ? "text-success" : "text-warning" - } + leadingIconClassName={env.paused ? "text-success" : "text-warning"} className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1277,15 +1151,13 @@ function EnvironmentPauseResumeButton({ - - {env.paused ? "Resume environment?" : "Pause environment?"} - + {env.paused ? "Resume environment?" : "Pause environment?"}
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env, + env )}. Any executing runs will continue to run.`} setIsOpen(false)}> @@ -1301,13 +1173,7 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? ( - - ) : env.paused ? ( - PlayIcon - ) : ( - PauseIcon - ) + isLoading ? : env.paused ? PlayIcon : PauseIcon } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1331,7 +1197,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined, + formData: FormData | undefined ) { if (!formMethod || !formData) { return false; @@ -1345,13 +1211,7 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ( - - ); + return ; } type MetricTileRow = Record; @@ -1426,10 +1286,7 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce( - (max, p) => (p.value === null ? max : Math.max(max, p.value)), - 0, - ); + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1442,8 +1299,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns{" "} - above 100%, when it's into burst capacity. + How much of the environment's concurrency is in use. Turns above 100%, + when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1452,16 +1309,14 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => - v > 100 ? `${v}% — over the environment limit` : `${v}%`, + formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: - limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); return { @@ -1494,8 +1349,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns{" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns {" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1524,9 +1379,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => - tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, - 0, + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 ); return { total: worst, @@ -1540,8 +1394,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: - "The share of the selected window with at least one blocked dequeue.", + totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1562,8 +1415,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = - rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1629,13 +1481,10 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = - points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && - plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = - tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1654,12 +1503,11 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor], + [tile.id, tile.label, lineColor] ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = - data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1694,7 +1542,7 @@ function QueueEnvMetricChart({ {peak} @@ -1708,7 +1556,7 @@ function QueueEnvMetricChart({ {peak} @@ -1755,9 +1603,7 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={ - tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined - } + yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1789,21 +1635,11 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = - | "Paused" - | "At capacity" - | "Backlogged" - | "Active" - | "Idle"; +type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ - paused, - running, - queued, - limit, -}: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1814,10 +1650,8 @@ function queueHealthLabel({ // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": - "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: - "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1828,7 +1662,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1850,21 +1684,14 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) - ? percent.toString() - : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { - environment, - queues, - pagination, - hasFilters, - autoReloadPollIntervalMs, - } = useTypedLoaderData(); + const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = + useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1873,8 +1700,7 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = - getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); return ( @@ -1899,11 +1725,7 @@ function ClassicQueuesView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={
@@ -1925,9 +1747,7 @@ function ClassicQueuesView() { />
} - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including{" "} - {environment.running - environment.concurrencyLimit} burst - runs + Including {environment.running - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1977,19 +1796,17 @@ function ClassicQueuesView() { - Burst limit{" "} - {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns - .canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( ) : (
@@ -2056,8 +1864,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's - concurrency limit of {environment.concurrencyLimit}. + This queue is limited by your environment's concurrency limit of{" "} + {environment.concurrencyLimit}.
@@ -2067,8 +1875,7 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in - your code. + This queue is limited by a concurrency limit set in your code.
@@ -2078,8 +1885,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually - overridden from the dashboard or API. + This queue's concurrency limit has been manually overridden from the + dashboard or API.
@@ -2095,8 +1902,7 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -2112,10 +1918,7 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -2125,26 +1928,17 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -2155,7 +1949,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -2169,11 +1963,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning", + isAtConcurrencyLimit && "text-warning" )} > {queue.running} @@ -2183,8 +1977,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {limit} @@ -2197,7 +1990,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -2207,11 +2000,10 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -2224,8 +2016,7 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {queue.concurrency?.overriddenAt ? ( @@ -2238,16 +2029,8 @@ function ClassicQueuesView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -2300,9 +2083,7 @@ function ClassicQueuesView() { /> } @@ -2315,9 +2096,7 @@ function ClassicQueuesView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -2355,12 +2134,9 @@ const limitTooltip = ( How many runs can execute at once.{" "} - 1 (20) means 1 run - per concurrency key, but at most 20 runs across all keys. Set using{" "} - - combinedConcurrencyLimit - {" "} - in your code. + 1 (20) means 1 run per concurrency key, + but at most 20 runs across all keys. Set using{" "} + combinedConcurrencyLimit in your code. ); From a188bcb4f3676205ffea5232452e269b414e8f36 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:47:42 +0100 Subject: [PATCH 13/77] fix(run-engine): sample the combined gauge after batch admission The dequeue gauge ran before the loop, so a queue's first batch from idle and its final drain were never sampled with their runs in flight and the combined chart under-reported. The successful path now re-samples after admissions; early returns keep the entry sample. Also documents that combined.current is the declared cap, clamped at admit time. --- internal-packages/run-engine/src/run-queue/index.ts | 4 ++++ packages/core/src/v3/schemas/queues.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index a1aa60664f7..6eed6ee7a5c 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -5275,6 +5275,10 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} + return __qmret(results) `, }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 4ae190e76a5..8fbab8a29e0 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -48,7 +48,7 @@ export const QueueItem = z.object({ /** The combined concurrency cap across all concurrencyKey values of the queue */ combined: z .object({ - /** The effective/current combined concurrency limit (null = no cap) */ + /** The current combined concurrency limit as declared or overridden (null = no cap). Enforcement clamps it to the environment concurrency limit at admit time. */ current: z.number().nullable(), /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), From fdd0af1edac801555e2f09aa4052501aaf5cee17 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:17 +0100 Subject: [PATCH 14/77] refactor(run-engine,webapp): drop per-key override admit reads and limit column Removes the override-aware admit and gauge reads from the CK Lua scripts and the per-key limit column, following the removal of runtime per-key overrides from this stack. --- .../route.tsx | 29 +-- apps/webapp/app/v3/querySchemas.ts | 2 +- ...add_queue_metrics_combined_concurrency.sql | 2 +- .../run-engine/src/run-queue/index.ts | 170 ++---------------- 4 files changed, 16 insertions(+), 187 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 8e445fa6111..933baafc43a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -407,13 +407,7 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -966,15 +960,10 @@ function KeyStatsTable({ ids, timeRange, queueName, - defaultKeyLimit, - envLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; - /** The limit a key inherits when it has no override (the queue's limit, else the env limit). */ - defaultKeyLimit: number; - envLimit: number; }) { const { value, replace, del } = useSearchParams(); const selectedKey = value("key"); @@ -1017,12 +1006,6 @@ function KeyStatsTable({ Key Queued now Running now - - Limit - Oldest wait Started Peak backlog @@ -1031,11 +1014,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -1049,12 +1032,6 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} - - {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} - {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 05cd7f0b394..3ec1523c83f 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", fillMode: "carry", }), }, diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 03cb133799a..57c6b290efc 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -4,7 +4,7 @@ -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. +-- limit in queue_limit, surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 6eed6ee7a5c..bc804dc34b0 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -116,18 +116,12 @@ local function __gateReconcile(setKey, msgKeyPrefix, reconcileKeyPrefix) end end -local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix, ckOverridesEnabled) +local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix) if not msg.gates then return true end for _, gate in ipairs(msg.gates) do local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) - if ckOverridesEnabled and gateKey and gateKey ~= '' then - local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) - if gateOverride then - perKeyLimit = math.min(tonumber(gateOverride), envLimit) - end - end if occupancy >= perKeyLimit and redis.call('SISMEMBER', variant .. ':currentConcurrency', messageId) == 0 then __gateReconcile(variant .. ':currentConcurrency', msgKeyPrefix, gatesKeyPrefix) return false @@ -225,8 +219,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: - "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -273,13 +266,6 @@ export interface RunQueueMetricsEmitter { emitGauge(shardKey: string, fields: Record): void; } -export class RunQueueConcurrencyKeyLimitExceededError extends Error { - constructor(message: string) { - super(message); - this.name = "RunQueueConcurrencyKeyLimitExceededError"; - } -} - export type RunQueueOptions = { name: string; tracer: Tracer; @@ -344,7 +330,6 @@ export type RunQueueOptions = { */ gatesEnabled?: boolean; /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ - maxConcurrencyKeyOverridesPerQueue?: number; workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -458,7 +443,6 @@ export class RunQueue { private queueSelectionStrategy: RunQueueSelectionStrategy; private shardCount: number; private counterTtlSeconds: number; - private maxConcurrencyKeyOverridesPerQueue: number; private abortController: AbortController; private worker: Worker; private workerQueueResolver: WorkerQueueResolver; @@ -469,7 +453,6 @@ export class RunQueue { constructor(public readonly options: RunQueueOptions) { this.shardCount = options.shardCount ?? 2; this.counterTtlSeconds = options.counterTtlSeconds ?? 86400; - this.maxConcurrencyKeyOverridesPerQueue = options.maxConcurrencyKeyOverridesPerQueue ?? 1000; this.retryOptions = options.retryOptions ?? defaultRetrySettings; this.redis = createRedisClient(options.redis, { onError: (error) => { @@ -664,85 +647,6 @@ export class RunQueue { return this.redis.scard(this.keys.queueGroupConcurrencyKey(env, queue)); } - /** - * Sets a per-concurrency-key limit override for a queue. The stored value is the - * raw requested limit; admit paths clamp to the environment limit at read time. - * Throws RunQueueConcurrencyKeyLimitExceededError when a NEW key would push the - * queue past maxConcurrencyKeyOverridesPerQueue (updates to existing keys always - * succeed). - */ - public async updateQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string, - limit: number - ) { - const result = await this.redis.setQueueConcurrencyKeyLimit( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey), - String(limit), - String(this.maxConcurrencyKeyOverridesPerQueue) - ); - - if (result === 0) { - throw new RunQueueConcurrencyKeyLimitExceededError( - `Cannot add a concurrency key override to queue ${queue}: the queue already has ${this.maxConcurrencyKeyOverridesPerQueue} overrides` - ); - } - } - - public async removeQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string - ) { - return this.redis.hdel( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey) - ); - } - - /** Returns the raw per-concurrency-key limit overrides for a queue, keyed by concurrency key value. */ - public async getQueueConcurrencyKeyLimits( - env: MinimalAuthenticatedEnvironment, - queue: string - ): Promise> { - const raw = await this.redis.hgetall(this.keys.queueCkLimitsKey(env, queue)); - - const limits: Record = {}; - for (const [variantName, value] of Object.entries(raw)) { - const ckIndex = variantName.indexOf(":ck:"); - if (ckIndex === -1) { - continue; - } - limits[variantName.slice(ckIndex + 4)] = Number(value); - } - return limits; - } - - /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ - public async getQueueConcurrencyKeyLimitsForKeys( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKeys: string[] - ): Promise> { - if (concurrencyKeys.length === 0) { - return {}; - } - - const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); - const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); - - const limits: Record = {}; - concurrencyKeys.forEach((key, index) => { - const value = values[index]; - if (value != null) { - limits[key] = Number(value); - } - }); - return limits; - } - /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, @@ -2536,7 +2440,6 @@ export class RunQueue { const totalConcurrencyLimitKey = this.keys.queueTotalConcurrencyLimitKeyFromQueue( message.queue ); - const ckLimitsKey = this.keys.queueCkLimitsKeyFromQueue(message.queue); const totalConcurrencyEnabledArg = this.options.totalConcurrencyEnabled ? "1" : "0"; if (ttlInfo) { @@ -2560,7 +2463,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2600,7 +2502,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2891,7 +2792,6 @@ export class RunQueue { runningCounterKey, this.keys.queueGroupConcurrencyKeyFromQueue(ckWildcardQueue), this.keys.queueTotalConcurrencyLimitKeyFromQueue(ckWildcardQueue), - this.keys.queueCkLimitsKeyFromQueue(ckWildcardQueue), //args ckWildcardQueue, String(Date.now()), @@ -3788,7 +3688,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -3903,7 +3803,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4182,7 +4082,7 @@ return __qmret(0) // *Tracked variants of dequeueMessageFromKey and the ack/nack/dlq/release/clear // scripts. this.redis.defineCommand("enqueueMessageCkTracked", { - numberOfKeys: 18, + numberOfKeys: 17, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4204,7 +4104,6 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] -local ckLimitsKey = KEYS[18] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4243,10 +4142,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4270,7 +4165,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4359,7 +4254,7 @@ return __qmret(0) }); this.redis.defineCommand("enqueueMessageWithTtlCkTracked", { - numberOfKeys: 19, + numberOfKeys: 18, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4382,7 +4277,6 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] -local ckLimitsKey = KEYS[19] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4423,10 +4317,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4448,7 +4338,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4851,7 +4741,7 @@ for i = 1, #messages, 2 do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if gatesAllow then @@ -5053,7 +4943,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 14, + numberOfKeys: 13, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -5068,7 +4958,6 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] -local ckLimitsKey = KEYS[14] local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5160,12 +5049,6 @@ for _, ckQueueName in ipairs(ckQueues) do local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') local perKeyLimit = queueConcurrencyLimit - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, ckQueueName) - if perKeyOverride then - perKeyLimit = math.min(tonumber(perKeyOverride), envConcurrencyLimit) - end - end if ckCurrentConcurrency >= perKeyLimit then -- Back a blocked variant off so it cannot pin the bounded candidate window @@ -5200,7 +5083,7 @@ for _, ckQueueName in ipairs(ckQueues) do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if not gatesAllow then blockedByGates = true @@ -6102,26 +5985,6 @@ __gatesRelease(keyPrefix, redis.call('GET', messageKey), messageId) `, }); - this.redis.defineCommand("setQueueConcurrencyKeyLimit", { - numberOfKeys: 1, - lua: ` -local ckLimitsKey = KEYS[1] - -local fieldName = ARGV[1] -local limit = ARGV[2] -local maxFields = tonumber(ARGV[3]) - -if redis.call('HEXISTS', ckLimitsKey, fieldName) == 0 then - if redis.call('HLEN', ckLimitsKey) >= maxFields then - return 0 - end -end - -redis.call('HSET', ckLimitsKey, fieldName, limit) -return 1 -`, - }); - this.redis.defineCommand("updateEnvironmentConcurrencyLimits", { numberOfKeys: 2, lua: ` @@ -6486,14 +6349,6 @@ declare module "@internal/redis" { callback?: Callback ): Result; - setQueueConcurrencyKeyLimit( - ckLimitsKey: string, - fieldName: string, - limit: string, - maxFields: string, - callback?: Callback - ): Result; - updateEnvironmentConcurrencyLimits( // keys envConcurrencyLimitKey: string, @@ -6680,7 +6535,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6718,7 +6572,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6753,7 +6606,6 @@ declare module "@internal/redis" { runningCounterKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, From d9f64125f734748773f0c4b48847cb8f3c724c54 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:38 +0100 Subject: [PATCH 15/77] refactor(webapp): concurrency keys resource stops reading per-key overrides --- .../app/routes/resources.queues.concurrency-keys.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 8c590554e51..67c2b9f500a 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,8 +43,6 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; - /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ - limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -153,11 +151,8 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. - const [live, keyLimitOverrides] = await Promise.all([ - engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), - ]); + // Enrich just this page's keys with live "now" counts from Redis. + const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -173,7 +168,6 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, - limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); From 36e873b232471e4f35e18662b771243d806af4ee Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:11:29 +0100 Subject: [PATCH 16/77] refactor(run-engine): drop the now-unreferenced ck-limits key builders --- internal-packages/run-engine/src/run-queue/keyProducer.ts | 8 -------- internal-packages/run-engine/src/run-queue/types.ts | 3 --- 2 files changed, 11 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 120e04f8c38..98028f5af7b 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -366,14 +366,6 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { - return `${this.queueKey(env, queue)}:ckLimits`; - } - - queueCkLimitsKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:ckLimits`; - } - isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index ad358a04cbb..75651a1f847 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -112,9 +112,6 @@ export interface RunQueueKeyProducer { queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; - queueCkLimitsKeyFromQueue(queue: string): string; - //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; envCurrentConcurrencyKey(env: RunQueueKeyProducerEnvironment): string; From aca9667a062d2ec177fe47e34399544587977265 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:44:36 +0100 Subject: [PATCH 17/77] chore: lift the run-queue knip ignore The class the ignore covered is deleted at this level, so the merged result carries no dead-code exemption. --- knip.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/knip.json b/knip.json index c8e7f4027dd..84456756ca1 100644 --- a/knip.json +++ b/knip.json @@ -27,9 +27,6 @@ ], "ignoreDependencies": ["@sentry/cli", "assert", "util"] }, - "internal-packages/run-engine": { - "ignore": ["src/run-queue/index.ts"] - }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], "ignoreBinaries": ["rg"] From 9671fb4c3bc8959574f249a616965f114d239c89 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:19 +0100 Subject: [PATCH 18/77] fix(run-engine,webapp,clickhouse): review fixes for the metrics tier The plain dequeue gauge re-samples after admissions like the keyed one, the repair path clears a keyed run's variant and group slots by concurrency key, pause responses include the combined limit, stale wording and a leftover changeset from before the rename are cleaned up, and two empty flag blocks are removed from the fast-path scripts. --- apps/webapp/app/v3/querySchemas.ts | 2 +- .../app/v3/services/pauseQueue.server.ts | 3 ++ ...add_queue_metrics_combined_concurrency.sql | 5 +-- .../run-engine/src/engine/index.ts | 2 ++ .../run-engine/src/run-queue/index.ts | 34 +++++++++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 3ec1523c83f..b33b205cc3c 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket (1000000 = no explicit limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/services/pauseQueue.server.ts b/apps/webapp/app/v3/services/pauseQueue.server.ts index 87d2d339864..e379ef3d754 100644 --- a/apps/webapp/app/v3/services/pauseQueue.server.ts +++ b/apps/webapp/app/v3/services/pauseQueue.server.ts @@ -101,6 +101,9 @@ export class PauseQueueService extends BaseService { concurrencyLimitOverriddenAt: updatedQueue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: updatedQueue.paused, + totalConcurrencyLimit: updatedQueue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: updatedQueue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: updatedQueue.totalConcurrencyLimitOverriddenAt ?? null, }), }; } catch (error) { diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 57c6b290efc..dd0c4c53b34 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -3,8 +3,9 @@ -- Total-concurrency gauges: combined_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on --- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit, surfaced in the ck tier as max_limit. +-- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency +-- limit that applied in queue_limit, surfaced in the ck tier as max_limit +-- (1000000 = no explicit limit). ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 1909df7e9c6..6347a2d1e60 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -3001,6 +3001,7 @@ export class RunEngine { { select: { queue: true, + concurrencyKey: true, }, }, this.prisma @@ -3022,6 +3023,7 @@ export class RunEngine { runId, orgId: latestSnapshot.organizationId, queue: taskRun.queue, + concurrencyKey: taskRun.concurrencyKey ?? undefined, env: { id: latestSnapshot.environmentId, type: latestSnapshot.environmentType, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index bc804dc34b0..3a7d7af4a39 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -329,7 +329,6 @@ export type RunQueueOptions = { * the total cap covering releases from builds without the mirror. */ gatesEnabled?: boolean; - /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -1547,6 +1546,7 @@ export class RunQueue { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { return this.#callClearMessageFromConcurrencySets(params); @@ -3082,18 +3082,30 @@ export class RunQueue { runId, orgId, queue, + concurrencyKey, env, }: { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { const messageId = runId; const messageKey = this.keys.messageKey(orgId, messageId); - const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey(env, queue); + /** + * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's + * slots live on the ck variant, and the tracked clear additionally mirrors the + * group set and counters that only keyed queues maintain. + */ + const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue; + const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey( + env, + queue, + concurrencyKey + ); const envCurrentConcurrencyKey = this.keys.envCurrentConcurrencyKey(env); - const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue); + const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue, concurrencyKey); const envCurrentDequeuedKey = this.keys.envCurrentDequeuedKey(env); this.logger.debug("Calling clearMessageFromConcurrencySets", { @@ -3108,15 +3120,15 @@ export class RunQueue { service: this.name, }); - if (queue.includes(":ck:")) { + if (fullQueue.includes(":ck:")) { return this.redis.clearMessageFromConcurrencySetsTracked( queueCurrentConcurrencyKey, envCurrentConcurrencyKey, queueCurrentDequeuedKey, envCurrentDequeuedKey, - this.keys.queueRunningCounterKeyFromQueue(queue), - this.keys.ckIndexKeyFromQueue(queue), - this.keys.queueGroupConcurrencyKeyFromQueue(queue), + this.keys.queueRunningCounterKeyFromQueue(fullQueue), + this.keys.ckIndexKeyFromQueue(fullQueue), + this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), messageKey, messageId, this.options.redis.keyPrefix ?? "", @@ -4141,8 +4153,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: a fast-path admit consumes a group slot, so it must @@ -4316,8 +4326,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: see enqueueMessageCkTracked. @@ -4783,6 +4791,10 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_GAUGE_LUA} + -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, From 2bc50f788eab31fbfc66c0ba9a59aa43935609eb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:43 +0100 Subject: [PATCH 19/77] chore: drop the pre-rename changeset superseded by the combined one --- .changeset/queue-total-concurrency-stats.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/queue-total-concurrency-stats.md diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md deleted file mode 100644 index a70da24d1fb..00000000000 --- a/.changeset/queue-total-concurrency-stats.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. From 2d15c4fd71b19fa6be97f0b0e30b519ff3ed9853 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:25:54 +0100 Subject: [PATCH 20/77] perf(run-engine): share the combined-limit read between the admit gate and gauges The CK enqueue and dequeue scripts read the combined concurrency limit key once for admission and again for the metrics gauge tail. A per-call memo makes whichever runs first do the single GET; limits cannot change mid-script, so the value stays exact. Group cardinality remains a fresh read because gauges must reflect post-admission state. --- .../run-engine/src/run-queue/index.ts | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 3a7d7af4a39..b25b7552070 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -207,11 +207,12 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { }; // Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. -// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually -// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +// Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with +// the total-cap gate); the CK scripts that run this (the Tracked variants and the CK +// dequeue) declare both. The group SCARD stays a fresh read: it must be post-admission. const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + totalLimit: "__totalLimitRaw() or '0'", }; // CK enqueue variants of the two gauges above, extended with the CK-health tail. @@ -4116,6 +4117,13 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4160,7 +4168,7 @@ if enableFastPath == '1' then -- slow path (the message queues; the dequeue gate holds it). local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4287,6 +4295,13 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4331,7 +4346,7 @@ if enableFastPath == '1' then -- Total-cap gate: see enqueueMessageCkTracked. local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4970,6 +4985,13 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5012,7 +5034,7 @@ local actualMaxCount = math.min(maxCount, envAvailableCapacity) -- behind, and blocking on it would deadlock the run against itself). local totalHeadroom = nil if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalConcurrencyLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') From 5d8db93c9b77727f1fbbeb5417e39dc8ff8ae2dd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:52:44 +0100 Subject: [PATCH 21/77] perf(run-engine): dequeue gauges sample once, at return The gauge slot is single-valued and the last write wins, so on the success path the post-admission resample made the entry sample pure waste. A return wrapper computes the gauge exactly once per call at exit, keeping every emitted value identical while dropping the discarded reads (about six per plain dequeue, ten per keyed dequeue at full sampling). --- .../run-engine/src/run-queue/index.ts | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index b25b7552070..d5b7b031224 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -171,8 +171,9 @@ local __qm_g = false local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; // Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before -// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the -// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV). +// return 0) and the base dequeue's sample-at-return wrapper. Gated on the last ARGV so it +// is inert unless the caller opts in. CK queues emit per-subqueue depth (queue_name +// aggregates via the MV). const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", @@ -4694,7 +4695,16 @@ local gatesEnabled = ARGV[7] == '1' local totalConcurrencyEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end -- Check current env concurrency against the limit local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') @@ -4806,10 +4816,6 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_GAUGE_LUA} - -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, @@ -5003,7 +5009,16 @@ local totalConcurrencyEnabled = ARGV[7] == '1' local gatesEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end local function decrLengthCounter() if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then @@ -5192,10 +5207,6 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} - return __qmret(results) `, }); From 9553fff7907714ac838819a07d9721fa6430de8d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:07:55 +0100 Subject: [PATCH 22/77] test(run-engine): pin dequeue-emitted gauges so a sampling regression fails the suite The gauge assertions were all satisfiable by enqueue-emitted gauges, so breaking the dequeue scripts' sample-at-return wrapper left the suite green. The base test now requires the post-admission reading (running 1, queued 0) and the CK test requires the wildcard aggregate only the CK dequeue emits. Verified by mutation: disabling the wrapper fails both. --- .../run-engine/src/run-queue/metrics.test.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index ebfc295470e..d16b8048d95 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -123,7 +123,10 @@ describe("RunQueue queue-metrics emission", () => { const entries = await waitForEntries(redis, definition, (es) => { const seen = es.map((e) => e.fields.op); - return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o)); + if (!["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o))) return false; + return es.some( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); }); const ops = entries.map((e) => e.fields.op); expect(ops).toContain("enqueue"); @@ -141,6 +144,14 @@ describe("RunQueue queue-metrics emission", () => { expect(gauge!.fields.ckq).toBeUndefined(); expect(gauge!.fields.ckw).toBeUndefined(); + // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the + // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse. + const dequeueGauge = entries.find( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); + assertGauge(dequeueGauge); + expect(dequeueGauge!.fields.q).toContain("task/my-task"); + // The first counter emission also seeds a cum=0 baseline (no wait); the real reading // carries wait. Pick the reading (cum > 0). const started = entries.find((e) => e.fields.op === "started" && Number(e.fields.cum) > 0); @@ -283,14 +294,13 @@ describe("RunQueue queue-metrics emission", () => { expect(dequeued?.messageId).toBe(message.runId); const entries = await waitForEntries(redis, definition, (es) => - es.some( - (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0" - ) + es.some((e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:*")) ); const gauges = entries.filter((e) => e.fields.op === "gauge"); expect(gauges.length).toBeGreaterThan(0); - // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr. - const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0"); + // The aggregate gauge targets the CK wildcard and only the CK dequeue script emits + // it, so this pins that script's sample-at-return wrapper. + const aggregate = gauges.find((e) => e.fields.q.includes(":ck:*")); assertGauge(aggregate); expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0); expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0); From 600610dd955d3ecd4f44cf0940b375d1ce694858 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:33:33 +0100 Subject: [PATCH 23/77] test(run-engine): wait for the metrics emitter connection before exercising it The emitter drops emissions until its Redis client is ready, and the tests enqueued immediately after constructing it, so the first counter entry was occasionally lost and the suite flaked roughly two runs in eighty. Awaiting readiness makes every emission land. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d16b8048d95..efd00cd6aa1 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -81,6 +81,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", @@ -183,6 +184,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -255,6 +257,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -354,6 +357,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From 6000cca094e4f95b942a67b71f408340ef4a971d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:49:08 +0100 Subject: [PATCH 24/77] test(run-engine,metrics-pipeline): bound emitter-readiness waits and cover the consumer round-trip An unreachable Redis leaves waitUntilReady pending forever, so the bounded wait fails fast with a descriptive error instead of burning the test timeout. The consumer round-trip test gains the same readiness wait its gauge sibling already had, closing the remaining first-emission drop flake. --- .../metrics-pipeline/src/consumer.test.ts | 1 + .../run-engine/src/run-queue/metrics.test.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index 672fa426999..cb111ee1534 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -43,6 +43,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit("queueA", { op: "enqueue", q: "queueA" }); emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 }); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index efd00cd6aa1..8b3872ddc2f 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -24,6 +24,17 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; +// A dead Redis leaves waitUntilReady() pending forever (the client retries +// indefinitely), which would burn the whole test timeout with no diagnostic. +async function emitterReady(emitter: MetricsStreamEmitter) { + await Promise.race([ + emitter.waitUntilReady(), + setTimeout(15_000).then(() => { + throw new Error("metrics emitter Redis connection never became ready"); + }), + ]); +} + async function readAllEntries( redisOptions: { host: string; @@ -81,7 +92,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", @@ -184,7 +195,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -257,7 +268,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -357,7 +368,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From cc1358e12900e17e4679e7643e4c392f495d3cb7 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:51:27 +0100 Subject: [PATCH 25/77] test(run-engine): abort the readiness race timer so its losing branch cannot reject unhandled --- .../run-engine/src/run-queue/metrics.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 8b3872ddc2f..4c3f17125cf 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,13 +26,16 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. +// Races values, not throws: a rejection in the losing branch of a settled race +// is an unhandled rejection, so the timer is aborted and swallowed instead. async function emitterReady(emitter: MetricsStreamEmitter) { - await Promise.race([ - emitter.waitUntilReady(), - setTimeout(15_000).then(() => { - throw new Error("metrics emitter Redis connection never became ready"); - }), - ]); + const abort = new AbortController(); + const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); + const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); + abort.abort(); + if (winner === "timeout") { + throw new Error("metrics emitter Redis connection never became ready"); + } } async function readAllEntries( From d0e5cfadbb389c86a9b575c28d7a8098215ac1fc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:56:07 +0100 Subject: [PATCH 26/77] test(run-engine): close the emitter when the readiness wait times out --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 4c3f17125cf..d82ddb35305 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,6 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { + await emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } } From 6212f6e12f98192b0b19484e4cf2c62536618538 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:01:29 +0100 Subject: [PATCH 27/77] test(metrics-pipeline,run-engine): readiness wait for the per-stream test, honest timer comment The per-stream batches test carried the same first-emission drop race as its siblings; it now waits for the emitter connection too. The readiness helper's comment claimed a losing race branch rejects unhandled, which is not how Promise.race behaves (it handles every input); the abort's real benefit is releasing the timer promptly. --- internal-packages/metrics-pipeline/src/consumer.test.ts | 1 + internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index cb111ee1534..f9f59335249 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -157,6 +157,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit(a, { op: "enqueue", q: a }); emitter.emit(b, { op: "enqueue", q: b }); await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d82ddb35305..1b47238db89 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,8 +26,8 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. -// Races values, not throws: a rejection in the losing branch of a settled race -// is an unhandled rejection, so the timer is aborted and swallowed instead. +// The abort releases the losing timer promptly so it cannot hold an event +// loop open for the remaining 15s after a fast ready. async function emitterReady(emitter: MetricsStreamEmitter) { const abort = new AbortController(); const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); From 8198c110a16d89667a33c6706deb1e339a4bb460 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:10:57 +0100 Subject: [PATCH 28/77] test(run-engine): fire-and-forget the emitter close on readiness timeout A quit written to a socket that accepted but never completes the handshake never settles, which made the diagnostic throw unreachable. Closing without awaiting keeps the fast, descriptive failure. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 1b47238db89..edae6f8cc30 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,7 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { - await emitter.close().catch(() => {}); + void emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } } From bb24193b6595b6dbce8f9f7b4f2be8cd415b6b21 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:19:40 +0100 Subject: [PATCH 29/77] rename(webapp,clickhouse): concurrency vocabulary for the unlaunched metrics surfaces The Query tables become concurrency_metrics and concurrency_metrics_by_key, and the combined_* columns become total_* across the query schemas, the insert mapping, the ClickHouse migration, and every TRQL chart, matching the perKey/total shape the SDK will expose. Internal names (stream keys, env vars, ClickHouse table names) are unchanged. None of these surfaces have launched, so this is the last free moment for the rename. --- .../components/queues/QueueMetricCards.tsx | 2 +- .../presenters/v3/BuiltInDashboards.server.ts | 10 ++--- .../v3/reports/health/health-data.ts | 4 +- .../presenters/v3/reports/report-registry.ts | 4 +- .../route.tsx | 4 +- .../route.tsx | 43 ++++++++++--------- .../route.tsx | 2 +- .../api.v1.queues.$queueParam.metrics.ts | 2 +- .../route.tsx | 4 +- apps/webapp/app/v3/querySchemas.ts | 16 +++---- apps/webapp/app/v3/queueMetricsMapping.ts | 4 +- apps/webapp/test/reportHealthData.test.ts | 4 +- ...2_add_queue_metrics_total_concurrency.sql} | 30 ++++++------- .../clickhouse/src/queueMetrics.ts | 4 +- 14 files changed, 68 insertions(+), 65 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_combined_concurrency.sql => 042_add_queue_metrics_total_concurrency.sql} (91%) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 7c87a1e2d02..6b6692bcd9e 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -340,7 +340,7 @@ export function QueueSidebarStats({ }; const { rows, showLoading } = useQueueMetric( - `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`, + `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM concurrency_metrics`, { ids, timeRange, queueName, defaultPeriod } ); const row = rows[0]; diff --git a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts index d831568248d..7c609322ab2 100644 --- a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts +++ b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts @@ -634,7 +634,7 @@ const queuesDashboard: BuiltInDashboard = { "t-pressure": { title: "Queue pressure", query: "", display: { type: "title" } }, pressure: { title: "Queue pressure", - query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM queue_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, + query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM concurrency_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, display: { type: "table", prettyFormatting: true, @@ -644,7 +644,7 @@ const queuesDashboard: BuiltInDashboard = { "t-trends": { title: "Per-queue trends", query: "", display: { type: "title" } }, "running-q": { title: "Running by queue", - query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's running across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -661,7 +661,7 @@ const queuesDashboard: BuiltInDashboard = { }, "queued-q": { title: "Queue depth (backlog) by queue", - query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's backlog across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -678,7 +678,7 @@ const queuesDashboard: BuiltInDashboard = { }, "throttled-q": { title: "Throttled buckets by queue", - query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped counter: per-group zero-fill so idle buckets read 0, not a gap. fillGaps: true, display: { @@ -697,7 +697,7 @@ const queuesDashboard: BuiltInDashboard = { title: "Enqueued vs started", // Counter states merge per queue, then sum outside: a single merge across queues // mixes unrelated odometers and returns wrong totals. - query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM queue_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, + query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM concurrency_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, display: { type: "chart", chartType: "line", diff --git a/apps/webapp/app/presenters/v3/reports/health/health-data.ts b/apps/webapp/app/presenters/v3/reports/health/health-data.ts index 3c14b498e57..de2cc01b4cd 100644 --- a/apps/webapp/app/presenters/v3/reports/health/health-data.ts +++ b/apps/webapp/app/presenters/v3/reports/health/health-data.ts @@ -212,7 +212,7 @@ function queueWorstQuery(): string { return `SELECT queue AS name, argMax(max_queued, bucket_start) AS latest_queued -FROM queue_metrics +FROM concurrency_metrics GROUP BY queue ORDER BY latest_queued DESC LIMIT 20`; @@ -228,7 +228,7 @@ FROM ( SELECT deltaSumTimestampMerge(dlq_delta) AS dlq, argMax(max_queued, bucket_start) AS latest_queued - FROM queue_metrics + FROM concurrency_metrics GROUP BY queue )`; } diff --git a/apps/webapp/app/presenters/v3/reports/report-registry.ts b/apps/webapp/app/presenters/v3/reports/report-registry.ts index 23cc178befb..d8b3bc4e798 100644 --- a/apps/webapp/app/presenters/v3/reports/report-registry.ts +++ b/apps/webapp/app/presenters/v3/reports/report-registry.ts @@ -4,7 +4,7 @@ import { loadHealthInput } from "./health/health-data"; import { type ReportViewModel } from "./report-view-model"; /** A query table a report may read. Same table names the query API authorizes against. */ -export type ReportQueryTable = "runs" | "env_metrics" | "queue_metrics"; +export type ReportQueryTable = "runs" | "env_metrics" | "concurrency_metrics"; export type ReportLoader = { /** Authorization metadata: the route derives its per-table JWT scope check from this. */ @@ -19,7 +19,7 @@ function defineReport(loader: ReportLoader): ReportLoader> = { health: defineReport({ - tables: ["runs", "env_metrics", "queue_metrics"], + tables: ["runs", "env_metrics", "concurrency_metrics"], load: (env, period) => loadHealthInput(env, period), interpret: interpretHealth, }), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 12b29480751..b3eab7a7249 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -128,7 +128,7 @@ const SearchParamsSchema = z.object({ // The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay // current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup -// of queue_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless +// of concurrency_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless // of the chart/table period, and are NOT scoped to the visible queue set (the blocks are env-wide). const QUEUE_LIVE_BLOCKS_PERIOD = "15m"; const QUEUE_LIVE_BLOCKS_QUERY = @@ -1670,7 +1670,7 @@ function QueueHealthBadge(health: QueueHealth) { ); } -// The `queue_metrics`-prefixed key a queue is stored under (task queues are prefixed `task/`). +// The metrics-row key a queue is stored under (task queues are prefixed `task/`). function queueMetricsKey(queue: { type: string; name: string }): string { return `${queue.type === "task" ? "task/" : ""}${queue.name}`; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 933baafc43a..ab2f7f5eb46 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -266,7 +266,7 @@ export default function Page() { // The Concurrency keys tab exists only for queues with key activity: live keys in the // ckIndex, or nonzero CK history in the selected range (one cached scalar query decides). const { rows: gateRows, showLoading: gateLoading } = useQueueMetric( - `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM queue_metrics`, + `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM concurrency_metrics`, { ids, timeRange, queueName: fullName } ); const gateRow = gateRows[0]; @@ -463,7 +463,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_combined_running) AS running, least(max(max_combined_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -521,7 +521,7 @@ function OverviewCharts({ title="Queue depth" info="How many runs are waiting in this queue over time." className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -541,7 +541,7 @@ function OverviewCharts({ showLegend extraLegend={[{ color: "var(--color-warning)", label: "Falling behind" }]} className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -560,7 +560,7 @@ function OverviewCharts({ info="How long runs wait before they start." showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" @@ -583,7 +583,7 @@ function OverviewCharts({ } className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" - query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -698,7 +698,7 @@ function ConcurrencyKeyCharts({ title="Keys with backlog" info="Keys with runs waiting at once." className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -720,7 +720,7 @@ function ConcurrencyKeyCharts({ ) : null } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -782,7 +782,7 @@ type GroupedKeyChartProps = { // search can match keys outside the top 8; then filter by the search and keep the top 8 of those. function GroupedKeyChartCard(props: GroupedKeyChartProps) { const { rows, showLoading, failed } = useQueueMetric( - `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM queue_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, + `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM concurrency_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, { ids: props.ids, timeRange: props.timeRange, queueName: props.queueName } ); const keyFilter = props.keyFilter; @@ -810,7 +810,7 @@ function GroupedKeySeries({ }: GroupedKeyChartProps & { keys: string[] }) { const inList = keys.map((k) => `'${trqlString(k)}'`).join(", "); const { rows, showLoading, failed } = useQueueMetric( - `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM queue_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, + `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM concurrency_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, { ids, timeRange, queueName, fillGaps } ); @@ -1074,7 +1074,7 @@ function KeyDrilldown({ } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -1089,7 +1089,7 @@ function KeyDrilldown({ 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" @@ -1145,18 +1145,21 @@ function QueueStats({ timeRange: TimeRangeParams; queueName: string; }) { - const { rows } = useQueueMetric(`SELECT max(max_queued) AS peak_queued\nFROM queue_metrics`, { - ids, - timeRange, - queueName, - }); + const { rows } = useQueueMetric( + `SELECT max(max_queued) AS peak_queued\nFROM concurrency_metrics`, + { + ids, + timeRange, + queueName, + } + ); const peakQueued = rows[0] ? toNumber(rows[0].peak_queued) : 0; // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the // *Live values stay null, so the blocks show the loader values instead of flashing 0. const { rows: liveRows, responseReceivedAt } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM concurrency_metrics GROUP BY t ORDER BY t`, { ids, timeRange: { period: "15m", from: null, to: null }, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index 56f553547b1..ab8e5be48d6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -527,7 +527,7 @@ function TaskActivityCard({ > {view === "queue" ? ( 1, // dummy — the queue name isn't resolved against Postgres authorization: { action: "read", - resource: () => ({ type: "query", id: "queue_metrics" }), + resource: () => ({ type: "query", id: "concurrency_metrics" }), }, }, async ({ params, searchParams, authentication }) => { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 02571272fec..d81a5a4b664 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -1342,7 +1342,7 @@ function WaitingInQueueBlock({ responseReceivedAt, lastSuccessfulResponseAt, } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`, { ids: waiting.ids, timeRange: { period: "15m", from: null, to: null }, @@ -1441,7 +1441,7 @@ function WaitingInQueueBlock({
{ - it("measured path: queue_metrics source, real pending, parsed dlq, window from timeRange", async () => { + it("measured path: concurrency_metrics source, real pending, parsed dlq, window from timeRange", async () => { const input = await loadHealthInput( fakeEnv, "1h", diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql similarity index 91% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql rename to internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index dd0c4c53b34..f0cef36b4c8 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -1,23 +1,23 @@ -- +goose Up --- Total-concurrency gauges: combined_running is the in-flight count across ALL --- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency -- limit that applied in queue_limit, surfaced in the ck tier as max_limit -- (1000000 = no explicit limit). ALTER TABLE trigger_dev.queue_metrics_raw_v1 - ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, - ADD COLUMN IF NOT EXISTS combined_limit UInt32 DEFAULT 0; + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; ALTER TABLE trigger_dev.queue_metrics_v1 - ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_5m_v1 - ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_ck_v1 ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); @@ -46,8 +46,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(combined_running) AS max_combined_running, - max(combined_limit) AS max_combined_limit, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -74,8 +74,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(combined_running) AS max_combined_running, - max(combined_limit) AS max_combined_limit, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -105,9 +105,9 @@ DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; -ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; -ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; -ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS combined_running, DROP COLUMN IF EXISTS combined_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; -- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps -- feeding every aggregate table after a rollback. diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index f3a6be695e4..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,8 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), - combined_running: z.number().optional(), - combined_limit: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); From ca58abd2eb725d99dd72c5352a3e9f9635845c71 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:33:29 +0100 Subject: [PATCH 30/77] fix(webapp,clickhouse,run-engine): rename follow-ups from review Renumbers the metrics migration to 044 (main took 042 and 043), updates the two tests that pin the query-table and authorization ids, keeps the combined-cap chart from drawing a zero cap for history before a cap existed, and drops the flag docblock's stale per-key override paragraph. --- .../route.tsx | 2 +- apps/webapp/test/dashboardAgentToolScopes.test.ts | 2 +- apps/webapp/test/reportsApiRoute.test.ts | 10 +++++++--- ...sql => 044_add_queue_metrics_total_concurrency.sql} | 0 internal-packages/run-engine/src/run-queue/index.ts | 4 ---- 5 files changed, 9 insertions(+), 9 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_total_concurrency.sql => 044_add_queue_metrics_total_concurrency.sql} (100%) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ab2f7f5eb46..3e830514338 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} diff --git a/apps/webapp/test/dashboardAgentToolScopes.test.ts b/apps/webapp/test/dashboardAgentToolScopes.test.ts index 6eabb2d8eb7..7f850c3d3a4 100644 --- a/apps/webapp/test/dashboardAgentToolScopes.test.ts +++ b/apps/webapp/test/dashboardAgentToolScopes.test.ts @@ -34,7 +34,7 @@ const VIA_ENV_JWT: Read[] = [ { tool: "get_queue (metrics)", path: "/api/v1/queues/:name/metrics", - resource: { type: "query", id: "queue_metrics" }, + resource: { type: "query", id: "concurrency_metrics" }, }, { tool: "get_queue (live row)", path: "/api/v1/queues/:name", resource: { type: "queues" } }, { diff --git a/apps/webapp/test/reportsApiRoute.test.ts b/apps/webapp/test/reportsApiRoute.test.ts index f027c0d9eb3..37043b6513b 100644 --- a/apps/webapp/test/reportsApiRoute.test.ts +++ b/apps/webapp/test/reportsApiRoute.test.ts @@ -105,7 +105,7 @@ describe("api.v1.reports.$key — authorization", () => { expect(requiredResources("health")).toEqual([ { type: "query", id: "runs" }, { type: "query", id: "env_metrics" }, - { type: "query", id: "queue_metrics" }, + { type: "query", id: "concurrency_metrics" }, ]); }); @@ -118,7 +118,7 @@ describe("api.v1.reports.$key — authorization", () => { describe("reportQueryTables — scope derivation from the registry", () => { const registry: Record = { - health: { tables: ["runs", "env_metrics", "queue_metrics"] }, + health: { tables: ["runs", "env_metrics", "concurrency_metrics"] }, narrow: { tables: ["runs"] }, }; @@ -127,7 +127,11 @@ describe("reportQueryTables — scope derivation from the registry", () => { }); it("still gives the wider report all of its tables", () => { - expect(reportQueryTables("health", registry)).toEqual(["runs", "env_metrics", "queue_metrics"]); + expect(reportQueryTables("health", registry)).toEqual([ + "runs", + "env_metrics", + "concurrency_metrics", + ]); }); it("returns no tables for an unknown key", () => { diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql similarity index 100% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql rename to internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index d5b7b031224..3b38dc5b3a2 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -316,10 +316,6 @@ export type RunQueueOptions = { * that dead-lettered or suspended through a mirror-less path. Enabling only after * every instance runs this build avoids the noise but is no longer load-bearing * for correctness. - * - * Per-concurrency-key limit overrides are part of the same concurrency-limits - * feature and are deliberately enforced behind this flag too: writes are always - * accepted and durable, and enforcement of both arrives together. */ totalConcurrencyEnabled?: boolean; /** From 604582eac92a2a66dc1a0e37b2fdac1c53e9bb1f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:52:21 +0100 Subject: [PATCH 31/77] fix(webapp,clickhouse): pre-cap history keeps its truthful gap in the combined chart The config-gauge back-fill exists to cover leading buckets with no samples, but it also overwrote sampled history from before a combined cap existed, showing a cap that was never in force. The cap series now carries a sampled guard column and the back-fill skips sampled buckets. Also corrects the renumbered migration's pre-044 comment. --- .../app/components/queues/QueueMetricCards.tsx | 14 ++++++++++++-- .../route.tsx | 3 ++- .../044_add_queue_metrics_total_concurrency.sql | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 6b6692bcd9e..e878d5536e2 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -102,6 +102,12 @@ type QueueMetricChartProps = { * are config values that existed all along, so carry the first value backward instead. */ carryBackfill?: string[]; + /** + * Column that marks a bucket as genuinely sampled. When set, carryBackfill only + * overwrites buckets where this column is absent or zero, so history from before + * a config value existed keeps its truthful gap instead of inheriting the value. + */ + carryBackfillGuard?: string; /** Show the series legend below the chart (use for multi-series charts). */ showLegend?: boolean; /** @@ -141,6 +147,7 @@ export function QueueMetricChart({ defaultPeriod, warningOverlay, carryBackfill, + carryBackfillGuard, thresholdStroke, onHasDataChange, minBucketSeconds, @@ -174,12 +181,15 @@ export function QueueMetricChart({ const first = points.findIndex((p) => toNumber(p[key]) > 0); if (first > 0) { const value = points[first]![key]!; - for (let i = 0; i < first; i++) points[i]![key] = value; + for (let i = 0; i < first; i++) { + if (carryBackfillGuard && toNumber(points[i]![carryBackfillGuard]) > 0) continue; + points[i]![key] = value; + } } } } return points; - }, [rows, series, carryBackfill, sampleCountColumn]); + }, [rows, series, carryBackfill, carryBackfillGuard, sampleCountColumn]); const chartConfig = useMemo(() => { const cfg: ChartConfig = {}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 3e830514338..ffffc195b3a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -499,7 +499,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap, max(max_env_limit) AS sampled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -515,6 +515,7 @@ function OverviewCharts({ aboveColor: "var(--color-warning)", }} carryBackfill={["cap"]} + carryBackfillGuard="sampled" /> ) : null} Date: Sun, 6 Sep 2026 14:00:37 +0100 Subject: [PATCH 32/77] fix(webapp): carry-guard column reaches the chart points, Total naming in the UI The sampled guard was queried but dropped when rows became chart points, so the back-fill guard always read zero and pre-cap history was still overwritten; the guard column is now copied onto each point. User-visible strings move from Combined to Total, matching the perKey/total vocabulary. --- apps/webapp/app/components/queues/QueueMetricCards.tsx | 1 + .../route.tsx | 4 ++-- .../route.tsx | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index e878d5536e2..8a9fa8f5549 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -170,6 +170,7 @@ export function QueueMetricChart({ }; const hasSamples = sampleCountColumn ? toNumber(r[sampleCountColumn]) > 0 : true; for (const s of series) point[s.key] = hasSamples ? toNumber(r[s.key]) : null; + if (carryBackfillGuard) point[carryBackfillGuard] = toNumber(r[carryBackfillGuard]); return point; }) .filter((p) => Number.isFinite(p.bucket)); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index b3eab7a7249..1919196bdd0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -903,7 +903,7 @@ function QueuesWithMetricsView() { } content={ <> - Combined limit: at most{" "} + Total limit: at most{" "} {Math.min( queue.concurrency.combined.current, environment.concurrencyLimit @@ -1997,7 +1997,7 @@ function ClassicQueuesView() { } content={ <> - Combined limit: at most{" "} + Total limit: at most{" "} {Math.min( queue.concurrency.combined.current, environment.concurrencyLimit diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index ffffc195b3a..375ba2ca0f2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -488,7 +488,7 @@ function OverviewCharts({ /> {hasTotalLimit ? ( Runs in flight across ALL concurrency keys ( @@ -506,7 +506,7 @@ function OverviewCharts({ timeRange={timeRange} queueName={queueName} series={[ - { key: "cap", label: "Combined limit", color: COLORS.limit }, + { key: "cap", label: "Total limit", color: COLORS.limit }, { key: "running", label: "Running", color: COLORS.running }, ]} thresholdStroke={{ From 903af5ad9b38c8fd34888c47f464da133c88644e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:42:03 +0100 Subject: [PATCH 33/77] feat(webapp): rename the Concurrency page to Concurrency limits The env-level concurrency management page moves from /concurrency to /concurrency-limits, freeing the /concurrency URL for the upcoming queue concurrency page. Nothing outside the dashboard links to the old URL. chore(webapp): drop the old concurrency route left by the page rename --- .../components/dashboard-agent/DashboardAgentPanel.tsx | 4 ++-- .../webapp/app/components/navigation/sideMenuSections.tsx | 6 +++--- .../route.tsx | 6 +++--- .../route.tsx | 8 ++++---- .../route.tsx | 6 +++--- apps/webapp/app/utils/pathBuilder.ts | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-) rename apps/webapp/app/routes/{_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency => _app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits}/route.tsx (99%) diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx index 986c3f9bdf7..6bcc738869c 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx @@ -50,7 +50,7 @@ import { } from "./unread-counts"; import { AgentPanelColumn } from "./panel-layout"; import { markerAfterActiveChat, markerAfterActivity } from "./thinking-marker"; -import { concurrencyPath } from "~/utils/pathBuilder"; +import { concurrencyLimitsPath } from "~/utils/pathBuilder"; function serializePageContext(pageContext: AgentPageContext): string | undefined { try { @@ -131,7 +131,7 @@ export function DashboardAgentPanel({ const currentPage = agentPageLabel(pageContext, location.pathname); const pagePaths = useMemo>( - () => ({ raise_env_limit: concurrencyPath(organization, project, environment) }), + () => ({ raise_env_limit: concurrencyLimitsPath(organization, project, environment) }), [organization, project, environment] ); diff --git a/apps/webapp/app/components/navigation/sideMenuSections.tsx b/apps/webapp/app/components/navigation/sideMenuSections.tsx index 87d5da61195..3a358f2c878 100644 --- a/apps/webapp/app/components/navigation/sideMenuSections.tsx +++ b/apps/webapp/app/components/navigation/sideMenuSections.tsx @@ -23,7 +23,7 @@ import { type OrgForPath, type ProjectForPath, branchesPath, - concurrencyPath, + concurrencyLimitsPath, limitsPath, queryPath, regionsPath, @@ -269,10 +269,10 @@ export function buildSideMenuSections({ ? [ { id: "concurrency", - name: "Concurrency", + name: "Concurrency limits", icon: ConcurrencyIcon, activeIconColor: "text-text-bright", - to: concurrencyPath(organization, project, environment), + to: concurrencyLimitsPath(organization, project, environment), dataAction: "concurrency", } satisfies SideMenuItemConfig, ] diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx similarity index 99% rename from apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx rename to apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx index 7b5b8b0ac63..1df5a2f6c1a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx @@ -65,7 +65,7 @@ import { textLinkClassName } from "~/components/primitives/TextLink"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { formatCurrency, formatNumber } from "~/utils/numberFormatter"; -import { concurrencyPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBuilder"; +import { concurrencyLimitsPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBuilder"; import { AllocateConcurrencyService } from "~/v3/services/allocateConcurrency.server"; import { SetConcurrencyAddOnService } from "~/v3/services/setConcurrencyAddOn.server"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; @@ -145,7 +145,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); const project = await findProjectBySlug(organizationSlug, projectParam, userId); - const redirectPath = concurrencyPath( + const redirectPath = concurrencyLimitsPath( { slug: organizationSlug }, { slug: projectParam }, { slug: envParam } @@ -257,7 +257,7 @@ export default function Page() { return ( - + diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx index e8fb74da399..045c67e592b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx @@ -43,7 +43,7 @@ import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { formatNumber } from "~/utils/numberFormatter"; import { - concurrencyPath, + concurrencyLimitsPath, docsPath, EnvironmentParamSchema, organizationBillingPath, @@ -153,7 +153,7 @@ export default function Page() { {/* Concurrency Section */} {/* Rate Limits Section */} @@ -231,7 +231,7 @@ function CurrentPlanSection({ ); } -function ConcurrencySection({ concurrencyPath }: { concurrencyPath: string }) { +function ConcurrencySection({ concurrencyLimitsPath }: { concurrencyLimitsPath: string }) { return (
@@ -247,7 +247,7 @@ function ConcurrencySection({ concurrencyPath }: { concurrencyPath: string }) { Concurrency - + Manage concurrency diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 1919196bdd0..42da830e5cd 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -83,7 +83,7 @@ import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; import { - concurrencyPath, + concurrencyLimitsPath, docsPath, EnvironmentParamSchema, v3BillingPath, @@ -612,7 +612,7 @@ function QueuesWithMetricsView() { plan ? ( plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( Date: Sun, 6 Sep 2026 16:58:07 +0100 Subject: [PATCH 34/77] fix(webapp): deeplinks, favorites and report links follow the concurrency-limits rename --- apps/webapp/app/components/navigation/favoritePages.tsx | 2 +- apps/webapp/app/presenters/v3/reports/health/flow.ts | 2 +- apps/webapp/app/utils/deeplinkPages.ts | 2 +- apps/webapp/test/reportHealth.test.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/components/navigation/favoritePages.tsx b/apps/webapp/app/components/navigation/favoritePages.tsx index 50668bae889..64227ad23ae 100644 --- a/apps/webapp/app/components/navigation/favoritePages.tsx +++ b/apps/webapp/app/components/navigation/favoritePages.tsx @@ -234,7 +234,7 @@ const ENV_PAGE_META: Record = { "bulk-actions": { icon: "bulk-actions", name: "Bulk actions", singular: "Bulk action" }, apikeys: { icon: "apikeys", name: "API keys" }, alerts: { icon: "alerts", name: "Alerts", singular: "Alert" }, - concurrency: { icon: "concurrency", name: "Concurrency" }, + "concurrency-limits": { icon: "concurrency", name: "Concurrency limits" }, limits: { icon: "limits", name: "Limits" }, schedules: { icon: "schedules", name: "Schedules", singular: "Schedule" }, test: { icon: "test", name: "Test", singular: "Test" }, diff --git a/apps/webapp/app/presenters/v3/reports/health/flow.ts b/apps/webapp/app/presenters/v3/reports/health/flow.ts index ce6ba724527..e1d2e77654f 100644 --- a/apps/webapp/app/presenters/v3/reports/health/flow.ts +++ b/apps/webapp/app/presenters/v3/reports/health/flow.ts @@ -116,7 +116,7 @@ export function interpretFlow(metrics: Metric[], input: HealthInput): Finding { exclusions: [], observations: finishedPerMin > 0 ? [{ code: "not_workers_platform", evidence: { finishedPerMin } }] : [], - recommendation: { code: "raise_env_limit", link: "concurrency" }, + recommendation: { code: "raise_env_limit", link: "concurrency-limits" }, usesAttribution: true, }; } else if (ev.throttledShare >= t.throttledShare && !pinned) { diff --git a/apps/webapp/app/utils/deeplinkPages.ts b/apps/webapp/app/utils/deeplinkPages.ts index 1ba60b354f9..957d2096868 100644 --- a/apps/webapp/app/utils/deeplinkPages.ts +++ b/apps/webapp/app/utils/deeplinkPages.ts @@ -14,7 +14,7 @@ export const ENV_PAGE_TARGETS: ReadonlyMap = new Map([ ["batches", page("batches")], ["branches", page("branches")], ["bulk-actions", page("bulk-actions")], - ["concurrency", page("concurrency")], + ["concurrency-limits", page("concurrency-limits")], ["dashboards", page("dashboards")], ["deployments", page("deployments")], ["dev-branches", page("dev-branches")], diff --git a/apps/webapp/test/reportHealth.test.ts b/apps/webapp/test/reportHealth.test.ts index 706291816f0..6357ac2f77b 100644 --- a/apps/webapp/test/reportHealth.test.ts +++ b/apps/webapp/test/reportHealth.test.ts @@ -94,8 +94,8 @@ describe("health cause tree (Golden A — env limit saturation)", () => { it("footer = raise the limit (self-serve) + docs + do-nothing (drains)", () => { expect(vm.footer).toEqual([ - { code: "raise_env_limit", link: "concurrency" }, - { code: "concurrency_docs", link: "concurrency" }, + { code: "raise_env_limit", link: "concurrency-limits" }, + { code: "concurrency_docs", link: "concurrency-limits" }, { code: "do_nothing_drains", value: 2.3 }, ]); }); From 8840228bde864d4b5582425f02e96be426adf8f8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:58:53 +0100 Subject: [PATCH 35/77] chore(webapp): remove the combined concurrency override and reset endpoints Queue-level total limits are not part of the public API surface; managing totals arrives with the concurrency limits API instead. --- ...ueueParam.concurrency.combined.override.ts | 101 ----------------- ....$queueParam.concurrency.combined.reset.ts | 102 ------------------ 2 files changed, 203 deletions(-) delete mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts delete mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts deleted file mode 100644 index 77688a9fcc5..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), - concurrencyLimit: z.number().int().min(0).max(100000), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .overrideTotalConcurrencyLimit(authentication.environment, input, body.concurrencyLimit) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - totalConcurrencyLimit: queue.totalConcurrencyLimit, - totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, - totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "invalid_override": - case "concurrency_limit_exceeds_maximum": { - return json({ error: error.message }, { status: 400 }); - } - case "queue_update_failed": { - return json( - { error: "Failed to update queue total concurrency limit" }, - { status: 500 } - ); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to update queue total concurrency limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to update queue total concurrency limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts deleted file mode 100644 index 0e588716658..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .resetTotalConcurrencyLimit(authentication.environment, input) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - totalConcurrencyLimit: queue.totalConcurrencyLimit, - totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, - totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "queue_not_overridden": { - return json( - { error: "The queue total concurrency limit is not overridden" }, - { status: 400 } - ); - } - case "queue_update_failed": { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { status: 500 } - ); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; From ed25b3dac6aecb0adc5ff8d54ddd373c0eaa18a9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 17:56:45 +0100 Subject: [PATCH 36/77] fix(webapp): agent page labels follow the concurrency-limits rename --- apps/webapp/app/components/dashboard-agent/page-label.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/components/dashboard-agent/page-label.ts b/apps/webapp/app/components/dashboard-agent/page-label.ts index c5e949117fa..153cb7a6109 100644 --- a/apps/webapp/app/components/dashboard-agent/page-label.ts +++ b/apps/webapp/app/components/dashboard-agent/page-label.ts @@ -24,7 +24,7 @@ const KIND_LABELS: Record, string> = { alerts: "Alerts", apikeys: "API keys", envvars: "Environment variables", - concurrency: "Concurrency", + concurrency: "Concurrency limits", regions: "Regions", settings: "Settings", waitpoints: "Waitpoints", @@ -49,7 +49,7 @@ const SECTION_LABELS: Record = { batches: "Batches", "bulk-actions": "Bulk actions", branches: "Branches", - concurrency: "Concurrency", + "concurrency-limits": "Concurrency limits", dashboards: "Dashboards", deployments: "Deployments", "dev-branches": "Branches", From 9c7c130a230cf6a39d3b365a57ef575c20d257c0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:17:34 +0100 Subject: [PATCH 37/77] fix(run-engine): base-queue gauges reuse the shared total-limit read The remaining gauge plumbing for the base scripts rides at this level because the shared total-limit memo was introduced here. --- internal-packages/run-engine/src/run-queue/index.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 3b38dc5b3a2..1bebd20d3c2 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -207,15 +207,6 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { ckMaxWaitMs: "__ckwait", }; -// Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. -// Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with -// the total-cap gate); the CK scripts that run this (the Tracked variants and the CK -// dequeue) declare both. The group SCARD stays a fresh read: it must be post-admission. -const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { - totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "__totalLimitRaw() or '0'", -}; - // CK enqueue variants of the two gauges above, extended with the CK-health tail. const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", From 818270306002908bde3dea37986bb7427f598e63 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:16:46 +0100 Subject: [PATCH 38/77] fix(run-engine): base-queue admit paths enforce the total limit for keyless runs Keyless admits join the per-base-queue group set and are gated on the env-clamped total limit across the base enqueue fast path, the base dequeue and the queue mover, with the bounded reconcile at saturation and the release-side group mirror, matching the keyed tracked paths. Base-queue gauges carry the total running/limit fields. --- .../run-engine/src/run-queue/index.ts | 147 ++++++++++++++++-- 1 file changed, 135 insertions(+), 12 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 1bebd20d3c2..c4ac367f2b1 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -170,6 +170,22 @@ const QUEUE_METRICS_GAUGE_PRELUDE = ` local __qm_g = false local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; +/** Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. + * Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with + * the total-cap gate); every script that runs a gauge with this tail declares both. The + * group SCARD stays a fresh read: it must be post-admission. */ +const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "__totalLimitRaw() or '0'", +}; + +/** The gauge layout is positional (totals ride behind the CK slots), so the plain + * scripts zero-fill the CK health fields: a base queue has no CK variants to backlog. */ +const QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS = { + ckBacklogged: "0", + ckMaxWaitMs: "0", +}; + // Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before // return 0) and the base dequeue's sample-at-return wrapper. Gated on the last ARGV so it // is inert unless the caller opts in. CK queues emit per-subqueue depth (queue_name @@ -182,6 +198,8 @@ const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", + ...QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // Enqueue fast-path gauge: the admission check already computed queueCurrent/envCurrent/ @@ -195,6 +213,8 @@ const QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "envCurrent", envLimit: "envLimit", + ...QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK-health extras: distinct backlogged keys + most-starved head-of-line wait (ckIndex scores @@ -288,8 +308,8 @@ export type RunQueueOptions = { */ counterTtlSeconds?: number; /** - * When true, concurrency-keyed queues maintain a per-base-queue groupConcurrency SET - * (total in-flight across all key variants) and enforce the queue's total concurrency + * When true, queues maintain a per-base-queue groupConcurrency SET (total in-flight + * across all key variants AND keyless runs) and enforce the queue's total concurrency * limit at admit time. Default false: admit paths are byte-identical to before, and * only the release-side SREM mirror runs (a no-op on an absent set), so the flag can * be flipped on a fleet that has fully rolled onto this build without draining queues. @@ -625,7 +645,7 @@ export class RunQueue { } /** - * Total in-flight runs across all concurrency-key variants of a queue (the + * Total in-flight runs on a queue, keyed and keyless together (the * groupConcurrency SET cardinality). Admits only populate the set while * totalConcurrencyEnabled is on. After the flag is turned off the set drains * to zero through the release-side mirrors, so a nonzero read reflects real @@ -2526,6 +2546,8 @@ export class RunQueue { queueConcurrencyLimitKey, envConcurrencyLimitKey, envConcurrencyLimitBurstFactorKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(message.queue), // args queueName, messageId, @@ -2558,6 +2580,8 @@ export class RunQueue { queueConcurrencyLimitKey, envConcurrencyLimitKey, envConcurrencyLimitBurstFactorKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(message.queue), // args queueName, messageId, @@ -2644,6 +2668,8 @@ export class RunQueue { envQueueKey, masterQueueKey, ttlQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(messageQueue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(messageQueue), //args messageQueue, String(Date.now()), @@ -3633,7 +3659,7 @@ end // When enableFastPath == '0', the script skips the fast-path check entirely and behaves // identically to the pre-fast-path version (with the addition of returning 0). this.redis.defineCommand("enqueueMessage", { - numberOfKeys: 12, + numberOfKeys: 14, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -3648,6 +3674,16 @@ local workerQueueKey = KEYS[9] local queueConcurrencyLimitKey = KEYS[10] local envConcurrencyLimitKey = KEYS[11] local envConcurrencyLimitBurstFactorKey = KEYS[12] +-- Total-cap keys (KEYS 13-14) +local groupConcurrencyKey = KEYS[13] +local totalConcurrencyLimitKey = KEYS[14] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -3683,6 +3719,20 @@ if enableFastPath == '1' then ) if queueCurrent < queueLimit then + -- Total-cap gate: a fast-path admit consumes a group slot, so it must + -- respect the env-clamped total limit. At the cap we fall through to the + -- slow path (the message queues; the dequeue gate holds it). + local totalAllowsFastPath = true + if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) + if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then + totalAllowsFastPath = false + end + end + end + local gateMsg = nil local gatesAllowFastPath = true if gatesEnabled and string.find(messageData, '"gates"', 1, true) then @@ -3693,10 +3743,13 @@ if enableFastPath == '1' then end end - if gatesAllowFastPath then + if totalAllowsFastPath and gatesAllowFastPath then redis.call('SET', messageKey, messageData) redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gateMsg then __gatesAcquire(keyPrefix, gateMsg, messageId) end @@ -3728,8 +3781,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -3745,7 +3802,7 @@ return __qmret(0) // (scheduled independently before enqueue) handles TTL expiry. This mirrors what // dequeueMessagesFromQueue does: it removes from the TTL set when dequeuing. this.redis.defineCommand("enqueueMessageWithTtl", { - numberOfKeys: 13, + numberOfKeys: 15, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -3761,6 +3818,16 @@ local workerQueueKey = KEYS[10] local queueConcurrencyLimitKey = KEYS[11] local envConcurrencyLimitKey = KEYS[12] local envConcurrencyLimitBurstFactorKey = KEYS[13] +-- Total-cap keys (KEYS 14-15) +local groupConcurrencyKey = KEYS[14] +local totalConcurrencyLimitKey = KEYS[15] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -3798,6 +3865,20 @@ if enableFastPath == '1' then ) if queueCurrent < queueLimit then + -- Total-cap gate: a fast-path admit consumes a group slot, so it must + -- respect the env-clamped total limit. At the cap we fall through to the + -- slow path (the message queues; the dequeue gate holds it). + local totalAllowsFastPath = true + if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) + if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then + totalAllowsFastPath = false + end + end + end + local gateMsg = nil local gatesAllowFastPath = true if gatesEnabled and string.find(messageData, '"gates"', 1, true) then @@ -3808,10 +3889,13 @@ if enableFastPath == '1' then end end - if gatesAllowFastPath then + if totalAllowsFastPath and gatesAllowFastPath then redis.call('SET', messageKey, messageData) redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gateMsg then __gatesAcquire(keyPrefix, gateMsg, messageId) end @@ -3847,8 +3931,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -4659,7 +4747,7 @@ return results }); this.redis.defineCommand("dequeueMessagesFromQueue", { - numberOfKeys: 10, + numberOfKeys: 12, lua: ` local queueKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -4671,6 +4759,16 @@ local messageKeyPrefix = KEYS[7] local envQueueKey = KEYS[8] local masterQueueKey = KEYS[9] local ttlQueueKey = KEYS[10] -- Optional: TTL sorted set key (empty string if not used) +-- Total-cap keys (KEYS 11-12) +local groupConcurrencyKey = KEYS[11] +local totalConcurrencyLimitKey = KEYS[12] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -4718,6 +4816,22 @@ local envAvailableCapacity = envConcurrencyLimitWithBurstFactor - envCurrentConc local queueAvailableCapacity = totalQueueConcurrencyLimit - queueCurrentConcurrency local actualMaxCount = math.min(maxCount, envAvailableCapacity, queueAvailableCapacity) +-- Total-cap gate: every admit joins the group set, so the batch is bounded by the +-- env-clamped total limit's remaining capacity. At saturation, run the bounded +-- reconcile once (heals leaked members) and re-check before giving up. +if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) + local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') + if groupCurrentConcurrency >= totalLimit then + __gateReconcile(groupConcurrencyKey, messageKeyPrefix, keyPrefix) + groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') + end + actualMaxCount = math.min(actualMaxCount, totalLimit - groupCurrentConcurrency) + end +end + if actualMaxCount <= 0 then return __qmret(nil) end @@ -4769,6 +4883,9 @@ for i = 1, #messages, 2 do redis.call('ZREM', envQueueKey, messageId) redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gatesEnabled then __gatesAcquire(keyPrefix, messageData, messageId) end @@ -6195,6 +6312,8 @@ declare module "@internal/redis" { queueConcurrencyLimitKey: string, envConcurrencyLimitKey: string, envConcurrencyLimitBurstFactorKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args queueName: string, messageId: string, @@ -6227,6 +6346,8 @@ declare module "@internal/redis" { queueConcurrencyLimitKey: string, envConcurrencyLimitKey: string, envConcurrencyLimitBurstFactorKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args queueName: string, messageId: string, @@ -6272,6 +6393,8 @@ declare module "@internal/redis" { envQueueKey: string, masterQueueKey: string, ttlQueueKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args childQueueName: string, currentTime: string, From e92efa41220ab57ad04b9c51241848aed4249462 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:16:25 +0100 Subject: [PATCH 39/77] fix(run-engine): the total concurrency limit spans keyed and keyless runs Keyless admits now join the same per-base-queue group set as keyed admits and are gated on the total limit, so a queue's total truly caps everything in flight together. The base enqueue fast path, the base dequeue and the queue mover all check the env-clamped total (with the bounded reconcile at saturation) and mirror the group membership on release, matching what the keyed tracked paths already did. Base-queue gauge snapshots now carry the total running/limit fields too. --- .../run-queue/tests/totalConcurrency.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts b/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts index f8f914b564f..0ccedd8881b 100644 --- a/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts @@ -207,6 +207,52 @@ describe("RunQueue total concurrency limit", () => { } ); + redisTest("the total limit caps keyed and keyless runs together", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + await queue.updateQueueTotalConcurrencyLimits(authenticatedEnvDev, "task/my-task", 1); + + const now = Date.now(); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "r0", timestamp: now - 1000 }), + workerQueue: "main", + }); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "r1", concurrencyKey: "ck-a", timestamp: now - 999 }), + workerQueue: "main", + }); + + const oneAdmitted = await waitFor( + async () => (await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")) === 1 + ); + expect(oneAdmitted).toBe(true); + + /** The second run must stay queued: the total pool spans keyed and keyless. */ + await setTimeout(2000); + expect(await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + + const dequeued = await queue.dequeueMessageFromWorkerQueue("consumer-1", "main"); + assertNonNullable(dequeued); + await queue.acknowledgeMessage(authenticatedEnvDev.organization.id, dequeued.messageId); + + /** Acking the first holder frees the total pool; the other run is admitted. */ + const secondAdmitted = await waitFor(async () => { + const next = await queue.dequeueMessageFromWorkerQueue("consumer-1", "main", { + blockingPop: false, + }); + return next !== undefined && next.messageId !== dequeued.messageId; + }); + expect(secondAdmitted).toBe(true); + expect(await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + } finally { + await queue.quit(); + } + }); + redisTest("enqueue fast path respects the total limit", async ({ redisContainer }) => { const queue = createQueue(redisContainer, true); try { From 86e77a0f2c8505a7ef1eea929027aff43e267036 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:30:37 +0100 Subject: [PATCH 40/77] fix(run-engine): keyless releases drain the total-concurrency group set The plain acknowledge, nack, dead-letter and release-concurrency scripts now mirror the group SREM off the base concurrency removal, exactly like their tracked keyed variants, so keyless holders admitted into the total pool always release their slot instead of lingering until a saturation reconcile prunes them. --- .../run-engine/src/run-queue/index.ts | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index c4ac367f2b1..b88abac59ac 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -1497,6 +1497,7 @@ export class RunQueue { this.keys.queueCurrentDequeuedKeyFromQueue(message.queue), this.keys.envCurrentDequeuedKeyFromQueue(message.queue), this.keys.messageKey(message.orgId, messageId), + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, this.options.redis.keyPrefix ?? "" ); @@ -3085,6 +3086,7 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, workerQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, messageQueue, messageKeyValue, @@ -3234,6 +3236,7 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, envQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), //args messageId, messageQueue, @@ -3295,6 +3298,7 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, deadLetterQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, messageQueue, this.options.redis.keyPrefix ?? "" @@ -5437,7 +5441,7 @@ return message }); this.redis.defineCommand("acknowledgeMessage", { - numberOfKeys: 9, + numberOfKeys: 10, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5449,6 +5453,7 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local workerQueueKey = KEYS[9] +local groupConcurrencyKey = KEYS[10] -- Args: local messageId = ARGV[1] @@ -5475,8 +5480,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], messageQueueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -5490,7 +5499,7 @@ end }); this.redis.defineCommand("nackMessage", { - numberOfKeys: 8, + numberOfKeys: 9, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5501,6 +5510,7 @@ local envCurrentConcurrencyKey = KEYS[5] local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] +local groupConcurrencyKey = KEYS[9] -- Args: local messageId = ARGV[1] @@ -5513,8 +5523,12 @@ ${QUEUE_GATES_LUA_HELPERS} -- Update the message data redis.call('SET', messageKey, messageData) --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -5535,7 +5549,7 @@ end }); this.redis.defineCommand("moveToDeadLetterQueue", { - numberOfKeys: 9, + numberOfKeys: 10, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5547,6 +5561,7 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local deadLetterQueueKey = KEYS[9] +local groupConcurrencyKey = KEYS[10] -- Args: local messageId = ARGV[1] @@ -5571,8 +5586,12 @@ end -- Add the message to the dead letter queue redis.call('ZADD', deadLetterQueueKey, tonumber(redis.call('TIME')[1]), messageId) --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -6056,7 +6075,7 @@ __gatesRelease(keyPrefix, rawPayload, messageId) }); this.redis.defineCommand("releaseConcurrency", { - numberOfKeys: 5, + numberOfKeys: 6, lua: ` -- Keys: local queueCurrentConcurrencyKey = KEYS[1] @@ -6064,14 +6083,19 @@ local envCurrentConcurrencyKey = KEYS[2] local queueCurrentDequeuedKey = KEYS[3] local envCurrentDequeuedKey = KEYS[4] local messageKey = KEYS[5] +local groupConcurrencyKey = KEYS[6] -- Args: local messageId = ARGV[1] local keyPrefix = ARGV[2] ${QUEUE_GATES_LUA_HELPERS} --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -6432,6 +6456,7 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, workerQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6464,6 +6489,7 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, envQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6484,6 +6510,7 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, deadLetterQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6498,6 +6525,7 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, messageKey: string, + groupConcurrencyKey: string, // args messageId: string, keyPrefix: string, From 803145a550f11646c9a39020505e5eb0c3e1e7ea Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:44:46 +0100 Subject: [PATCH 41/77] fix(run-engine): the repair clear drains the group set for keyless runs clearMessageFromConcurrencySets now mirrors the group SREM like every other release path, so a stuck keyless run cleared by the repair sweep frees its total-concurrency slot instead of phantom-holding it until a saturation reconcile. The plain gauge test pins the new tail: zeroed CK fields plus the total pair. --- .../run-engine/src/run-queue/index.ts | 18 +++++++++++++----- .../run-engine/src/run-queue/metrics.test.ts | 12 +++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index b88abac59ac..8b2c7f98436 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -3112,8 +3112,9 @@ export class RunQueue { const messageKey = this.keys.messageKey(orgId, messageId); /** * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's - * slots live on the ck variant, and the tracked clear additionally mirrors the - * group set and counters that only keyed queues maintain. + * slots live on the ck variant. Both variants mirror the per-base-queue group + * set (keyed and keyless admits populate it); the tracked clear additionally + * maintains the counters that only keyed queues keep. */ const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue; const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey( @@ -3159,6 +3160,7 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, messageKey, + this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), messageId, this.options.redis.keyPrefix ?? "" ); @@ -6238,7 +6240,7 @@ return results }); this.redis.defineCommand("clearMessageFromConcurrencySets", { - numberOfKeys: 5, + numberOfKeys: 6, lua: ` -- Keys: local queueCurrentConcurrencyKey = KEYS[1] @@ -6246,14 +6248,19 @@ local envCurrentConcurrencyKey = KEYS[2] local queueCurrentDequeuedKey = KEYS[3] local envCurrentDequeuedKey = KEYS[4] local messageKey = KEYS[5] +local groupConcurrencyKey = KEYS[6] -- Args: local messageId = ARGV[1] local keyPrefix = ARGV[2] ${QUEUE_GATES_LUA_HELPERS} --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -6473,6 +6480,7 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, messageKey: string, + groupConcurrencyKey: string, // args messageId: string, keyPrefix: string, diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index edae6f8cc30..c239b4aa937 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -156,9 +156,15 @@ describe("RunQueue queue-metrics emission", () => { for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) { expect(gauge!.fields[f]).toBeDefined(); } - // Non-CK scripts keep the 7-field gauge (no CK-health tail). - expect(gauge!.fields.ckq).toBeUndefined(); - expect(gauge!.fields.ckw).toBeUndefined(); + /** + * Non-CK scripts emit the full gauge tail too: zeroed CK-health fields (a base + * queue has no CK variants) followed by the total running/limit pair, so a + * keyless queue with a total limit still charts total concurrency. + */ + expect(gauge!.fields.ckq).toBe("0"); + expect(gauge!.fields.ckw).toBe("0"); + expect(gauge!.fields.tcc).toBeDefined(); + expect(gauge!.fields.tlim).toBeDefined(); // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse. From c4b933f43854d9bcfccc4216dedd347b8923a395 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:50:18 +0100 Subject: [PATCH 42/77] fix(run-engine): the keyless repair clear derives the group key from the environment The bare queue name is not a full queue key, so the FromQueue producer built a wrong group key for keyless repairs; the env-based producer matches the key the admit paths populate. --- internal-packages/run-engine/src/run-queue/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 8b2c7f98436..d2eabe5e8d9 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -3160,7 +3160,7 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, messageKey, - this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), + this.keys.queueGroupConcurrencyKey(env, queue), messageId, this.options.redis.keyPrefix ?? "" ); From 403cc6dcbdf0c216cf94069fb2559e50cb228343 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:58:40 +0100 Subject: [PATCH 43/77] fix(run-engine): the TTL sweep's defensive removal drains the group set for keyless queues The defensive currentConcurrency removal in the TTL expiry script now mirrors into the base groupConcurrency set whether or not the queue is keyed, closing the last live release path where the mirror was conditional on the queue shape instead of the removal itself. --- .../run-engine/src/run-queue/index.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index d2eabe5e8d9..4c4baf6bcf6 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -4709,8 +4709,16 @@ for i, member in ipairs(expiredMembers) do redis.call('SREM', envConcurrencyKey, runId) redis.call('SREM', envDequeuedKey, runId) - -- Rebalance CK index AND update counters if this is a CK queue + -- Mirror the currentConcurrency SREM into the base groupConcurrency set for + -- keyed and keyless queues alike, so a defensive removal always drains the + -- total pool too. local ckMatch = string.match(rawQueueKey, "(.-):ck:") + if removedFromCurrent == 1 then + local groupBase = ckMatch or rawQueueKey + redis.call('SREM', keyPrefix .. groupBase .. ":groupConcurrency", runId) + end + + -- Rebalance CK index AND update counters if this is a CK queue if ckMatch then local lengthCounterKey = keyPrefix .. ckMatch .. ":lengthCounter" local runningCounterKey = keyPrefix .. ckMatch .. ":runningCounter" @@ -4720,10 +4728,6 @@ for i, member in ipairs(expiredMembers) do if removedFromDequeued == 1 then decrFloored(runningCounterKey) end - -- Mirror the per-CK currentConcurrency SREM into the base groupConcurrency set - if removedFromCurrent == 1 then - redis.call('SREM', keyPrefix .. ckMatch .. ":groupConcurrency", runId) - end local ckIndexKey = keyPrefix .. ckMatch .. ":ckIndex" local earliest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES') From 49183d01c4596e6ad9dbb87e388366e272ef6106 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 02:12:54 +0100 Subject: [PATCH 44/77] feat(run-engine): exact per-gate queued counters Every gate queue keeps a counter of runs that are queued and must clear it to execute: incremented per gate when a run enters a queue zset, decremented when it leaves (admit, ack while queued, nack re-queue, TTL expiry, dead-letter), always guarded on the zset transition so re-enqueues and already-removed members never double count, and floored at zero. Read via gateQueuedCountOfQueue and the batched variant; this backs the queued field on the upcoming concurrency limits API. --- .../run-engine/src/run-queue/index.ts | 114 ++++++++++++++++-- .../run-engine/src/run-queue/keyProducer.ts | 11 ++ .../src/run-queue/tests/queueGates.test.ts | 45 +++++++ .../run-engine/src/run-queue/types.ts | 1 + 4 files changed, 162 insertions(+), 9 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 4c4baf6bcf6..9f4061c8474 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -160,6 +160,32 @@ local function __gatesRelease(gatesKeyPrefix, rawPayload, messageId) redis.call('SREM', base .. ':groupConcurrency', messageId) end end +end + +-- Per-gate queued counter: runs that are queued and must clear the gate to execute. +-- Callers gate the delta on the actual queue-zset transition (ZADD added == 1 / +-- ZREM removed == 1) so re-enqueues and already-removed members never double count. +-- Payload-driven and flag-independent, like release, so counts stay exact across +-- flag flips. Floored at zero: a missed increment can never push a counter negative. +local function __gateQueuedDelta(gatesKeyPrefix, msg, delta) + if type(msg) ~= 'table' or not msg.gates then return end + for _, gate in ipairs(msg.gates) do + local base = __gateKeys(gatesKeyPrefix, msg, gate) + local counterKey = base .. ':gateQueuedCounter' + if delta > 0 then + redis.call('INCRBY', counterKey, delta) + elseif tonumber(redis.call('GET', counterKey) or '0') > 0 then + redis.call('DECRBY', counterKey, -delta) + end + end +end + +local function __gateQueuedDeltaRaw(gatesKeyPrefix, rawPayload, delta) + if not rawPayload or rawPayload == false then return end + if not string.find(rawPayload, '"gates"', 1, true) then return end + local ok, msg = pcall(cjson.decode, rawPayload) + if not ok then return end + __gateQueuedDelta(gatesKeyPrefix, msg, delta) end`; // Prelude spliced at the top of every gauge-carrying script: declares the gauge slot and @@ -655,6 +681,39 @@ export class RunQueue { return this.redis.scard(this.keys.queueGroupConcurrencyKey(env, queue)); } + /** + * Runs that are queued and must clear this gate queue to execute (the per-gate + * queued counter). Exact by construction: incremented per gate on enqueue and + * decremented on admit and on every queued-removal path, floored at zero. + */ + public async gateQueuedCountOfQueue(env: MinimalAuthenticatedEnvironment, queue: string) { + const result = await this.redis.get(this.keys.gateQueuedCounterKey(env, queue)); + return result ? Math.max(Number(result), 0) : 0; + } + + /** Batch variant of gateQueuedCountOfQueue: one pipeline of GETs. */ + public async gateQueuedCountOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + pipeline.get(this.keys.gateQueuedCounterKey(env, queue)); + }); + + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index]?.[1]; + const parsed = typeof value === "string" ? Number(value) : 0; + acc[queue] = Number.isFinite(parsed) ? Math.max(parsed, 0) : 0; + return acc; + }, + {} as Record + ); + } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, @@ -3773,7 +3832,10 @@ end redis.call('SET', messageKey, messageData) -- Add the message to the queue -redis.call('ZADD', queueKey, messageScore, messageId) +local added = redis.call('ZADD', queueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end -- Add the message to the env queue redis.call('ZADD', envQueueKey, messageScore, messageId) @@ -3920,7 +3982,10 @@ end redis.call('SET', messageKey, messageData) -- Add the message to the queue -redis.call('ZADD', queueKey, messageScore, messageId) +local added = redis.call('ZADD', queueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end -- Add the message to the env queue redis.call('ZADD', envQueueKey, messageScore, messageId) @@ -4315,6 +4380,7 @@ local added = redis.call('ZADD', queueKey, messageScore, messageId) redis.call('ZADD', envQueueKey, messageScore, messageId) if added == 1 then redis.call('INCR', lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) end -- Rebalance CK index @@ -4486,6 +4552,7 @@ redis.call('ZADD', envQueueKey, messageScore, messageId) redis.call('ZADD', ttlQueueKey, ttlScore, ttlMember) if added == 1 then redis.call('INCR', lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) end -- Rebalance CK index @@ -4690,6 +4757,9 @@ for i, member in ipairs(expiredMembers) do -- ZREM from queue; if successful AND this is a CK variant, DECR lengthCounter. local removedFromZset = redis.call('ZREM', queueKey, runId) + if removedFromZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) + end local envMatch = string.match(rawQueueKey, ":env:([^:]+)") if envMatch then @@ -4876,8 +4946,11 @@ for i = 1, #messages, 2 do -- leave messageKey intact, and (re-)register the TTL entry so the -- TTL consumer can discover and properly expire the run. The entry -- is removed on first dequeue, so it cannot be assumed to exist. - redis.call('ZREM', queueKey, messageId) + local removedExpired = redis.call('ZREM', queueKey, messageId) redis.call('ZREM', envQueueKey, messageId) + if removedExpired == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end if ttlQueueKey and ttlQueueKey ~= '' then local ttlMember = queueName .. '|' .. messageId .. '|' .. (messageData.orgId or '') redis.call('ZADD', ttlQueueKey, ttlExpiresAt, ttlMember) @@ -4889,8 +4962,11 @@ for i = 1, #messages, 2 do end if gatesAllow then - redis.call('ZREM', queueKey, messageId) + local removedFromQueue = redis.call('ZREM', queueKey, messageId) redis.call('ZREM', envQueueKey, messageId) + if removedFromQueue == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) if totalConcurrencyEnabled then @@ -5236,9 +5312,12 @@ for _, ckQueueName in ipairs(ckQueues) do local ttlExpiresAt = messageData and messageData.ttlExpiresAt if ttlExpiresAt and ttlExpiresAt <= currentTime then - redis.call('ZREM', fullQueueKey, messageId) + local removedExpired = redis.call('ZREM', fullQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) decrLengthCounter() + if removedExpired == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end if ttlQueueKey and ttlQueueKey ~= '' then local ttlMember = ckQueueName .. '|' .. messageId .. '|' .. (messageData.orgId or '') redis.call('ZADD', ttlQueueKey, ttlExpiresAt, ttlMember) @@ -5260,9 +5339,12 @@ for _, ckQueueName in ipairs(ckQueues) do end if gatesAllow and totalAllows then - redis.call('ZREM', fullQueueKey, messageId) + local removedFromQueue = redis.call('ZREM', fullQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) decrLengthCounter() + if removedFromQueue == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end redis.call('SADD', ckConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) if totalConcurrencyEnabled then @@ -5475,8 +5557,11 @@ local rawPayload = redis.call('GET', messageKey) redis.call('DEL', messageKey) -- Remove the message from the queue -redis.call('ZREM', messageQueueKey, messageId) +local removedFromQueueZset = redis.call('ZREM', messageQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) +if removedFromQueueZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) +end -- Rebalance the parent queues local earliestMessage = redis.call('ZRANGE', messageQueueKey, 0, 0, 'WITHSCORES') @@ -5541,7 +5626,10 @@ redis.call('SREM', envCurrentDequeuedKey, messageId) __gatesRelease(keyPrefix, messageData, messageId) -- Enqueue the message into the queue -redis.call('ZADD', messageQueueKey, messageScore, messageId) +local added = redis.call('ZADD', messageQueueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end redis.call('ZADD', envQueueKey, messageScore, messageId) -- Rebalance the parent queues @@ -5578,8 +5666,11 @@ ${QUEUE_GATES_LUA_HELPERS} local rawPayload = redis.call('GET', messageKey) -- Remove the message from the queue -redis.call('ZREM', messageQueue, messageId) +local removedFromQueueZset = redis.call('ZREM', messageQueue, messageId) redis.call('ZREM', envQueueKey, messageId) +if removedFromQueueZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) +end -- Rebalance the parent queues local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') @@ -5840,6 +5931,7 @@ local removedFromZset = redis.call('ZREM', messageQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) if removedFromZset == 1 then decrFloored(lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) end -- Rebalance CK index @@ -5963,6 +6055,9 @@ end -- Enqueue the message back into the CK-specific queue. INCR lengthCounter only if -- it's a new entry (ZADD returns 1). local added = redis.call('ZADD', messageQueueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end redis.call('ZADD', envQueueKey, messageScore, messageId) if added == 1 then redis.call('INCR', lengthCounterKey) @@ -6034,6 +6129,7 @@ local removedFromZset = redis.call('ZREM', messageQueue, messageId) redis.call('ZREM', envQueueKey, messageId) if removedFromZset == 1 then decrFloored(lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) end -- Rebalance CK index diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 98028f5af7b..ff7a51f91f3 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -26,6 +26,7 @@ const constants = { RUNNING_COUNTER_PART: "runningCounter", GROUP_CONCURRENCY_PART: "groupConcurrency", TOTAL_CONCURRENCY_LIMIT_PART: "totalConcurrency", + GATE_QUEUED_COUNTER_PART: "gateQueuedCounter", } as const; export class RunQueueFullKeyProducer implements RunQueueKeyProducer { @@ -353,6 +354,16 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.GROUP_CONCURRENCY_PART}`; } + /** + * Counter of queued runs holding this queue as a gate: runs that are not + * executing, are queued, and must clear this gate to execute. Maintained + * exactly like the CK length counter: incremented per gate on enqueue, + * decremented on admit and on every queued-removal path. + */ + gateQueuedCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string { + return `${this.queueKey(env, queue)}:${constants.GATE_QUEUED_COUNTER_PART}`; + } + /** * String key holding the queue's total concurrency limit (the cap across all * concurrency-key variants). Absent = no total cap. Readers clamp to the diff --git a/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts b/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts index c2195074426..272523b9595 100644 --- a/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts @@ -231,6 +231,51 @@ describe("RunQueue gates", () => { } ); + redisTest( + "counts queued runs per gate and drains the counter on admit and ack", + async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "shared-gate", 1); + + const now = Date.now(); + for (const i of [0, 1]) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `r${i}`, + timestamp: now - 1000 + i, + gates: [{ queue: "shared-gate" }], + }), + workerQueue: "main", + }); + } + + const oneAdmitted = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "shared-gate")) === 1 + ); + expect(oneAdmitted).toBe(true); + + /** One run executes, one waits: the gate's queued counter holds the waiter. */ + await setTimeout(2000); + expect(await queue.gateQueuedCountOfQueue(authenticatedEnvDev, "shared-gate")).toBe(1); + + expect(await popWorkerQueue(queue, "r0")).toBe(true); + await queue.acknowledgeMessage(authenticatedEnvDev.organization.id, "r0"); + + const drained = await waitFor(async () => { + if (!(await popWorkerQueue(queue, "r1"))) return false; + return (await queue.gateQueuedCountOfQueue(authenticatedEnvDev, "shared-gate")) === 0; + }); + expect(drained).toBe(true); + } finally { + await queue.quit(); + } + } + ); + redisTest("ignores gates and holds no gate slots when disabled", async ({ redisContainer }) => { const queue = createQueue(redisContainer, false); try { diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 75651a1f847..80565b0f732 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,6 +111,7 @@ export interface RunQueueKeyProducer { queueGroupConcurrencyKeyFromQueue(queue: string): string; queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; + gateQueuedCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string; //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; From d97ba349cf6096f819ed08a6f537acd238f32fb0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 04:25:26 +0100 Subject: [PATCH 45/77] fix(run-engine): gate queued counters dedupe by base and re-anchor on a 24h TTL Gates sharing a base within one run count once, and each counter carries an absolute 24h TTL set at creation so drift from paths without the delta (a rolling deploy, a stale-entry cleanup) clears within a day: the reset counter converges back to exact through the zero floor as the pre-reset backlog drains. --- .../run-engine/src/run-queue/index.ts | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 9f4061c8474..068b90b54ef 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -164,18 +164,29 @@ end -- Per-gate queued counter: runs that are queued and must clear the gate to execute. -- Callers gate the delta on the actual queue-zset transition (ZADD added == 1 / --- ZREM removed == 1) so re-enqueues and already-removed members never double count. --- Payload-driven and flag-independent, like release, so counts stay exact across --- flag flips. Floored at zero: a missed increment can never push a counter negative. +-- ZREM removed == 1) so re-enqueues and already-removed members never double count; +-- gates sharing a base (duplicate entries, key variants) count once per run. The +-- 24h absolute TTL (set at creation, never extended) re-anchors drift from paths +-- without the delta (rolling deploys, stale-entry cleanup): the counter resets, +-- floored decrements absorb the pre-reset backlog as it drains, and counts converge +-- to exact for every run enqueued after the reset. Payload-driven and +-- flag-independent, like release, so counts stay exact across flag flips. local function __gateQueuedDelta(gatesKeyPrefix, msg, delta) if type(msg) ~= 'table' or not msg.gates then return end + local seenBases = {} for _, gate in ipairs(msg.gates) do local base = __gateKeys(gatesKeyPrefix, msg, gate) - local counterKey = base .. ':gateQueuedCounter' - if delta > 0 then - redis.call('INCRBY', counterKey, delta) - elseif tonumber(redis.call('GET', counterKey) or '0') > 0 then - redis.call('DECRBY', counterKey, -delta) + if not seenBases[base] then + seenBases[base] = true + local counterKey = base .. ':gateQueuedCounter' + if delta > 0 then + if redis.call('EXISTS', counterKey) == 0 then + redis.call('SET', counterKey, '0', 'EX', '86400') + end + redis.call('INCRBY', counterKey, delta) + elseif tonumber(redis.call('GET', counterKey) or '0') > 0 then + redis.call('DECRBY', counterKey, -delta) + end end end end From 5b6663409be753f8f7551ce83924c7e4009b52f9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 04:25:13 +0100 Subject: [PATCH 46/77] fix(webapp): limit rows mirror the declared shape exactly A total-only limit no longer copies the total into the per-key column; the group set caps keyless runs, so the copy was redundant and made the row unable to say whether a per-key bound was declared. Reads of limit rows are now faithful: perKey and total columns hold exactly what the user declared. --- .../app/v3/services/createBackgroundWorker.server.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 9712d8d06f1..85ff6ba9297 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -426,7 +426,7 @@ async function createWorkerTask( if (concurrency?.inline) { if (!task.queue?.name) { - queueConcurrencyLimit = concurrency.inline.perKey ?? concurrency.inline.total; + queueConcurrencyLimit = concurrency.inline.perKey; queueTotalConcurrencyLimit = concurrency.inline.total; } else { if (compiledGates.length > 1) { @@ -438,7 +438,7 @@ async function createWorkerTask( await createWorkerQueue( { name: anonymousQueueName, - concurrencyLimit: concurrency.inline.perKey ?? concurrency.inline.total ?? null, + concurrencyLimit: concurrency.inline.perKey ?? null, combinedConcurrencyLimit: concurrency.inline.total ?? null, }, `task/${task.id}`, @@ -707,9 +707,9 @@ function assertNotReservedQueueName(name: string, context: string): void { /** * Materializes the worker's declared named concurrency limits (plus any names tasks - * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. A - * total-only limit stores the total as its per-key limit too, so no single key (or - * the keyless pool) can exceed it even before the group check applies. + * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. The + * columns mirror the declared shape exactly: perKey caps each key pool (and the + * keyless pool); total caps everything together via the group set. */ async function createWorkerConcurrencyLimits( metadata: BackgroundWorkerMetadata, @@ -731,7 +731,7 @@ async function createWorkerConcurrencyLimits( await createWorkerQueue( { name: concurrencyLimitQueueName(limit.name), - concurrencyLimit: limit.perKey ?? limit.total ?? null, + concurrencyLimit: limit.perKey ?? null, combinedConcurrencyLimit: limit.total ?? null, }, limit.name, From 1cadf893f5df8d6e3ebd178e5555cb13eef89719 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:05:30 +0100 Subject: [PATCH 47/77] feat(run-engine): expose the per-gate queued counts on the engine --- internal-packages/run-engine/src/engine/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 6347a2d1e60..85b9441efbb 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1762,6 +1762,13 @@ export class RunEngine { return this.runQueue.totalConcurrencyLimitsOfQueues(environment, queues); } + async gateQueuedCountOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.gateQueuedCountOfQueues(environment, queues); + } + async concurrencyKeyBreakdown( environment: MinimalAuthenticatedEnvironment, queue: string, From 3347b6314efeb075a2305f41aa800cc59445aec6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:05:42 +0100 Subject: [PATCH 48/77] feat(webapp): concurrency limits management API Adds list, retrieve, override and reset endpoints for named concurrency limits. Limits resolve by name against the LIMIT-role rows; running is the limit's group cardinality (keyed and keyless holders together) and queued is the exact per-gate counter of runs that must clear the limit to execute. Overrides change only the given bounds, zero blocks every holder (how a limit is paused), and reset restores the declared values. Both bounds sync to the engine on every change. --- ...pi.v1.concurrency-limits.$name.override.ts | 41 +++ .../api.v1.concurrency-limits.$name.reset.ts | 37 +++ .../routes/api.v1.concurrency-limits.$name.ts | 31 ++ .../app/routes/api.v1.concurrency-limits.ts | 43 +++ .../concurrencyLimitsSystem.server.ts | 302 ++++++++++++++++++ .../concurrencyLimitsSystemInstance.server.ts | 15 + 6 files changed, 469 insertions(+) create mode 100644 apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts create mode 100644 apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts create mode 100644 apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts create mode 100644 apps/webapp/app/routes/api.v1.concurrency-limits.ts create mode 100644 apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts create mode 100644 apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts new file mode 100644 index 00000000000..ee9c2e48ffb --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts @@ -0,0 +1,41 @@ +import { json } from "@remix-run/server-runtime"; +import { OverrideConcurrencyLimitRequestBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const ParamsSchema = z.object({ + name: z.string().transform((val) => decodeURIComponent(val)), +}); + +const route = createActionApiRoute( + { + params: ParamsSchema, + body: OverrideConcurrencyLimitRequestBody, + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, body, authentication }) => { + return concurrencyLimitsSystem.limits + .override(authentication.environment, params.name, body) + .match( + (limit) => json(limit), + (error) => { + switch (error.type) { + case "limit_not_found": + return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "invalid_override": + return json({ error: error.message }, { status: 400 }); + default: + return json({ error: "Failed to override concurrency limit" }, { status: 500 }); + } + } + ); + } +); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts new file mode 100644 index 00000000000..2ffe86d33c2 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts @@ -0,0 +1,37 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const ParamsSchema = z.object({ + name: z.string().transform((val) => decodeURIComponent(val)), +}); + +const route = createActionApiRoute( + { + params: ParamsSchema, + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, authentication }) => { + return concurrencyLimitsSystem.limits.reset(authentication.environment, params.name).match( + (limit) => json(limit), + (error) => { + switch (error.type) { + case "limit_not_found": + return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "limit_not_overridden": + return json({ error: "Concurrency limit has no override to reset" }, { status: 400 }); + default: + return json({ error: "Failed to reset concurrency limit" }, { status: 500 }); + } + } + ); + } +); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts new file mode 100644 index 00000000000..6f738d05b10 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts @@ -0,0 +1,31 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const ParamsSchema = z.object({ + name: z.string().transform((val) => decodeURIComponent(val)), +}); + +export const loader = createLoaderApiRoute( + { + params: ParamsSchema, + findResource: async () => 1, + authorization: { + action: "read", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, authentication }) => { + return concurrencyLimitsSystem.limits.retrieve(authentication.environment, params.name).match( + (limit) => json(limit), + (error) => { + if (error.type === "limit_not_found") { + return json({ error: "Concurrency limit not found" }, { status: 404 }); + } + return json({ error: "Failed to retrieve concurrency limit" }, { status: 500 }); + } + ); + } +); diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.ts new file mode 100644 index 00000000000..0b9c936019d --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.ts @@ -0,0 +1,43 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const SearchParamsSchema = z.object({ + page: z.coerce.number().int().positive().default(1), + perPage: z.coerce.number().int().positive().max(100).default(25), +}); + +export const loader = createLoaderApiRoute( + { + searchParams: SearchParamsSchema, + findResource: async () => 1, + authorization: { + action: "read", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ searchParams, authentication }) => { + const [data, count] = await Promise.all([ + concurrencyLimitsSystem.limits.list(authentication.environment, { + page: searchParams.page, + perPage: searchParams.perPage, + }), + concurrencyLimitsSystem.limits.totalCount(authentication.environment), + ]); + + if (data.isErr() || count.isErr()) { + return json({ error: "Failed to list concurrency limits" }, { status: 500 }); + } + + return json({ + data: data.value, + pagination: { + currentPage: searchParams.page, + totalPages: Math.ceil(count.value / searchParams.perPage), + count: count.value, + }, + }); + } +); diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts new file mode 100644 index 00000000000..621d7cc9ec0 --- /dev/null +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -0,0 +1,302 @@ +import type { TaskQueue, User } from "@trigger.dev/database"; +import { errAsync, fromPromise, okAsync } from "neverthrow"; +import type { PrismaClientOrTransaction } from "~/db.server"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + removeQueueConcurrencyLimits, + removeQueueTotalConcurrencyLimits, + updateQueueConcurrencyLimits, + updateQueueTotalConcurrencyLimits, +} from "../runQueue.server"; +import { engine } from "../runEngine.server"; + +export type ConcurrencyLimitsSystemOptions = { + db: PrismaClientOrTransaction; + reader: PrismaClientOrTransaction; +}; + +/** Queue rows that back named concurrency limits live under this reserved prefix. */ +const LIMIT_QUEUE_PREFIX = "limit/"; + +const LIMIT_NAME_PATTERN = /^[a-zA-Z0-9_/-]{1,122}$/; + +export type ConcurrencyLimitBoundValue = { + current: number | null; + base: number | null; + override: number | null; + overriddenAt: Date | null; +}; + +export type ConcurrencyLimitItem = { + id: string; + name: string; + perKey: ConcurrencyLimitBoundValue; + total: ConcurrencyLimitBoundValue; + running: number; + queued: number; +}; + +/** + * An override changes only the given bounds; each bound is a non-negative integer + * (zero blocks every run holding the limit, which is how a limit is paused). + */ +export type ConcurrencyLimitOverrideInput = { + perKey?: number; + total?: number; +}; + +export class ConcurrencyLimitsSystem { + constructor(private readonly options: ConcurrencyLimitsSystemOptions) {} + + private get db() { + return this.options.db; + } + + private get reader() { + return this.options.reader; + } + + get limits() { + return { + list: (environment: AuthenticatedEnvironment, page: { page: number; perPage: number }) => { + return fromPromise( + this.reader.taskQueue.findMany({ + where: { + runtimeEnvironmentId: environment.id, + role: "LIMIT", + }, + orderBy: { name: "asc" }, + skip: (page.page - 1) * page.perPage, + take: page.perPage, + }), + (error) => ({ type: "other" as const, cause: error }) + ).andThen((rows) => + fromPromise(toLimitItems(environment, rows), (error) => ({ + type: "other" as const, + cause: error, + })) + ); + }, + totalCount: (environment: AuthenticatedEnvironment) => { + return fromPromise( + this.reader.taskQueue.count({ + where: { runtimeEnvironmentId: environment.id, role: "LIMIT" }, + }), + (error) => ({ type: "other" as const, cause: error }) + ); + }, + retrieve: (environment: AuthenticatedEnvironment, name: string) => { + return findLimitByName(this.db, environment, name).andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + override: ( + environment: AuthenticatedEnvironment, + name: string, + override: ConcurrencyLimitOverrideInput, + overriddenBy?: User + ) => { + if (override.perKey === undefined && override.total === undefined) { + return errAsync({ + type: "invalid_override" as const, + message: "Provide at least one of `perKey` or `total`", + }); + } + + for (const [field, value] of Object.entries(override)) { + if (value === undefined) continue; + if (!Number.isInteger(value) || value < 0 || value > 100000) { + return errAsync({ + type: "invalid_override" as const, + message: `\`${field}\` must be an integer between 0 and 100000`, + }); + } + } + + return findLimitByName(this.db, environment, name) + .andThen((row) => applyLimitOverride(this.db, row, override, overriddenBy)) + .andThen((row) => syncLimitToEngine(environment, row)) + .andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + reset: (environment: AuthenticatedEnvironment, name: string) => { + return findLimitByName(this.db, environment, name) + .andThen((row) => resetLimitOverrides(this.db, row)) + .andThen((row) => syncLimitToEngine(environment, row)) + .andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + }; + } +} + +export function concurrencyLimitDisplayId(row: Pick): string { + return `climit_${row.friendlyId.replace(/^queue_/, "")}`; +} + +export function concurrencyLimitNameFromRow(row: Pick): string { + return row.name.startsWith(LIMIT_QUEUE_PREFIX) + ? row.name.slice(LIMIT_QUEUE_PREFIX.length) + : row.name; +} + +function findLimitByName( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + name: string +) { + if (!LIMIT_NAME_PATTERN.test(name)) { + return errAsync({ type: "limit_not_found" as const }); + } + + return fromPromise( + db.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name: `${LIMIT_QUEUE_PREFIX}${name}`, + role: "LIMIT", + }, + }), + (error) => ({ type: "other" as const, cause: error }) + ).andThen((row) => { + if (!row) { + return errAsync({ type: "limit_not_found" as const }); + } + return okAsync(row); + }); +} + +async function toLimitItems( + environment: AuthenticatedEnvironment, + rows: TaskQueue[] +): Promise { + const names = rows.map((row) => row.name); + const [running, queued] = await Promise.all([ + engine.totalConcurrencyOfQueues(environment, names), + engine.gateQueuedCountOfQueues(environment, names), + ]); + + return rows.map((row) => ({ + id: concurrencyLimitDisplayId(row), + name: concurrencyLimitNameFromRow(row), + perKey: toBound( + row.concurrencyLimit, + row.concurrencyLimitBase, + row.concurrencyLimitOverriddenAt + ), + total: toBound( + row.totalConcurrencyLimit, + row.totalConcurrencyLimitBase, + row.totalConcurrencyLimitOverriddenAt + ), + running: running[row.name] ?? 0, + queued: queued[row.name] ?? 0, + })); +} + +function toBound( + current: number | null, + base: number | null, + overriddenAt: Date | null +): ConcurrencyLimitBoundValue { + const overridden = overriddenAt !== null; + return { + current, + base: overridden ? base : current, + override: overridden ? current : null, + overriddenAt, + }; +} + +function applyLimitOverride( + db: PrismaClientOrTransaction, + row: TaskQueue, + override: ConcurrencyLimitOverrideInput, + overriddenBy?: User +) { + const now = new Date(); + const data: Record = {}; + + if (override.perKey !== undefined) { + data.concurrencyLimit = override.perKey; + data.concurrencyLimitBase = row.concurrencyLimitOverriddenAt + ? row.concurrencyLimitBase + : (row.concurrencyLimit ?? null); + data.concurrencyLimitOverriddenAt = now; + data.concurrencyLimitOverriddenBy = overriddenBy?.id ?? null; + } + + if (override.total !== undefined) { + data.totalConcurrencyLimit = override.total; + data.totalConcurrencyLimitBase = row.totalConcurrencyLimitOverriddenAt + ? row.totalConcurrencyLimitBase + : (row.totalConcurrencyLimit ?? null); + data.totalConcurrencyLimitOverriddenAt = now; + data.totalConcurrencyLimitOverriddenBy = overriddenBy?.id ?? null; + } + + return fromPromise(db.taskQueue.update({ where: { id: row.id }, data }), (error) => ({ + type: "limit_update_failed" as const, + cause: error, + })); +} + +function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { + if (row.concurrencyLimitOverriddenAt === null && row.totalConcurrencyLimitOverriddenAt === null) { + return errAsync({ type: "limit_not_overridden" as const }); + } + + const data: Record = {}; + + if (row.concurrencyLimitOverriddenAt !== null) { + data.concurrencyLimit = row.concurrencyLimitBase; + data.concurrencyLimitBase = null; + data.concurrencyLimitOverriddenAt = null; + data.concurrencyLimitOverriddenBy = null; + } + + if (row.totalConcurrencyLimitOverriddenAt !== null) { + data.totalConcurrencyLimit = row.totalConcurrencyLimitBase; + data.totalConcurrencyLimitBase = null; + data.totalConcurrencyLimitOverriddenAt = null; + data.totalConcurrencyLimitOverriddenBy = null; + } + + return fromPromise(db.taskQueue.update({ where: { id: row.id }, data }), (error) => ({ + type: "limit_update_failed" as const, + cause: error, + })); +} + +/** + * Pushes both engine keys from the row: the per-key limit and the total. Limit + * rows are never paused (pausing a limit is an override to `{ total: 0 }`), so + * both keys sync unconditionally, unlike queue rows. + */ +function syncLimitToEngine(environment: AuthenticatedEnvironment, row: TaskQueue) { + const perKeySync = + typeof row.concurrencyLimit === "number" + ? updateQueueConcurrencyLimits(environment, row.name, row.concurrencyLimit) + : removeQueueConcurrencyLimits(environment, row.name); + + const totalSync = + typeof row.totalConcurrencyLimit === "number" + ? updateQueueTotalConcurrencyLimits(environment, row.name, row.totalConcurrencyLimit) + : removeQueueTotalConcurrencyLimits(environment, row.name); + + return fromPromise(Promise.all([perKeySync, totalSync]), (error) => ({ + type: "sync_limit_to_engine_failed" as const, + cause: error, + })).map(() => row); +} diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts new file mode 100644 index 00000000000..eeaa6882e9f --- /dev/null +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts @@ -0,0 +1,15 @@ +import { prisma, $replica } from "~/db.server"; +import { ConcurrencyLimitsSystem } from "./concurrencyLimitsSystem.server"; +import { singleton } from "~/utils/singleton"; + +export const concurrencyLimitsSystem = singleton( + "concurrency-limits-system", + initializeConcurrencyLimitsSystemInstance +); + +function initializeConcurrencyLimitsSystemInstance() { + return new ConcurrencyLimitsSystem({ + db: prisma, + reader: $replica, + }); +} From ebd6a63d82e9773513127ad2b70f15a4a7b16807 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:10:44 +0100 Subject: [PATCH 49/77] chore(webapp): keep concurrency limit mapping helpers module-local --- .../app/v3/services/concurrencyLimitsSystem.server.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index 621d7cc9ec0..bf337a2183e 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -20,14 +20,14 @@ const LIMIT_QUEUE_PREFIX = "limit/"; const LIMIT_NAME_PATTERN = /^[a-zA-Z0-9_/-]{1,122}$/; -export type ConcurrencyLimitBoundValue = { +type ConcurrencyLimitBoundValue = { current: number | null; base: number | null; override: number | null; overriddenAt: Date | null; }; -export type ConcurrencyLimitItem = { +type ConcurrencyLimitItem = { id: string; name: string; perKey: ConcurrencyLimitBoundValue; @@ -40,7 +40,7 @@ export type ConcurrencyLimitItem = { * An override changes only the given bounds; each bound is a non-negative integer * (zero blocks every run holding the limit, which is how a limit is paused). */ -export type ConcurrencyLimitOverrideInput = { +type ConcurrencyLimitOverrideInput = { perKey?: number; total?: number; }; @@ -141,11 +141,11 @@ export class ConcurrencyLimitsSystem { } } -export function concurrencyLimitDisplayId(row: Pick): string { +function concurrencyLimitDisplayId(row: Pick): string { return `climit_${row.friendlyId.replace(/^queue_/, "")}`; } -export function concurrencyLimitNameFromRow(row: Pick): string { +function concurrencyLimitNameFromRow(row: Pick): string { return row.name.startsWith(LIMIT_QUEUE_PREFIX) ? row.name.slice(LIMIT_QUEUE_PREFIX.length) : row.name; From c1af5bc5de2360f25848058bfc4ed8d228956334 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:14:51 +0100 Subject: [PATCH 50/77] fix(webapp,run-engine): recoverable resets, conflict-guarded limit mutations, activity-refreshed counter TTL Limit resets sync the engine to the declared base before clearing the markers, so an engine failure leaves the override intact and a retry converges. Both limit mutations carry the read markers in their where clause and surface a conflict instead of clobbering a concurrent change. The gate queued counter's TTL now refreshes on every delta, so an active gate's count never resets while drift from delta-less paths still clears once the gate goes quiet; the flag comment documents that total bounds enforce solely through the total-concurrency flag. --- .../concurrencyLimitsSystem.server.ts | 73 ++++++++++++++++--- .../run-engine/src/run-queue/index.ts | 18 +++-- 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index bf337a2183e..f3a7ce2159c 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -1,6 +1,6 @@ import type { TaskQueue, User } from "@trigger.dev/database"; import { errAsync, fromPromise, okAsync } from "neverthrow"; -import type { PrismaClientOrTransaction } from "~/db.server"; +import { Prisma, type PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { removeQueueConcurrencyLimits, @@ -128,8 +128,8 @@ export class ConcurrencyLimitsSystem { }, reset: (environment: AuthenticatedEnvironment, name: string) => { return findLimitByName(this.db, environment, name) + .andThen((row) => syncResetToEngine(environment, row)) .andThen((row) => resetLimitOverrides(this.db, row)) - .andThen((row) => syncLimitToEngine(environment, row)) .andThen((row) => fromPromise(toLimitItems(environment, [row]), (error) => ({ type: "other" as const, @@ -246,17 +246,43 @@ function applyLimitOverride( data.totalConcurrencyLimitOverriddenBy = overriddenBy?.id ?? null; } - return fromPromise(db.taskQueue.update({ where: { id: row.id }, data }), (error) => ({ - type: "limit_update_failed" as const, - cause: error, - })); + return guardedLimitUpdate(db, row, data); } -function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { +/** + * Enforce first, then persist: the engine syncs to the declared base BEFORE the + * override markers clear, so an engine failure leaves the markers set and a retry + * converges instead of being rejected while the overridden limit stays enforced. + */ +function syncResetToEngine(environment: AuthenticatedEnvironment, row: TaskQueue) { if (row.concurrencyLimitOverriddenAt === null && row.totalConcurrencyLimitOverriddenAt === null) { return errAsync({ type: "limit_not_overridden" as const }); } + const perKeyTarget = row.concurrencyLimitOverriddenAt + ? row.concurrencyLimitBase + : row.concurrencyLimit; + const totalTarget = row.totalConcurrencyLimitOverriddenAt + ? row.totalConcurrencyLimitBase + : row.totalConcurrencyLimit; + + const perKeySync = + typeof perKeyTarget === "number" + ? updateQueueConcurrencyLimits(environment, row.name, perKeyTarget) + : removeQueueConcurrencyLimits(environment, row.name); + + const totalSync = + typeof totalTarget === "number" + ? updateQueueTotalConcurrencyLimits(environment, row.name, totalTarget) + : removeQueueTotalConcurrencyLimits(environment, row.name); + + return fromPromise(Promise.all([perKeySync, totalSync]), (error) => ({ + type: "sync_limit_to_engine_failed" as const, + cause: error, + })).map(() => row); +} + +function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { const data: Record = {}; if (row.concurrencyLimitOverriddenAt !== null) { @@ -273,10 +299,35 @@ function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { data.totalConcurrencyLimitOverriddenBy = null; } - return fromPromise(db.taskQueue.update({ where: { id: row.id }, data }), (error) => ({ - type: "limit_update_failed" as const, - cause: error, - })); + return guardedLimitUpdate(db, row, data); +} + +/** + * Optimistic update: the where clause carries the override markers as read, so a + * concurrent override or reset makes this update miss (P2025) and the caller gets + * a conflict instead of silently clobbering the newer state. + */ +function guardedLimitUpdate( + db: PrismaClientOrTransaction, + row: TaskQueue, + data: Record +) { + return fromPromise( + db.taskQueue.update({ + where: { + id: row.id, + concurrencyLimitOverriddenAt: row.concurrencyLimitOverriddenAt, + totalConcurrencyLimitOverriddenAt: row.totalConcurrencyLimitOverriddenAt, + }, + data, + }), + (error) => { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2025") { + return { type: "conflict" as const }; + } + return { type: "limit_update_failed" as const, cause: error }; + } + ); } /** diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 068b90b54ef..c29057f824b 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -166,10 +166,11 @@ end -- Callers gate the delta on the actual queue-zset transition (ZADD added == 1 / -- ZREM removed == 1) so re-enqueues and already-removed members never double count; -- gates sharing a base (duplicate entries, key variants) count once per run. The --- 24h absolute TTL (set at creation, never extended) re-anchors drift from paths --- without the delta (rolling deploys, stale-entry cleanup): the counter resets, --- floored decrements absorb the pre-reset backlog as it drains, and counts converge --- to exact for every run enqueued after the reset. Payload-driven and +-- 24h TTL refreshes on every delta, so an ACTIVE gate's count never resets while +-- drift from delta-less paths (a mixed-version rollout, a stale-entry cleanup) +-- clears once the gate has been quiet for a day. The residual gap is a gate idle +-- for 24h with runs still queued (e.g. paused with no new enqueues): its counter +-- expires and under-counts until the backlog fully drains. Payload-driven and -- flag-independent, like release, so counts stay exact across flag flips. local function __gateQueuedDelta(gatesKeyPrefix, msg, delta) if type(msg) ~= 'table' or not msg.gates then return end @@ -180,12 +181,11 @@ local function __gateQueuedDelta(gatesKeyPrefix, msg, delta) seenBases[base] = true local counterKey = base .. ':gateQueuedCounter' if delta > 0 then - if redis.call('EXISTS', counterKey) == 0 then - redis.call('SET', counterKey, '0', 'EX', '86400') - end redis.call('INCRBY', counterKey, delta) + redis.call('EXPIRE', counterKey, '86400') elseif tonumber(redis.call('GET', counterKey) or '0') > 0 then redis.call('DECRBY', counterKey, -delta) + redis.call('EXPIRE', counterKey, '86400') end end end @@ -347,7 +347,9 @@ export type RunQueueOptions = { /** * When true, queues maintain a per-base-queue groupConcurrency SET (total in-flight * across all key variants AND keyless runs) and enforce the queue's total concurrency - * limit at admit time. Default false: admit paths are byte-identical to before, and + * limit at admit time. V2 concurrency semantics (a limit's `total` bound, including + * total-only declarations) are enforced solely through this flag: with it off, a + * total-only limit caps nothing. Default false: admit paths are byte-identical to before, and * only the release-side SREM mirror runs (a no-op on an absent set), so the flag can * be flipped on a fleet that has fully rolled onto this build without draining queues. * From b7593427f28024c9ea33370f74829fdb63ad182a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:15:47 +0100 Subject: [PATCH 51/77] fix(webapp): limit mutations answer concurrent changes with a 409 --- .../app/routes/api.v1.concurrency-limits.$name.override.ts | 5 +++++ .../app/routes/api.v1.concurrency-limits.$name.reset.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts index ee9c2e48ffb..e00d078bdb0 100644 --- a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts @@ -27,6 +27,11 @@ const route = createActionApiRoute( switch (error.type) { case "limit_not_found": return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "conflict": + return json( + { error: "The limit changed concurrently; retry the request" }, + { status: 409 } + ); case "invalid_override": return json({ error: error.message }, { status: 400 }); default: diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts index 2ffe86d33c2..9e62dfee46f 100644 --- a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts @@ -23,6 +23,11 @@ const route = createActionApiRoute( switch (error.type) { case "limit_not_found": return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "conflict": + return json( + { error: "The limit changed concurrently; retry the request" }, + { status: 409 } + ); case "limit_not_overridden": return json({ error: "Concurrency limit has no override to reset" }, { status: 400 }); default: From dd2aeffc60f74000c1fe34f883a683f1c1c5ed68 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:23:26 +0100 Subject: [PATCH 52/77] test(webapp): focused coverage for concurrency limit mutations Covers partial overrides and base preservation, the zero-total pause, reset round trips, the enforce-first ordering under an engine failure (marker survives, retry converges), the optimistic conflict when the markers move underneath a mutation, and role/name resolution misses. --- .../test/concurrencyLimitsSystem.test.ts | 225 ++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 apps/webapp/test/concurrencyLimitsSystem.test.ts diff --git a/apps/webapp/test/concurrencyLimitsSystem.test.ts b/apps/webapp/test/concurrencyLimitsSystem.test.ts new file mode 100644 index 00000000000..850d9c96286 --- /dev/null +++ b/apps/webapp/test/concurrencyLimitsSystem.test.ts @@ -0,0 +1,225 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { ConcurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystem.server"; + +/** + * These tests exercise the DB-write, marker and ordering logic against a real + * Postgres. The engine syncs are spies so tests can assert ordering and inject + * failures; the Redis side itself is covered by the run-engine suites. + */ +const { perKeySyncMock, perKeyRemoveMock, totalSyncMock, totalRemoveMock } = vi.hoisted(() => ({ + perKeySyncMock: vi.fn(async (..._args: unknown[]) => undefined), + perKeyRemoveMock: vi.fn(async (..._args: unknown[]) => undefined), + totalSyncMock: vi.fn(async (..._args: unknown[]) => undefined), + totalRemoveMock: vi.fn(async (..._args: unknown[]) => undefined), +})); + +vi.mock("~/v3/runQueue.server", () => ({ + updateQueueConcurrencyLimits: perKeySyncMock, + removeQueueConcurrencyLimits: perKeyRemoveMock, + updateQueueTotalConcurrencyLimits: totalSyncMock, + removeQueueTotalConcurrencyLimits: totalRemoveMock, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + totalConcurrencyOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + gateQueuedCountOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + }, +})); + +vi.setConfig({ testTimeout: 30_000 }); + +async function seedEnvAndLimit( + prisma: PrismaClient, + opts: { perKey?: number | null; total?: number | null } = {} +) { + const slug = `s${Math.random().toString(36).slice(2, 10)}`; + + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: slug, + pkApiKey: slug, + shortcode: slug, + maximumConcurrencyLimit: 100, + }, + }); + + const row = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${slug}`, + name: "limit/openai", + orderableName: "openai", + projectId: project.id, + runtimeEnvironmentId: environment.id, + role: "LIMIT", + concurrencyVersion: "V2", + concurrencyLimit: opts.perKey ?? null, + totalConcurrencyLimit: opts.total ?? null, + }, + }); + + const authEnv = { + id: environment.id, + maximumConcurrencyLimit: environment.maximumConcurrencyLimit, + } as unknown as AuthenticatedEnvironment; + + const system = new ConcurrencyLimitsSystem({ db: prisma, reader: prisma }); + + return { environment, row, authEnv, system }; +} + +describe("ConcurrencyLimitsSystem", () => { + postgresTest( + "override changes only the given bound and keeps the declared base", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.override(authEnv, "openai", { total: 50 }); + expect(result.isOk()).toBe(true); + if (result.isOk()) { + expect(result.value.total).toMatchObject({ current: 50, base: 25, override: 50 }); + expect(result.value.perKey).toMatchObject({ current: null, override: null }); + } + + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.totalConcurrencyLimit).toBe(50); + expect(updated.totalConcurrencyLimitBase).toBe(25); + expect(updated.totalConcurrencyLimitOverriddenAt).not.toBeNull(); + expect(updated.concurrencyLimitOverriddenAt).toBeNull(); + + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "limit/openai", 50); + expect(perKeyRemoveMock).toHaveBeenCalledWith(authEnv, "limit/openai"); + } + ); + + postgresTest("override to zero pauses the limit in the engine", async ({ prisma }) => { + const { authEnv, system } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.override(authEnv, "openai", { total: 0 }); + expect(result.isOk()).toBe(true); + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "limit/openai", 0); + }); + + postgresTest("reset restores the declared values and clears the markers", async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { perKey: 2, total: 25 }); + + await system.limits.override(authEnv, "openai", { perKey: 10, total: 50 }); + const result = await system.limits.reset(authEnv, "openai"); + expect(result.isOk()).toBe(true); + if (result.isOk()) { + expect(result.value.total).toMatchObject({ current: 25, base: 25, override: null }); + expect(result.value.perKey).toMatchObject({ current: 2, base: 2, override: null }); + } + + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.concurrencyLimit).toBe(2); + expect(updated.totalConcurrencyLimit).toBe(25); + expect(updated.concurrencyLimitOverriddenAt).toBeNull(); + expect(updated.totalConcurrencyLimitOverriddenAt).toBeNull(); + }); + + postgresTest("reset without an override is rejected", async ({ prisma }) => { + const { authEnv, system } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.reset(authEnv, "openai"); + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error.type).toBe("limit_not_overridden"); + } + }); + + postgresTest( + "a failed engine sync during reset leaves the override intact so a retry converges", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + await system.limits.override(authEnv, "openai", { total: 50 }); + + totalSyncMock.mockRejectedValueOnce(new Error("redis down")); + const failed = await system.limits.reset(authEnv, "openai"); + expect(failed.isErr()).toBe(true); + + /** The marker must survive the failed sync: the DB still says overridden. */ + const midway = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(midway.totalConcurrencyLimitOverriddenAt).not.toBeNull(); + expect(midway.totalConcurrencyLimit).toBe(50); + + const retried = await system.limits.reset(authEnv, "openai"); + expect(retried.isOk()).toBe(true); + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(final.totalConcurrencyLimit).toBe(25); + expect(final.totalConcurrencyLimitOverriddenAt).toBeNull(); + } + ); + + postgresTest( + "a mutation whose markers moved underneath it conflicts instead of clobbering", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + await system.limits.override(authEnv, "openai", { total: 50 }); + + /** + * Interleave a concurrent reset between this mutation's read and its write: + * the engine sync hook is the seam after the read, so clearing the markers + * there makes the guarded update miss and surface a conflict. + */ + totalSyncMock.mockImplementationOnce(async () => { + await prisma.taskQueue.update({ + where: { id: row.id }, + data: { + totalConcurrencyLimit: 25, + totalConcurrencyLimitBase: null, + totalConcurrencyLimitOverriddenAt: null, + totalConcurrencyLimitOverriddenBy: null, + }, + }); + }); + + const raced = await system.limits.reset(authEnv, "openai"); + expect(raced.isErr()).toBe(true); + if (raced.isErr()) { + expect(raced.error.type).toBe("conflict"); + } + + /** The concurrent actor's state stands untouched. */ + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(final.totalConcurrencyLimit).toBe(25); + expect(final.totalConcurrencyLimitOverriddenAt).toBeNull(); + } + ); + + postgresTest("retrieve misses queue-role rows and unknown names", async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_q${environment.slug}`, + name: "limit/shadow", + orderableName: "shadow", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + }, + }); + + const missing = await system.limits.retrieve(authEnv, "missing"); + expect(missing.isErr()).toBe(true); + + const shadow = await system.limits.retrieve(authEnv, "shadow"); + expect(shadow.isErr()).toBe(true); + }); +}); From d9e9fbbd569f652f47df02550796c808f897e317 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:32:13 +0100 Subject: [PATCH 53/77] fix(webapp,run-engine): limit mutations survive every race direction The optimistic guard now carries the row's updatedAt, so a deploy refreshing declared values (which deliberately keeps the markers) conflicts a concurrent mutation instead of losing its write. A reset whose persist fails after the engine already reverted re-syncs the engine from a fresh read, so a concurrent actor's override or pause is enforced again. Override bounds are validated against the environment maximum like queue overrides. Counter reads refresh the TTL so an observed gate never re-anchors while idle, and the routes drop a redundant decode that turned malformed names into 500s. --- ...pi.v1.concurrency-limits.$name.override.ts | 2 +- .../api.v1.concurrency-limits.$name.reset.ts | 2 +- .../routes/api.v1.concurrency-limits.$name.ts | 2 +- .../concurrencyLimitsSystem.server.ts | 42 ++++++++++++++++--- .../run-engine/src/run-queue/index.ts | 23 ++++++---- 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts index e00d078bdb0..3c2f79afd13 100644 --- a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.override.ts @@ -5,7 +5,7 @@ import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; const ParamsSchema = z.object({ - name: z.string().transform((val) => decodeURIComponent(val)), + name: z.string(), }); const route = createActionApiRoute( diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts index 9e62dfee46f..5e0af9192b1 100644 --- a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts @@ -4,7 +4,7 @@ import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; const ParamsSchema = z.object({ - name: z.string().transform((val) => decodeURIComponent(val)), + name: z.string(), }); const route = createActionApiRoute( diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts index 6f738d05b10..3035de12baf 100644 --- a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts @@ -4,7 +4,7 @@ import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; const ParamsSchema = z.object({ - name: z.string().transform((val) => decodeURIComponent(val)), + name: z.string(), }); export const loader = createLoaderApiRoute( diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index f3a7ce2159c..d757b844daa 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -114,6 +114,12 @@ export class ConcurrencyLimitsSystem { message: `\`${field}\` must be an integer between 0 and 100000`, }); } + if (value > environment.maximumConcurrencyLimit) { + return errAsync({ + type: "invalid_override" as const, + message: `\`${field}\` (${value}) cannot exceed the environment limit (${environment.maximumConcurrencyLimit})`, + }); + } } return findLimitByName(this.db, environment, name) @@ -129,7 +135,13 @@ export class ConcurrencyLimitsSystem { reset: (environment: AuthenticatedEnvironment, name: string) => { return findLimitByName(this.db, environment, name) .andThen((row) => syncResetToEngine(environment, row)) - .andThen((row) => resetLimitOverrides(this.db, row)) + .andThen((row) => + resetLimitOverrides(this.db, row).orElse((error) => + compensateEngineFromFreshRow(this.db, environment, row.id).andThen(() => + errAsync(error) + ) + ) + ) .andThen((row) => fromPromise(toLimitItems(environment, [row]), (error) => ({ type: "other" as const, @@ -303,9 +315,10 @@ function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { } /** - * Optimistic update: the where clause carries the override markers as read, so a - * concurrent override or reset makes this update miss (P2025) and the caller gets - * a conflict instead of silently clobbering the newer state. + * Optimistic update: the where clause carries the row's updatedAt as read, so ANY + * concurrent write — another override or reset, or a deploy refreshing the declared + * values — makes this update miss (P2025) and the caller gets a conflict instead of + * persisting values computed from a stale row. */ function guardedLimitUpdate( db: PrismaClientOrTransaction, @@ -316,8 +329,7 @@ function guardedLimitUpdate( db.taskQueue.update({ where: { id: row.id, - concurrencyLimitOverriddenAt: row.concurrencyLimitOverriddenAt, - totalConcurrencyLimitOverriddenAt: row.totalConcurrencyLimitOverriddenAt, + updatedAt: row.updatedAt, }, data, }), @@ -330,6 +342,24 @@ function guardedLimitUpdate( ); } +/** + * A reset's engine write precedes its guarded persist (enforce-first, so an engine + * failure retries cleanly), which leaves the engine reverted when the persist + * conflicts or fails. This re-syncs the engine from a fresh read of the row so a + * concurrent actor's state (or the still-standing override) is enforced again; + * the original error still reaches the caller. + */ +function compensateEngineFromFreshRow( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + rowId: string +) { + return fromPromise(db.taskQueue.findFirst({ where: { id: rowId } }), (error) => ({ + type: "other" as const, + cause: error, + })).andThen((fresh) => (fresh ? syncLimitToEngine(environment, fresh) : okAsync(null))); +} + /** * Pushes both engine keys from the row: the per-key limit and the total. Limit * rows are never paused (pausing a limit is an override to `{ total: 0 }`), so diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index c29057f824b..bc03d61f301 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -78,6 +78,10 @@ const SemanticAttributes = { * during a rolling upgrade), including runs parked in the DLQ or suspended on * checkpoints, which older rules based on the queue zset could never prune. */ +/** TTL for gate queued counters: refreshed on every delta and on every read, so an + * active or observed gate never re-anchors; the Lua helper inlines the same value. */ +const GATE_QUEUED_COUNTER_TTL_SECONDS = 86400; + const QUEUE_GATES_LUA_HELPERS = ` local function __gateKeys(gatesKeyPrefix, msg, gate) local base = gatesKeyPrefix .. '{org:' .. msg.orgId .. '}:proj:' .. msg.projectId .. ':env:' .. msg.environmentId .. ':queue:' .. gate.queue @@ -696,29 +700,34 @@ export class RunQueue { /** * Runs that are queued and must clear this gate queue to execute (the per-gate - * queued counter). Exact by construction: incremented per gate on enqueue and - * decremented on admit and on every queued-removal path, floored at zero. + * queued counter): incremented per gate on enqueue and decremented on admit and + * on every queued-removal path, floored at zero. Reads refresh the counter's + * TTL, so a gate anyone observes (dashboard, API) never expires while idle — + * e.g. a paused limit with a stalled backlog keeps its count over a quiet + * weekend; only gates nobody touches or reads for a day re-anchor. */ public async gateQueuedCountOfQueue(env: MinimalAuthenticatedEnvironment, queue: string) { - const result = await this.redis.get(this.keys.gateQueuedCounterKey(env, queue)); - return result ? Math.max(Number(result), 0) : 0; + const counts = await this.gateQueuedCountOfQueues(env, [queue]); + return counts[queue] ?? 0; } - /** Batch variant of gateQueuedCountOfQueue: one pipeline of GETs. */ + /** Batch variant: one pipeline of GETs, each with a TTL refresh. */ public async gateQueuedCountOfQueues( env: MinimalAuthenticatedEnvironment, queues: string[] ): Promise> { const pipeline = this.redis.pipeline(); queues.forEach((queue) => { - pipeline.get(this.keys.gateQueuedCounterKey(env, queue)); + const key = this.keys.gateQueuedCounterKey(env, queue); + pipeline.get(key); + pipeline.expire(key, GATE_QUEUED_COUNTER_TTL_SECONDS, "XX"); }); const results = await pipeline.exec(); return queues.reduce( (acc, queue, index) => { - const value = results?.[index]?.[1]; + const value = results?.[index * 2]?.[1]; const parsed = typeof value === "string" ? Number(value) : 0; acc[queue] = Number.isFinite(parsed) ? Math.max(parsed, 0) : 0; return acc; From 18ffedfc461986c138991fb5e0dbc10bd86ad5a1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:38:28 +0100 Subject: [PATCH 54/77] fix(webapp): reset compensation converges instead of restoring a stale row The compensating engine re-sync now re-reads and re-syncs until the row's updatedAt stops moving (bounded), the same convergence the deploy sync uses, so a mutation landing mid-compensation is the last engine write instead of being overwritten by a stale snapshot. --- .../concurrencyLimitsSystem.server.ts | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index d757b844daa..b0259515c1f 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -345,19 +345,41 @@ function guardedLimitUpdate( /** * A reset's engine write precedes its guarded persist (enforce-first, so an engine * failure retries cleanly), which leaves the engine reverted when the persist - * conflicts or fails. This re-syncs the engine from a fresh read of the row so a - * concurrent actor's state (or the still-standing override) is enforced again; - * the original error still reaches the caller. + * conflicts or fails. This re-syncs the engine from fresh reads of the row until + * updatedAt stops moving (bounded), the same convergence the deploy sync uses: + * every actor writes Postgres before its own engine sync, so re-syncing whatever + * is freshest converges. The original error still reaches the caller. */ function compensateEngineFromFreshRow( db: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, rowId: string ) { - return fromPromise(db.taskQueue.findFirst({ where: { id: rowId } }), (error) => ({ - type: "other" as const, - cause: error, - })).andThen((fresh) => (fresh ? syncLimitToEngine(environment, fresh) : okAsync(null))); + return fromPromise( + (async () => { + let lastSyncedAt: number | null = null; + for (let i = 0; i < 3; i++) { + const fresh = await db.taskQueue.findFirst({ where: { id: rowId } }); + if (!fresh || fresh.updatedAt.getTime() === lastSyncedAt) { + return; + } + await Promise.all([ + typeof fresh.concurrencyLimit === "number" + ? updateQueueConcurrencyLimits(environment, fresh.name, fresh.concurrencyLimit) + : removeQueueConcurrencyLimits(environment, fresh.name), + typeof fresh.totalConcurrencyLimit === "number" + ? updateQueueTotalConcurrencyLimits( + environment, + fresh.name, + fresh.totalConcurrencyLimit + ) + : removeQueueTotalConcurrencyLimits(environment, fresh.name), + ]); + lastSyncedAt = fresh.updatedAt.getTime(); + } + })(), + (error) => ({ type: "other" as const, cause: error }) + ); } /** From 804186a4a32c59735ce6a154b514c447e64d4f2d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:41:59 +0100 Subject: [PATCH 55/77] fix(webapp): reset failures keep their original error and every path compensates The compensating re-sync preserves the caller's error (a conflict stays a 409 even when the compensation itself flakes) and also runs when the enforce-first engine sync partially fails, so a half-reverted engine is re-synced to the standing override. The optimistic guard carries the override markers alongside updatedAt, narrowing the same-millisecond window to writes that leave both untouched. --- .../concurrencyLimitsSystem.server.ts | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index b0259515c1f..c539ac1487f 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -1,5 +1,5 @@ import type { TaskQueue, User } from "@trigger.dev/database"; -import { errAsync, fromPromise, okAsync } from "neverthrow"; +import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow"; import { Prisma, type PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { @@ -134,12 +134,18 @@ export class ConcurrencyLimitsSystem { }, reset: (environment: AuthenticatedEnvironment, name: string) => { return findLimitByName(this.db, environment, name) - .andThen((row) => syncResetToEngine(environment, row)) + .andThen((row) => + syncResetToEngine(environment, row).orElse((error) => + compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) + ) + ) .andThen((row) => resetLimitOverrides(this.db, row).orElse((error) => - compensateEngineFromFreshRow(this.db, environment, row.id).andThen(() => - errAsync(error) - ) + compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) ) ) .andThen((row) => @@ -266,7 +272,13 @@ function applyLimitOverride( * override markers clear, so an engine failure leaves the markers set and a retry * converges instead of being rejected while the overridden limit stays enforced. */ -function syncResetToEngine(environment: AuthenticatedEnvironment, row: TaskQueue) { +function syncResetToEngine( + environment: AuthenticatedEnvironment, + row: TaskQueue +): ResultAsync< + TaskQueue, + { type: "limit_not_overridden" } | { type: "sync_limit_to_engine_failed"; cause: unknown } +> { if (row.concurrencyLimitOverriddenAt === null && row.totalConcurrencyLimitOverriddenAt === null) { return errAsync({ type: "limit_not_overridden" as const }); } @@ -315,10 +327,12 @@ function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { } /** - * Optimistic update: the where clause carries the row's updatedAt as read, so ANY - * concurrent write — another override or reset, or a deploy refreshing the declared - * values — makes this update miss (P2025) and the caller gets a conflict instead of - * persisting values computed from a stale row. + * Optimistic update: the where clause carries the row's updatedAt plus both override + * markers as read, so ANY concurrent write — another override or reset, or a deploy + * refreshing the declared values — makes this update miss (P2025) and the caller + * gets a conflict instead of persisting values computed from a stale row. The + * markers narrow the same-millisecond updatedAt window to writes that also leave + * both markers untouched. */ function guardedLimitUpdate( db: PrismaClientOrTransaction, @@ -330,6 +344,8 @@ function guardedLimitUpdate( where: { id: row.id, updatedAt: row.updatedAt, + concurrencyLimitOverriddenAt: row.concurrencyLimitOverriddenAt, + totalConcurrencyLimitOverriddenAt: row.totalConcurrencyLimitOverriddenAt, }, data, }), From 1220066c4562c9497561c0cb5a692c7baa47a75e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:46:44 +0100 Subject: [PATCH 56/77] fix(webapp): engine writes settle before any failure is reported Both bounds' engine writes now settle before a sync step fails, so no write is still in flight when the compensating re-sync runs; a late sibling can never land after the compensation and leave one bound stale. --- .../concurrencyLimitsSystem.server.ts | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index c539ac1487f..3e16819af94 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -300,7 +300,7 @@ function syncResetToEngine( ? updateQueueTotalConcurrencyLimits(environment, row.name, totalTarget) : removeQueueTotalConcurrencyLimits(environment, row.name); - return fromPromise(Promise.all([perKeySync, totalSync]), (error) => ({ + return fromPromise(settleBothEngineWrites(perKeySync, totalSync), (error) => ({ type: "sync_limit_to_engine_failed" as const, cause: error, })).map(() => row); @@ -326,6 +326,19 @@ function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { return guardedLimitUpdate(db, row, data); } +/** + * Both engine writes settle before a failure is reported, so no write is still in + * flight when a caller's compensation runs — a late sibling can never land after + * the compensating re-sync and leave one bound stale. + */ +async function settleBothEngineWrites(a: Promise, b: Promise): Promise { + const results = await Promise.allSettled([a, b]); + const failed = results.find((result) => result.status === "rejected"); + if (failed && failed.status === "rejected") { + throw failed.reason; + } +} + /** * Optimistic update: the where clause carries the row's updatedAt plus both override * markers as read, so ANY concurrent write — another override or reset, or a deploy @@ -379,7 +392,7 @@ function compensateEngineFromFreshRow( if (!fresh || fresh.updatedAt.getTime() === lastSyncedAt) { return; } - await Promise.all([ + await settleBothEngineWrites( typeof fresh.concurrencyLimit === "number" ? updateQueueConcurrencyLimits(environment, fresh.name, fresh.concurrencyLimit) : removeQueueConcurrencyLimits(environment, fresh.name), @@ -389,8 +402,8 @@ function compensateEngineFromFreshRow( fresh.name, fresh.totalConcurrencyLimit ) - : removeQueueTotalConcurrencyLimits(environment, fresh.name), - ]); + : removeQueueTotalConcurrencyLimits(environment, fresh.name) + ); lastSyncedAt = fresh.updatedAt.getTime(); } })(), @@ -414,7 +427,7 @@ function syncLimitToEngine(environment: AuthenticatedEnvironment, row: TaskQueue ? updateQueueTotalConcurrencyLimits(environment, row.name, row.totalConcurrencyLimit) : removeQueueTotalConcurrencyLimits(environment, row.name); - return fromPromise(Promise.all([perKeySync, totalSync]), (error) => ({ + return fromPromise(settleBothEngineWrites(perKeySync, totalSync), (error) => ({ type: "sync_limit_to_engine_failed" as const, cause: error, })).map(() => row); From 8e2f27d3ae29f9dd0d673005ef4e9f9833362d5d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 05:48:08 +0100 Subject: [PATCH 57/77] fix(webapp): no compensation on the not-overridden validation error Resetting a limit with no override fails before any engine write, so the compensating re-sync is skipped for that error instead of issuing spurious reads and engine writes on a pure validation path. --- .../app/v3/services/concurrencyLimitsSystem.server.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts index 3e16819af94..9b89aa67d4e 100644 --- a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -136,9 +136,11 @@ export class ConcurrencyLimitsSystem { return findLimitByName(this.db, environment, name) .andThen((row) => syncResetToEngine(environment, row).orElse((error) => - compensateEngineFromFreshRow(this.db, environment, row.id) - .orElse(() => okAsync(undefined)) - .andThen(() => errAsync(error)) + error.type === "limit_not_overridden" + ? errAsync(error) + : compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) ) ) .andThen((row) => From 41ff31d0ca9d71d2e138892495dd9c999a68ea8d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 09:36:35 +0100 Subject: [PATCH 58/77] feat(webapp): named concurrency limits appear on the queues page The list interleaves LIMIT rows alongside queues (name order, shared pagination; the type filter stays queues-only): running is the limit's group cardinality, queued is its exact per-gate counter, the name strips the limit/ prefix with a Limit badge and its own icon, and V2 rows' Limit cell self-labels as per key or total. Limit rows open the detail page (their gauges already chart) and hide the queue-only pause and override actions. --- .../app/components/runs/v3/QueueName.tsx | 30 ++ .../v3/QueueListPresenter.server.ts | 80 +++-- .../v3/QueueRetrievePresenter.server.ts | 32 +- .../route.tsx | 324 +++++++++++------- .../route.tsx | 6 +- 5 files changed, 319 insertions(+), 153 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/QueueName.tsx b/apps/webapp/app/components/runs/v3/QueueName.tsx index e65b86a220e..056434652bb 100644 --- a/apps/webapp/app/components/runs/v3/QueueName.tsx +++ b/apps/webapp/app/components/runs/v3/QueueName.tsx @@ -1,19 +1,49 @@ import { TasksIcon } from "~/assets/icons/TasksIcon"; +import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { cn } from "~/utils/cn"; import { RectangleStackIcon } from "@heroicons/react/20/solid"; +const LIMIT_PREFIX = "limit/"; +const TASK_PREFIX = "task/"; + export function QueueName({ name, type, + kind, paused, className, }: { name: string; type: "task" | "custom"; + /** "limit" rows are named concurrency limits rather than queues. */ + kind?: "queue" | "limit"; paused?: boolean; className?: string; }) { + if (kind === "limit") { + const displayName = name.startsWith(LIMIT_PREFIX) ? name.slice(LIMIT_PREFIX.length) : name; + return ( + + + } + content={ + displayName.startsWith(TASK_PREFIX) + ? `This is the inline concurrency limit of your "${displayName.slice( + TASK_PREFIX.length + )}" task` + : "This is a named concurrency limit declared in your code." + } + /> + {displayName} + + ); + } + return ( {type === "task" ? ( diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 70b02f9a4da..c6c8a68a104 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -11,7 +11,10 @@ import { toQueueItem } from "./QueueRetrievePresenter.server"; type QueueListEngine = Pick< RunEngine, - "lengthOfQueues" | "currentConcurrencyOfQueues" | "totalConcurrencyOfQueues" + | "lengthOfQueues" + | "currentConcurrencyOfQueues" + | "totalConcurrencyOfQueues" + | "gateQueuedCountOfQueues" >; export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; @@ -42,6 +45,8 @@ const queueListSelect = { totalConcurrencyLimitOverriddenAt: true, type: true, paused: true, + role: true, + concurrencyVersion: true, } satisfies Prisma.TaskQueueSelect; type QueueListRow = Prisma.TaskQueueGetPayload<{ select: typeof queueListSelect }>; @@ -50,6 +55,10 @@ type QueueListRow = Prisma.TaskQueueGetPayload<{ select: typeof queueListSelect // schema (that's a public contract), so we surface it as an extra field on the list item. type QueueListItem = ReturnType & { concurrencyLimitOverridePercent: number | null; + /** "queue" rows wait and order runs; "limit" rows are named concurrency limits. */ + kind: "queue" | "limit"; + /** V2 rows hold the new perKey/total vocabulary in their limit columns. */ + concurrencyVersion: "V1" | "V2"; }; type QueueListPagination = @@ -76,7 +85,9 @@ function buildQueueListWhere( return { runtimeEnvironmentId: environmentId, - role: "QUEUE" as const, + /** The type filter names queue shapes, so applying it scopes the list to queue rows; + * without it the list interleaves named limits alongside queues. */ + role: type ? ("QUEUE" as const) : { in: ["QUEUE" as const, "LIMIT" as const] }, version: "V2", name: trimmedQuery ? { @@ -345,25 +356,44 @@ export class QueueListPresenter extends BasePresenter { totalConcurrencyLimitOverriddenAt: Date | null; type: TaskQueueType; paused: boolean; + role: "QUEUE" | "LIMIT"; + concurrencyVersion: "V1" | "V2"; }[] ): Promise { - const queuesWithTotalCap = queues.filter((q) => q.totalConcurrencyLimit !== null); - const [queuedByQueue, runningByQueue, totalRunningByQueue] = await Promise.all([ - this.engineClient.lengthOfQueues( - environment, - queues.map((q) => q.name) - ), - this.engineClient.currentConcurrencyOfQueues( - environment, - queues.map((q) => q.name) - ), - queuesWithTotalCap.length > 0 - ? this.engineClient.totalConcurrencyOfQueues( - environment, - queuesWithTotalCap.map((q) => q.name) - ) - : Promise.resolve({} as Record), - ]); + const queueRows = queues.filter((q) => q.role === "QUEUE"); + const limitRows = queues.filter((q) => q.role === "LIMIT"); + /** + * Queue rows read their zset length and home concurrency; limit rows read the + * group set (every holder, keyed or keyless) as running and the per-gate queued + * counter as queued. The group read also serves queue rows with a total cap. + */ + const rowsWithGroupRead = [ + ...queueRows.filter((q) => q.totalConcurrencyLimit !== null), + ...limitRows, + ]; + const [queuedByQueue, runningByQueue, totalRunningByQueue, gateQueuedByQueue] = + await Promise.all([ + this.engineClient.lengthOfQueues( + environment, + queueRows.map((q) => q.name) + ), + this.engineClient.currentConcurrencyOfQueues( + environment, + queueRows.map((q) => q.name) + ), + rowsWithGroupRead.length > 0 + ? this.engineClient.totalConcurrencyOfQueues( + environment, + rowsWithGroupRead.map((q) => q.name) + ) + : Promise.resolve({} as Record), + limitRows.length > 0 + ? this.engineClient.gateQueuedCountOfQueues( + environment, + limitRows.map((q) => q.name) + ) + : Promise.resolve({} as Record), + ]); // Manually "join" the overridden users because there is no way to implement the relationship // in prisma without adding a foreign key constraint @@ -381,8 +411,14 @@ export class QueueListPresenter extends BasePresenter { friendlyId: queue.friendlyId, name: queue.name, type: queue.type, - running: runningByQueue[queue.name] ?? 0, - queued: queuedByQueue[queue.name] ?? 0, + running: + queue.role === "LIMIT" + ? (totalRunningByQueue[queue.name] ?? 0) + : (runningByQueue[queue.name] ?? 0), + queued: + queue.role === "LIMIT" + ? (gateQueuedByQueue[queue.name] ?? 0) + : (queuedByQueue[queue.name] ?? 0), concurrencyLimit: queue.concurrencyLimit ?? null, concurrencyLimitBase: queue.concurrencyLimitBase ?? null, concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, @@ -401,6 +437,8 @@ export class QueueListPresenter extends BasePresenter { queue.concurrencyLimitOverridePercent !== null ? Number(queue.concurrencyLimitOverridePercent) : null, + kind: queue.role === "LIMIT" ? ("limit" as const) : ("queue" as const), + concurrencyVersion: queue.concurrencyVersion, })); } } diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 9d6e1d17712..cddaff28ae4 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -22,8 +22,19 @@ export type FoundQueue = Prettify< export async function getQueue( prismaClient: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, - queue: RetrieveQueueParam + queue: RetrieveQueueParam, + options?: { + /** + * The dashboard's detail page shows limit rows too; the public API and pause + * flows stay scoped to queue rows. + */ + includeLimits?: boolean; + } ) { + const role = options?.includeLimits + ? { in: ["QUEUE" as const, "LIMIT" as const] } + : ("QUEUE" as const); + if (typeof queue === "string") { return joinQueueWithUser( prismaClient, @@ -31,7 +42,7 @@ export async function getQueue( where: { friendlyId: queue, runtimeEnvironmentId: environment.id, - role: "QUEUE", + role, }, }) ); @@ -45,7 +56,7 @@ export async function getQueue( where: { name: queueName, runtimeEnvironmentId: environment.id, - role: "QUEUE", + role, }, }) ); @@ -77,11 +88,13 @@ export class QueueRetrievePresenter extends BasePresenter { public async call({ environment, queueInput, + includeLimits, }: { environment: AuthenticatedEnvironment; queueInput: RetrieveQueueParam; + includeLimits?: boolean; }) { - const queue = await getQueue(this._replica, environment, queueInput); + const queue = await getQueue(this._replica, environment, queueInput, { includeLimits }); if (!queue) { return { success: false as const, @@ -89,9 +102,14 @@ export class QueueRetrievePresenter extends BasePresenter { }; } + const isLimitRow = queue.role === "LIMIT"; const results = await Promise.all([ - engine.lengthOfQueues(environment, [queue.name]), - engine.currentConcurrencyOfQueues(environment, [queue.name]), + isLimitRow + ? engine.gateQueuedCountOfQueues(environment, [queue.name]) + : engine.lengthOfQueues(environment, [queue.name]), + isLimitRow + ? engine.totalConcurrencyOfQueues(environment, [queue.name]) + : engine.currentConcurrencyOfQueues(environment, [queue.name]), queue.totalConcurrencyLimit != null ? engine.totalConcurrencyOfQueues(environment, [queue.name]) : undefined, @@ -126,6 +144,8 @@ export class QueueRetrievePresenter extends BasePresenter { queue.concurrencyLimitOverridePercent !== null ? Number(queue.concurrencyLimitOverridePercent) : null, + kind: isLimitRow ? ("limit" as const) : ("queue" as const), + concurrencyVersion: queue.concurrencyVersion, }, }; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 42da830e5cd..d99f60e8817 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -781,6 +781,8 @@ function QueuesWithMetricsView() { const queueDetailPath = v3QueuePath(organization, project, env, { friendlyId: queue.id, }); + const isLimit = queue.kind === "limit"; + const displayName = isLimit ? queue.name.replace(/^limit\//, "") : queue.name; return ( + ) : queue.type === "task" ? ( } @@ -829,8 +845,9 @@ function QueuesWithMetricsView() { > - {queue.name} + {displayName} + {isLimit ? Limit : null} {queue.paused ? ( Paused @@ -887,7 +904,8 @@ function QueuesWithMetricsView() { // link (trailing) rather than nested inside the ; the number stays the // link. trailingContent={ - queue.concurrency?.combined?.current != null ? ( + queue.concurrency?.combined?.current != null && + !(queue.concurrencyVersion === "V2" && queue.concurrencyLimit == null) ? ( - {queue.concurrencyLimitOverridePercent !== null ? ( + {queue.concurrencyVersion === "V2" && + queue.concurrencyLimit == null && + queue.concurrency?.combined?.current != null ? ( + <> + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + + total + + + ) : queue.concurrencyVersion === "V2" && queue.concurrencyLimit != null ? ( + <> + {limit} + + per key + + + ) : queue.concurrencyLimitOverridePercent !== null ? ( <> {limit} @@ -1009,67 +1046,73 @@ function QueuesWithMetricsView() { } /> - } - hiddenButtons={!queue.paused && } - popoverContent={ - <> - {queue.paused ? ( - + {""} + + ) : ( + } + hiddenButtons={!queue.paused && } + popoverContent={ + <> + {queue.paused ? ( + + ) : ( + + )} + + + - ) : ( - + - )} - - - - - - - } - /> + + } + /> + )} ); }) @@ -1907,6 +1950,7 @@ function ClassicQueuesView() { const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; + const isLimit = queue.kind === "limit"; const queueFilterableName = `${queue.type === "task" ? "task/" : ""}${ queue.name }`; @@ -1915,6 +1959,7 @@ function ClassicQueuesView() { + {isLimit ? Limit : null} {queue.concurrency?.overriddenAt ? ( - {limit} - {queue.concurrency?.combined?.current != null ? ( + {queue.concurrencyVersion === "V2" && + queue.concurrencyLimit == null && + queue.concurrency?.combined?.current != null ? ( + <> + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + total + + ) : queue.concurrencyVersion === "V2" && + queue.concurrencyLimit != null ? ( + <> + {limit} + per key + + ) : ( + limit + )} + {queue.concurrency?.combined?.current != null && + !(queue.concurrencyVersion === "V2" && queue.concurrencyLimit == null) ? ( - } - hiddenButtons={!queue.paused && } - popoverContent={ - <> - {queue.paused ? ( - + {""} + + ) : ( + + } + hiddenButtons={ + !queue.paused && + } + popoverContent={ + <> + {queue.paused ? ( + + ) : ( + + )} + + - ) : ( - + + - )} - - - - - - - } - /> + + } + /> + )} ); }) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 375ba2ca0f2..28f5a5c9bc3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -119,7 +119,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { if (!environment) throw new Response(undefined, { status: 404, statusText: "Environment not found" }); - const retrieve = await new QueueRetrievePresenter().call({ environment, queueInput: queueParam }); + const retrieve = await new QueueRetrievePresenter().call({ + environment, + queueInput: queueParam, + includeLimits: true, + }); if (!retrieve.success) { throw new Response(undefined, { status: 404, statusText: "Queue not found" }); } From 98c8122024182942e4059353a75119e10b177e71 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 09:41:26 +0100 Subject: [PATCH 59/77] feat(webapp): the Queues page becomes Concurrency The page moves from /queues to /concurrency with the same layout, charts and columns; the old URL (list and detail, query intact) redirects permanently, and old deeplinks and favorites resolve to the new page. Path helpers, side menu, deeplink and favorites registries and agent page labels follow the rename. --- apps/webapp/app/components/billing/OrgBanner.tsx | 4 ++-- .../app/components/dashboard-agent/page-label.ts | 1 + .../webapp/app/components/navigation/favoritePages.tsx | 3 ++- .../app/components/navigation/sideMenuSections.tsx | 6 +++--- .../route.tsx | 8 ++++---- .../route.tsx | 0 ...ug.projects.$projectParam.env.$envParam.queues.$.ts | 10 ++++++++++ ...Slug.projects.$projectParam.env.$envParam.queues.ts | 8 ++++++++ .../route.tsx | 4 ++-- .../route.tsx | 8 ++++---- .../app/routes/_app.orgs.$organizationSlug/route.tsx | 2 +- ...anizationSlug.projects.$projectParam.concurrency.ts | 4 ++-- .../route.tsx | 4 ++-- apps/webapp/app/services/resolveTriggerUri.server.ts | 4 ++-- apps/webapp/app/utils/deeplinkPages.test.ts | 8 ++++---- apps/webapp/app/utils/deeplinkPages.ts | 3 ++- apps/webapp/app/utils/pageSwitching.test.ts | 6 +++--- apps/webapp/app/utils/pathBuilder.ts | 8 ++++---- 18 files changed, 56 insertions(+), 35 deletions(-) rename apps/webapp/app/routes/{_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues => _app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency}/route.tsx (99%) rename apps/webapp/app/routes/{_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam => _app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam}/route.tsx (100%) create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts diff --git a/apps/webapp/app/components/billing/OrgBanner.tsx b/apps/webapp/app/components/billing/OrgBanner.tsx index acf10f2469d..9fd0b7eaa62 100644 --- a/apps/webapp/app/components/billing/OrgBanner.tsx +++ b/apps/webapp/app/components/billing/OrgBanner.tsx @@ -14,7 +14,7 @@ import { import { useOptionalProject, useProject } from "~/hooks/useProject"; import { useShowSelfServe } from "~/hooks/useShowSelfServe"; import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; -import { v3BillingLimitsPath, v3BillingPath, v3QueuesPath } from "~/utils/pathBuilder"; +import { v3BillingLimitsPath, v3BillingPath, concurrencyPath } from "~/utils/pathBuilder"; import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; function getUpgradeResetDate(): Date { @@ -209,7 +209,7 @@ function PausedEnvironmentBanner({ hideButton }: { hideButton: boolean }) { hideButton ? undefined : ( Manage diff --git a/apps/webapp/app/components/dashboard-agent/page-label.ts b/apps/webapp/app/components/dashboard-agent/page-label.ts index 153cb7a6109..70126e0f4c4 100644 --- a/apps/webapp/app/components/dashboard-agent/page-label.ts +++ b/apps/webapp/app/components/dashboard-agent/page-label.ts @@ -49,6 +49,7 @@ const SECTION_LABELS: Record = { batches: "Batches", "bulk-actions": "Bulk actions", branches: "Branches", + concurrency: "Concurrency", "concurrency-limits": "Concurrency limits", dashboards: "Dashboards", deployments: "Deployments", diff --git a/apps/webapp/app/components/navigation/favoritePages.tsx b/apps/webapp/app/components/navigation/favoritePages.tsx index 64227ad23ae..0b9742dcca2 100644 --- a/apps/webapp/app/components/navigation/favoritePages.tsx +++ b/apps/webapp/app/components/navigation/favoritePages.tsx @@ -223,7 +223,8 @@ const ENV_PAGE_META: Record = { logs: { icon: "logs", name: "Logs" }, errors: { icon: "errors", name: "Errors", singular: "Error" }, query: { icon: "query", name: "Query" }, - queues: { icon: "queues", name: "Queues", singular: "Queue" }, + queues: { icon: "queues", name: "Concurrency" }, + concurrency: { icon: "queues", name: "Concurrency" }, dashboards: { icon: "dashboards", name: "Dashboards", singular: "Dashboard" }, deployments: { icon: "deployments", name: "Deploys", singular: "Deploy" }, "environment-variables": { icon: "environment-variables", name: "Environment variables" }, diff --git a/apps/webapp/app/components/navigation/sideMenuSections.tsx b/apps/webapp/app/components/navigation/sideMenuSections.tsx index 3a358f2c878..6a9c0f7145a 100644 --- a/apps/webapp/app/components/navigation/sideMenuSections.tsx +++ b/apps/webapp/app/components/navigation/sideMenuSections.tsx @@ -39,7 +39,7 @@ import { v3ProjectAlertsPath, v3ProjectSettingsIntegrationsPath, v3PromptsPath, - v3QueuesPath, + concurrencyPath, v3WaitpointTokensPath, } from "~/utils/pathBuilder"; import { AlphaBadge, NewBadge } from "../FeatureBadges"; @@ -160,10 +160,10 @@ export function buildSideMenuSections({ } satisfies SideMenuItemConfig, { id: "queues", - name: "Queues", + name: "Concurrency", icon: QueuesIcon, activeIconColor: "text-queues", - to: v3QueuesPath(organization, project, environment), + to: concurrencyPath(organization, project, environment), dataAction: "queues", } satisfies SideMenuItemConfig, { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx similarity index 99% rename from apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx rename to apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx index d99f60e8817..6a34c8f9067 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx @@ -87,7 +87,7 @@ import { docsPath, EnvironmentParamSchema, v3BillingPath, - v3QueuePath, + concurrencyQueuePath, v3RunsPath, } from "~/utils/pathBuilder"; import type { Handle } from "~/utils/handle"; @@ -485,7 +485,7 @@ function QueuesWithMetricsView() { return ( - + @@ -778,7 +778,7 @@ function QueuesWithMetricsView() { queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath(organization, project, env, { + const queueDetailPath = concurrencyQueuePath(organization, project, env, { friendlyId: queue.id, }); const isLimit = queue.kind === "limit"; @@ -1748,7 +1748,7 @@ function ClassicQueuesView() { return ( - + diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam/route.tsx similarity index 100% rename from apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx rename to apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam/route.tsx diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts new file mode 100644 index 00000000000..cc950eb6786 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts @@ -0,0 +1,10 @@ +import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; + +/** The queues page became the Concurrency page; old URLs (bookmarks, agent deep + * links, the queue detail path) redirect with their sub-path and query intact. */ +export const loader = async ({ params, request }: LoaderFunctionArgs) => { + const url = new URL(request.url); + const subPath = params["*"] ? `/${params["*"]}` : ""; + const base = url.pathname.replace(/\/queues(\/.*)?$/, "/concurrency"); + return redirect(`${base}${subPath}${url.search}`, 301); +}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts new file mode 100644 index 00000000000..69646e8c781 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts @@ -0,0 +1,8 @@ +import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; + +/** The queues page became the Concurrency page; the bare old URL redirects with + * its query intact. */ +export const loader = async ({ request }: LoaderFunctionArgs) => { + const url = new URL(request.url); + return redirect(`${url.pathname.replace(/\/queues$/, "/concurrency")}${url.search}`, 301); +}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index 75822975307..dc28a51527f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -97,7 +97,7 @@ import { v3EditSchedulePath, v3EnvironmentPath, v3NewSchedulePath, - v3QueuePath, + concurrencyQueuePath, v3RunsPath, v3SchedulePath, v3SchedulesAddOnPath, @@ -258,7 +258,7 @@ export default function Page() { taskIdentifier: task.slug, }); const queuePath = task.queue - ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) : undefined; const filters: TaskRunListSearchFilters = useMemo(() => ({ tasks: [task.slug] }), [task.slug]); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index ab8e5be48d6..bfa57df43c3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -63,8 +63,8 @@ import { requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, v3EnvironmentPath, - v3QueuePath, - v3QueuesPath, + concurrencyQueuePath, + concurrencyPath, v3TestTaskPath, } from "~/utils/pathBuilder"; import { parseFiniteInt } from "~/utils/searchParams"; @@ -198,9 +198,9 @@ export default function Page() { const testPath = v3TestTaskPath(organization, project, environment, { taskIdentifier: task.slug, }); - const queuesPath = v3QueuesPath(organization, project, environment); + const queuesPath = concurrencyPath(organization, project, environment); const queuePath = task.queue - ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) : undefined; const { value } = useSearchParams(); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx index 26133675e0d..931e455478f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx @@ -16,7 +16,7 @@ import { canManageBillingLimits } from "~/services/routeBuilders/permissions.ser import { requireUser } from "~/services/session.server"; import { telemetry } from "~/services/telemetry.server"; import { organizationPath } from "~/utils/pathBuilder"; -import { isEnvironmentPauseResumeFormSubmission } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route"; +import { isEnvironmentPauseResumeFormSubmission } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route"; import { isBillingLimitSettingsFormSubmission } from "../_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation"; const ParamsSchema = z.object({ diff --git a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts index 4bde0ccaef3..9459382e62b 100644 --- a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts +++ b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts @@ -2,7 +2,7 @@ import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { SelectBestEnvironmentPresenter } from "~/presenters/SelectBestEnvironmentPresenter.server"; import { requireUser } from "~/services/session.server"; -import { ProjectParamSchema, v3QueuesPath } from "~/utils/pathBuilder"; +import { ProjectParamSchema, concurrencyPath } from "~/utils/pathBuilder"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); @@ -41,5 +41,5 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const selector = new SelectBestEnvironmentPresenter(); const environment = await selector.selectBestEnvironment(project.id, user, project.environments); - return redirect(v3QueuesPath({ slug: organizationSlug }, project, environment)); + return redirect(concurrencyPath({ slug: organizationSlug }, project, environment)); }; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index d81a5a4b664..0558d7445a4 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -116,7 +116,7 @@ import { docsPath, v3BatchPath, v3DeploymentVersionPath, - v3QueuePath, + concurrencyQueuePath, v3RunDownloadLogsPath, v3RunIdempotencyKeyResetPath, v3RunPath, @@ -424,7 +424,7 @@ function RunBody({ const resetFetcher = useTypedFetcher(); const queuePath = queueMetrics?.queueFriendlyId - ? v3QueuePath(organization, project, environment, { + ? concurrencyQueuePath(organization, project, environment, { friendlyId: queueMetrics.queueFriendlyId, }) : undefined; diff --git a/apps/webapp/app/services/resolveTriggerUri.server.ts b/apps/webapp/app/services/resolveTriggerUri.server.ts index 1fe94bb6d35..98da896b5bf 100644 --- a/apps/webapp/app/services/resolveTriggerUri.server.ts +++ b/apps/webapp/app/services/resolveTriggerUri.server.ts @@ -10,7 +10,7 @@ import { import { v3DeploymentVersionPath, v3ErrorPath, - v3QueuesPath, + concurrencyPath, v3RunPath, v3RunSpanPath, v3RunsPath, @@ -127,7 +127,7 @@ function resolveInScope( // resolves to the queues list filtered to the name. return { label: parsed.name, - url: `${v3QueuesPath(organization, project, environment)}?query=${encodeURIComponent( + url: `${concurrencyPath(organization, project, environment)}?query=${encodeURIComponent( parsed.name )}`, }; diff --git a/apps/webapp/app/utils/deeplinkPages.test.ts b/apps/webapp/app/utils/deeplinkPages.test.ts index 597135cb311..04998b572fd 100644 --- a/apps/webapp/app/utils/deeplinkPages.test.ts +++ b/apps/webapp/app/utils/deeplinkPages.test.ts @@ -20,7 +20,7 @@ const ROUTES_DIR = join(APP_DIR, "routes"); const ENV_ROUTE_PREFIX = "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam."; // Route files that name no deeplink: the environment root, and Remix's layout-opt-out spelling. -const NOT_DEEPLINK_NAMES = new Set(["_index", "queues_"]); +const NOT_DEEPLINK_NAMES = new Set(["_index", "concurrency_", "queues"]); const PROBE = "probe_01ABC"; @@ -54,7 +54,7 @@ function isRouteModule(entry: string): boolean { return existsSync(join(path, "route.tsx")) || existsSync(join(path, "route.ts")); } -// A trailing `_` only opts out of the parent layout: `queues_.$queueParam` serves `/queues/{id}`. +// A trailing `_` only opts out of the parent layout: `concurrency_.$queueParam` serves `/concurrency/{id}`. const envRoutes: string[][] = routeEntries .filter((entry) => entry.startsWith(ENV_ROUTE_PREFIX) && isRouteModule(entry)) .map((entry) => @@ -281,9 +281,9 @@ describe("the route Remix compiles from the filename", () => { expect(compiledUrl("routes/login.magic")).toBe("/login/magic"); expect( compiledUrl( - "routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam" + "routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam" ) - ).toBe("/orgs/:organizationSlug/projects/:projectParam/env/:envParam/queues/:queueParam"); + ).toBe("/orgs/:organizationSlug/projects/:projectParam/env/:envParam/concurrency/:queueParam"); }); }); diff --git a/apps/webapp/app/utils/deeplinkPages.ts b/apps/webapp/app/utils/deeplinkPages.ts index 957d2096868..bea7bbc919c 100644 --- a/apps/webapp/app/utils/deeplinkPages.ts +++ b/apps/webapp/app/utils/deeplinkPages.ts @@ -14,6 +14,7 @@ export const ENV_PAGE_TARGETS: ReadonlyMap = new Map([ ["batches", page("batches")], ["branches", page("branches")], ["bulk-actions", page("bulk-actions")], + ["concurrency", page("concurrency")], ["concurrency-limits", page("concurrency-limits")], ["dashboards", page("dashboards")], ["deployments", page("deployments")], @@ -26,7 +27,7 @@ export const ENV_PAGE_TARGETS: ReadonlyMap = new Map([ ["playground", page("playground")], ["prompts", page("prompts")], ["query", page("query")], - ["queues", page("queues")], + ["queues", page("concurrency")], ["regions", page("regions")], ["runs", page("runs")], ["schedules", page("schedules")], diff --git a/apps/webapp/app/utils/pageSwitching.test.ts b/apps/webapp/app/utils/pageSwitching.test.ts index 5b8f8e69881..bdefb6de9ba 100644 --- a/apps/webapp/app/utils/pageSwitching.test.ts +++ b/apps/webapp/app/utils/pageSwitching.test.ts @@ -393,7 +393,7 @@ describe("pages named after a resource", () => { it("truncate to the list they were reached from", () => { expect(projectPortablePage("runs/run_123")).toBe("runs"); expect(projectPortablePage("batches/batch_123")).toBe("batches"); - expect(projectPortablePage("queues/my-queue")).toBe("queues"); + expect(projectPortablePage("concurrency/my-queue")).toBe("concurrency"); expect(projectPortablePage("schedules/sched_123")).toBe("schedules"); expect(projectPortablePage("schedules/edit/sched_123")).toBe("schedules"); expect(projectPortablePage("deployments/deploy_123")).toBe("deployments"); @@ -670,11 +670,11 @@ describe("pathForEnvironmentSwitch", () => { expect( pathForEnvironmentSwitch({ - location: locationOn("queues/my-queue", "?page=2"), + location: locationOn("concurrency/my-queue", "?page=2"), environmentPathname: environmentLocation.pathname, environmentSlug: "prod", }) - ).toBe("/orgs/acme/projects/api/env/prod/queues"); + ).toBe("/orgs/acme/projects/api/env/prod/concurrency"); }); it("only swaps the environment slug when it cannot tell where the environment path ends", () => { diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index 89e65d88510..a700b225997 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -587,21 +587,21 @@ export function v3SchedulesAddOnPath(organization: OrgForPath) { return `/resources/orgs/${organizationParam(organization)}/schedules-addon`; } -export function v3QueuesPath( +export function concurrencyPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath ) { - return `${v3EnvironmentPath(organization, project, environment)}/queues`; + return `${v3EnvironmentPath(organization, project, environment)}/concurrency`; } -export function v3QueuePath( +export function concurrencyQueuePath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, queue: { friendlyId: string } ) { - return `${v3QueuesPath(organization, project, environment)}/${queue.friendlyId}`; + return `${concurrencyPath(organization, project, environment)}/${queue.friendlyId}`; } export function v3WaitpointTokensPath( From cdff91ecef8db4a0dcaa096cb77ada1f259e3edd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 09:43:58 +0100 Subject: [PATCH 60/77] fix(webapp): limit rows interleave only on the dashboard list The public queues API and the runs filter resources share the list presenter, so the limit interleaving is opt-in and only the dashboard Concurrency page passes it; every other caller stays queue-only. --- .../v3/QueueListPresenter.server.ts | 49 +++++++++++++------ .../route.tsx | 1 + 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index c6c8a68a104..7c0feae2d5e 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -79,15 +79,18 @@ function formatClickhouseDateTime(date: Date): string { function buildQueueListWhere( environmentId: string, query: string | undefined, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ): Prisma.TaskQueueWhereInput { const trimmedQuery = query?.trim(); return { runtimeEnvironmentId: environmentId, - /** The type filter names queue shapes, so applying it scopes the list to queue rows; - * without it the list interleaves named limits alongside queues. */ - role: type ? ("QUEUE" as const) : { in: ["QUEUE" as const, "LIMIT" as const] }, + /** Only the dashboard interleaves named limits, and the type filter names queue + * shapes, so either condition scopes the list to queue rows; the public queues + * API always stays queue-only. */ + role: + includeLimits && !type ? { in: ["QUEUE" as const, "LIMIT" as const] } : ("QUEUE" as const), version: "V2", name: trimmedQuery ? { @@ -120,6 +123,7 @@ export class QueueListPresenter extends BasePresenter { page, type, sort = "name", + includeLimits = false, }: { environment: AuthenticatedEnvironment; query?: string; @@ -127,13 +131,21 @@ export class QueueListPresenter extends BasePresenter { perPage?: number; type?: "task" | "custom"; sort?: QueueListSort; + includeLimits?: boolean; }): Promise { const hasFilters = Boolean(query?.trim()) || type !== undefined; if (sort !== "name") { // Ranking is additive: any failure or unsupported input falls back to name order. try { - const ranked = await this.getRankedQueues(environment, query, page, type, sort); + const ranked = await this.getRankedQueues( + environment, + query, + page, + type, + sort, + includeLimits + ); if (ranked) { return ranked; } @@ -143,7 +155,13 @@ export class QueueListPresenter extends BasePresenter { } if (hasFilters) { - const { queues, hasMore } = await this.getFilteredQueues(environment, query, page, type); + const { queues, hasMore } = await this.getFilteredQueues( + environment, + query, + page, + type, + includeLimits + ); return { queues, @@ -157,11 +175,11 @@ export class QueueListPresenter extends BasePresenter { } const totalQueues = await this._replica.taskQueue.count({ - where: buildQueueListWhere(environment.id, query, type), + where: buildQueueListWhere(environment.id, query, type, includeLimits), }); return { - queues: await this.getUnfilteredQueues(environment, page, type), + queues: await this.getUnfilteredQueues(environment, page, type, includeLimits), pagination: { mode: "unfiltered" as const, currentPage: page, @@ -182,7 +200,8 @@ export class QueueListPresenter extends BasePresenter { query: string | undefined, page: number, type: "task" | "custom" | undefined, - sort: Exclude + sort: Exclude, + includeLimits: boolean ) { if (type !== undefined) { return null; @@ -231,7 +250,7 @@ export class QueueListPresenter extends BasePresenter { return null; } - const where = buildQueueListWhere(environment.id, query, type); + const where = buildQueueListWhere(environment.id, query, type, includeLimits); const totalQueues = await this._replica.taskQueue.count({ where }); let rankedPageQueues: QueueListRow[] = []; @@ -302,10 +321,11 @@ export class QueueListPresenter extends BasePresenter { environment: AuthenticatedEnvironment, query: string | undefined, page: number, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ) { const queues = await this._replica.taskQueue.findMany({ - where: buildQueueListWhere(environment.id, query, type), + where: buildQueueListWhere(environment.id, query, type, includeLimits), select: queueListSelect, orderBy: { orderableName: "asc", @@ -325,10 +345,11 @@ export class QueueListPresenter extends BasePresenter { private async getUnfilteredQueues( environment: AuthenticatedEnvironment, page: number, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ) { const queues = await this._replica.taskQueue.findMany({ - where: buildQueueListWhere(environment.id, undefined, type), + where: buildQueueListWhere(environment.id, undefined, type, includeLimits), select: queueListSelect, orderBy: { orderableName: "asc", diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx index 6a34c8f9067..d2fdca00afe 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx @@ -191,6 +191,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { environment, query, page, + includeLimits: true, // Relevance ordering rides the metrics pipeline, so it is part of the gated UI. sort: queueMetricsUiEnabled ? (sort ?? "busiest") : "name", }); From c9ba18445bf9a799dbbbc326319a894d640287cc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 10:02:42 +0100 Subject: [PATCH 61/77] fix(webapp): format the renamed task routes, trigger URIs resolve to the concurrency page --- .../route.tsx | 4 +++- .../route.tsx | 4 +++- apps/webapp/test/resolveTriggerUri.test.ts | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index dc28a51527f..ad292e51526 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -258,7 +258,9 @@ export default function Page() { taskIdentifier: task.slug, }); const queuePath = task.queue - ? concurrencyQueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { + friendlyId: task.queue.friendlyId, + }) : undefined; const filters: TaskRunListSearchFilters = useMemo(() => ({ tasks: [task.slug] }), [task.slug]); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index bfa57df43c3..6615a2fc8d6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -200,7 +200,9 @@ export default function Page() { }); const queuesPath = concurrencyPath(organization, project, environment); const queuePath = task.queue - ? concurrencyQueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { + friendlyId: task.queue.friendlyId, + }) : undefined; const { value } = useSearchParams(); diff --git a/apps/webapp/test/resolveTriggerUri.test.ts b/apps/webapp/test/resolveTriggerUri.test.ts index d394beea591..dbe9baaa1d6 100644 --- a/apps/webapp/test/resolveTriggerUri.test.ts +++ b/apps/webapp/test/resolveTriggerUri.test.ts @@ -4,7 +4,7 @@ import { resolveTriggerUri, type TriggerUriScope } from "~/services/resolveTrigg import { v3DeploymentVersionPath, v3ErrorPath, - v3QueuesPath, + concurrencyPath, v3RunPath, v3RunSpanPath, } from "~/utils/pathBuilder"; @@ -58,7 +58,7 @@ describe("resolveTriggerUri", () => { const uri = formatTriggerUri({ kind: "queue", ...uriScope, name: "task/send email" }); expect(resolveTriggerUri(scope, uri)).toEqual({ label: "task/send email", - url: `${v3QueuesPath(org, project, env)}?query=task%2Fsend%20email`, + url: `${concurrencyPath(org, project, env)}?query=task%2Fsend%20email`, }); }); From 0b0684342db32a042e6f141abf53f0780fb839bb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 10:41:06 +0100 Subject: [PATCH 62/77] fix(webapp): limit rows on the Concurrency page stay list-level, overrides converge the engine Limit rows no longer link to the queue detail page: that page is queue observability (queue-scoped metrics, run filters, pause/override actions) and a limit's activity lives on each holder's home queue, so the detail route and retrieve presenter go back to queue rows only. The page's post actions and the detail back-link now target /concurrency directly instead of bouncing through the /queues 301. concurrencyLimits.override now converges the run-engine with Postgres: a failed engine sync compensates from a fresh row read (original error still surfaces), and a successful sync re-checks freshness so a slower older override's engine write can never leave the engine behind a newer one. --- .../v3/QueueRetrievePresenter.server.ts | 28 ++-------- .../route.tsx | 17 ++++--- .../route.tsx | 7 ++- .../concurrencyLimitsSystem.server.ts | 35 +++++++++---- .../test/concurrencyLimitsSystem.test.ts | 51 +++++++++++++++++++ 5 files changed, 96 insertions(+), 42 deletions(-) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index cddaff28ae4..1e2bea93353 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -22,18 +22,9 @@ export type FoundQueue = Prettify< export async function getQueue( prismaClient: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, - queue: RetrieveQueueParam, - options?: { - /** - * The dashboard's detail page shows limit rows too; the public API and pause - * flows stay scoped to queue rows. - */ - includeLimits?: boolean; - } + queue: RetrieveQueueParam ) { - const role = options?.includeLimits - ? { in: ["QUEUE" as const, "LIMIT" as const] } - : ("QUEUE" as const); + const role = "QUEUE" as const; if (typeof queue === "string") { return joinQueueWithUser( @@ -88,13 +79,11 @@ export class QueueRetrievePresenter extends BasePresenter { public async call({ environment, queueInput, - includeLimits, }: { environment: AuthenticatedEnvironment; queueInput: RetrieveQueueParam; - includeLimits?: boolean; }) { - const queue = await getQueue(this._replica, environment, queueInput, { includeLimits }); + const queue = await getQueue(this._replica, environment, queueInput); if (!queue) { return { success: false as const, @@ -102,14 +91,9 @@ export class QueueRetrievePresenter extends BasePresenter { }; } - const isLimitRow = queue.role === "LIMIT"; const results = await Promise.all([ - isLimitRow - ? engine.gateQueuedCountOfQueues(environment, [queue.name]) - : engine.lengthOfQueues(environment, [queue.name]), - isLimitRow - ? engine.totalConcurrencyOfQueues(environment, [queue.name]) - : engine.currentConcurrencyOfQueues(environment, [queue.name]), + engine.lengthOfQueues(environment, [queue.name]), + engine.currentConcurrencyOfQueues(environment, [queue.name]), queue.totalConcurrencyLimit != null ? engine.totalConcurrencyOfQueues(environment, [queue.name]) : undefined, @@ -144,8 +128,6 @@ export class QueueRetrievePresenter extends BasePresenter { queue.concurrencyLimitOverridePercent !== null ? Number(queue.concurrencyLimitOverridePercent) : null, - kind: isLimitRow ? ("limit" as const) : ("queue" as const), - concurrencyVersion: queue.concurrencyVersion, }, }; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx index d2fdca00afe..1e00045d64c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx @@ -290,7 +290,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const userId = await requireUserId(request); if (request.method.toLowerCase() !== "post") { return redirectWithErrorMessage( - `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, + `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/concurrency`, request, "Wrong method" ); @@ -318,7 +318,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const action = formData.get("action"); const url = new URL(request.url); - const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; + const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/concurrency${url.search}`; if (environment.archivedAt) { return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); @@ -779,16 +779,21 @@ function QueuesWithMetricsView() { queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = concurrencyQueuePath(organization, project, env, { - friendlyId: queue.id, - }); const isLimit = queue.kind === "limit"; + /** The detail page is queue observability (queue metrics, run filters, pause + * and override actions); a limit's activity lives on each holder's home queue, + * so limit rows don't link anywhere. */ + const queueDetailPath = isLimit + ? undefined + : concurrencyQueuePath(organization, project, env, { + friendlyId: queue.id, + }); const displayName = isLimit ? queue.name.replace(/^limit\//, "") : queue.name; return ( s, so // they render beside the link (leading/trailing), never inside it — // otherwise the cell is invalid