From 510adff48f34ff4a008695601dc3af5534f96c6e Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 18:45:16 +0300 Subject: [PATCH 1/6] Fix steering reauthorization and LinkedIn message inspection --- docs/architecture.md | 4 + src/chrome/src/agent/adapters.js | 2 + src/chrome/src/agent/agent.js | 306 ++++++++++++- .../src/agent/conversation-persistence.js | 18 +- src/chrome/src/agent/planner.js | 5 +- src/chrome/src/content/content.js | 52 ++- src/firefox/src/agent/adapters.js | 2 + src/firefox/src/agent/agent.js | 306 ++++++++++++- .../src/agent/conversation-persistence.js | 18 +- src/firefox/src/agent/planner.js | 5 +- src/firefox/src/content/content.js | 52 ++- test/chat-steering-ui.mjs | 2 +- test/chat-steering.mjs | 410 +++++++++++++++++- .../fixtures/message-recipient-navigation.mjs | 195 +++++++++ 14 files changed, 1329 insertions(+), 48 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index ab1ab1b5fc..b93dfb1907 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,6 +75,10 @@ The user types a message, the panel sends a detached `{action: 'chat_start', tex While a run is active, Send / Enter queues follow-up messages per tab. **Steer** / **Alt+Enter** sends a text correction to that same run; queued messages also have a Steer action. The background accepts corrections only from the extension chat panel and only for the matching run request ID. Both agent loops consume corrections as user messages before the next model request. An already running tool finishes; the remaining tool calls in its batch receive skipped results before the correction enters the conversation. Steering preserves the current mode and permission checks. If the run ends before consuming a correction, a journal event puts it at the front of that tab's queue for the next turn. A transport error retains the draft without automatically resending it. +In Act and Dev, steering links the initiating request and ordered corrections into one task authority with internal run, message, and revision metadata. At the next safe boundary, both execution paths revalidate that authority through the configured intent or planning gate and refresh execution, recipient, workflow, progress, and tool policy. A newer correction supersedes an in-flight result and invalidates any pending plan-review card. Planner failure stops further dispatch with its concrete error. Ask stays read-only, and ordinary later chat messages remain new tasks. Task-authority messages survive compaction and session recovery; prior submission receipts survive separately from reset completion evidence. After a submission or publication attempt, revised work can inspect the existing effect but cannot submit it again in that run. + +LinkedIn message inspection recognizes image-preview controls in history and native conversation-list rows in the separate rail. A preview requires its image descendant, visible enabled control, matching hit target, and active modal scope outside the composer or form; its label alone is insufficient. Nested row actions and actual Send controls retain recipient verification. The adapter directs the agent to locate the named sender, scroll the history pane to earlier messages, and inspect each attachment in sender context before asking for any missing event details. + Each new user/assistant pair starts in a reading-first scroll state: the question stays visible while a long response grows instead of being pushed immediately to the live edge. A floating control changes between **Follow response**, diff --git a/src/chrome/src/agent/adapters.js b/src/chrome/src/agent/adapters.js index 10b213304e..35fcee7f56 100644 --- a/src/chrome/src/agent/adapters.js +++ b/src/chrome/src/agent/adapters.js @@ -16323,6 +16323,8 @@ const ADAPTERS = [ - The post composer ("Start a post") opens a dialog rendered inside an open shadow root (#interop-outlet). If it is missing from get_accessibility_tree / click({text}) results even though the screenshot shows it open, do NOT coordinate-click inside it — clicks often hit the dialog backdrop and CLOSE it (the next screenshot shows the regular feed again). Reliable path: click({text: "Start a post"}) to open it, then type_text({selector: '[contenteditable]:not([contenteditable="false"])', text: "...", clear: true}) — this focuses the editor directly without a click. Then click the Post button (find it via get_interactive_elements or shadow_dom_query({selector: "button", shadowPath: "div#interop-outlet"}) if a plain text click matches "Post impressions" instead). - "Connect" button on profiles often has a "Send without a note" prompt — read it before clicking. - Messages are at /messaging — the message composer is a contenteditable with image/file upload icons. +- For a task referring to a named sender's message or event, inspect that sender's messages in the current conversation first. Scroll the message-history pane upward to earlier messages; do not scroll the thread heading or switch to unrelated conversations. Verify the sender beside each attachment before treating it as the event source. +- Open a message image with its image-preview control and inspect the preview. If the event's date, time, time zone, or other required details remain missing, call clarify instead of guessing or claiming completion. - In Messaging, after filling the composer, the reliable send path is usually Enter. If the composer footer says "Press Enter to Send" or the send-options popover shows "Press Enter to Send", call press_keys({key:"Enter"}) from the composer. Do NOT keep scrolling to find a Send button that is already visible/implicit. - The three-dot / send-options control near the composer opens send preferences ("Press Enter to Send" vs "Click Send"); it is not the Send action. If you opened that popover by mistake, choose/keep "Press Enter to Send", close it if needed, then press Enter to send the focused composer. - Search has filters (People, Posts, Jobs, Companies) as tabs at the top.`, diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 65b4c9a777..bf3eb083c4 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -6058,6 +6058,8 @@ export class Agent extends LoopDetector { requestId: String(runOptions.detachedRequestId || ''), messages: [], acceptedIds: new Set(), + revision: 0, + authorizedRevision: 0, onUpdate, }); } @@ -6077,6 +6079,9 @@ export class Agent extends LoopDetector { if (!run.acceptedIds.has(id)) { run.acceptedIds.add(id); run.messages.push({ id, text: text.trim() }); + // A reviewed plan belongs to the task before this correction. Wake the + // planner so it can review the new revision without approving stale work. + this._cancelPendingPlans(tabId, 'superseded by steering'); // A cached fast-path action was prepared before this correction. const session = this._jevSessions?.get(tabId); if (session) { session.disabled = true; session.queue = []; } @@ -6094,14 +6099,163 @@ export class Agent extends LoopDetector { // Only consume at model/tool boundaries, after all tool results are present. // Human corrections stay ordinary user messages; no page context is added. for (const item of run.messages.splice(0)) { + const priorBinding = this._activeTaskBinding(messages); + const parentTaskKey = this._progressTaskKeyForText(priorBinding.text); + const guard = this._planExecutionGuards.get(tabId); + const submitted = this._completionSubmitStates.get(tabId); + const priorSubmission = guard?.steeringPriorSubmission || priorBinding.priorSubmission + || (submitted?.dispatched ? submitted : null) + || (Object.keys(guard?.socialPublication?.outcomes || {}).length + ? { dispatched: true, publicationOutcomes: guard.socialPublication.outcomes } : null); this._recordSocialPublicationSteering(tabId, item.text); - messages.push({ role: 'user', content: item.text }); + messages.push({ role: 'user', content: item.text, webbrainSteering: { + requestId: run.requestId, messageId: item.id, revision: ++run.revision, parentTaskKey, + // A durable attempt receipt is separate from revised completion proof. + ...(priorSubmission ? { priorSubmission: structuredClone(priorSubmission) } : {}), + } }); onUpdate('steering_applied', item); + this._traceSteeringRevision(tabId, 'steering_applied', { revision: run.revision, messageId: item.id }); } this._persist(tabId); return true; } + _traceSteeringRevision(tabId, note, extra) { + const runId = this.currentRunId.get(tabId); + if (runId) { + try { Promise.resolve(trace.recordNote(runId, 0, note, extra)).catch(() => {}); } catch {} + } + } + + _steeringGateSuperseded(tabId, runOptions) { + return runOptions.steeringRevision != null && ( + this._steeringRuns.get(tabId)?.revision !== runOptions.steeringRevision + || this._hasPendingSteering(tabId) + ); + } + + async _revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions = {}) { + const run = this._steeringRuns.get(tabId); + if (!run) return null; + this._applyPendingSteering(tabId, messages, onUpdate); + if (run.revision === run.authorizedRevision) return null; + if (!this._isActionMode(mode) || this._isStandaloneChatRun(runOptions)) { + run.authorizedRevision = run.revision; + return { changed: true }; + } + // A revision never inherits success counters or dispatch authorization. + // Keep actual attempts separately: resetting a guard must not make a sent + // message, an uncertain submit, or a publication eligible to run again. + const previous = this._planExecutionGuards.get(tabId); + const submitted = this._completionSubmitStates.get(tabId); + const priorSubmission = previous?.steeringPriorSubmission + || (submitted?.dispatched ? submitted : null) + || this._activeTaskBinding(messages).priorSubmission || null; + const attemptedPublication = previous?.socialPublication + && Object.keys(previous.socialPublication.outcomes || {}).length + ? previous.socialPublication : null; + while (!this._checkAbort(tabId)) { + this._applyPendingSteering(tabId, messages, onUpdate); + const revision = run.revision; + const enriched = { role: 'user', content: this._activeTaskBinding(messages).text }; + const revisionOptions = { + ...runOptions, steeringRevision: revision, trustedContinuation: false, + recommendedAction: null, + }; + const readState = this.readCompletenessStates.get(tabId); + if (readState) { + const pageUrl = await this._currentUrl(tabId); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + const adapterName = getActiveAdapter(pageUrl)?.name || ''; + // Let this revision's planner establish its own read obligation. + // Keep run ownership for teardown, but discard earlier coverage and + // obligations, including ones armed by a superseded planner result. + this.readCompletenessStates.set(tabId, createReadCompletenessState( + readState.runToken, false, isCommunicationThreadContext(pageUrl, adapterName), adapterName, + )); + } + const gate = await this._maybeRunPlannerGate( + tabId, messages, enriched, onUpdate, mode, costState, runId, null, revisionOptions, + ); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + run.authorizedRevision = revision; + this._traceSteeringRevision(tabId, 'steering_revalidated', { + revision, requestKind: gate.requestKind, proceed: gate.proceed, reason: gate.reason || null, + }); + if (!gate.proceed) return { changed: true, gate, enriched }; + this._completionSubmitStates.delete(tabId); + const guard = this._startPlanExecutionGuard(tabId, mode, gate, revisionOptions); + guard.steeringPriorSubmission = priorSubmission; + if (priorSubmission || attemptedPublication) guard.evidenceTaskKey = guard.taskKey; + guard.socialPublicationSteering = previous?.socialPublicationSteering || null; + guard.socialPublicationClarifications = previous?.socialPublicationClarifications || []; + if (attemptedPublication) { + guard.socialPublication = attemptedPublication; + this._invalidateSocialPublicationContract(guard); + } + this._setResponseLanguagePolicy(tabId, gate.responseLanguagePolicy, runOptions.locale || 'en', { + approvedPlanLanguageOverride: gate.responseLanguageApprovedPlanOverride === true, + }); + if (!gate.responseOnly) await this._ensureProgressSessionForCurrentTask(tabId, { + provider: this._activeProvider(tabId), costState, + progressLedgerPolicy: gate.progressLedgerPolicy, progressAction: gate.progressAction, + expectedItems: gate.expectedItems, + }); + // The system prompt is rebuilt by the new guard. Clear the old approved + // plan even when only intent classification is enabled for this revision. + if (!gate.approvedScratchpadText && previous?.approvedPlanText) { + const scratchIdx = this._findScratchpadIndex(messages); + if (scratchIdx >= 0) this._scratchpadWrite(tabId, { + replace: true, + text: this._extractScratchpadBody(messages[scratchIdx].content) + .replace(previous.approvedPlanText, '').trim(), + }); + } + this._persist(tabId); + return { changed: true, gate, enriched }; + } + return { changed: true, gate: { proceed: false, reason: 'cancelled', message: '[Stopped by user]' } }; + } + + _steeringPriorSubmissionBlock(tabId, name, args, detectedSubmit) { + if (!this._planExecutionGuards.get(tabId)?.steeringPriorSubmission) return null; + const resolvedNonSubmit = detectedSubmit?.resolvedNavigationTarget === true + || detectedSubmit?.resolvedEditableTarget === true + || detectedSubmit?.resolvedNonSubmitTarget === true; + if (!isNetworkMutation(name, args) && name !== 'chrome_web_store_publish' + && !(detectedSubmit?.isSubmit || (!resolvedNonSubmit && this._isFormValidationCandidate(name, args)))) return null; + return { + success: false, blocked: true, noDispatch: true, dispatched: false, + steeringPriorSubmission: true, + error: 'A submission was already attempted before the user revised this task. Inspect its result before proceeding; do not repeat or replace that submission in this run. Report the verified effect and any remaining revised work with outcome partial or failed.', + }; + } + + async _steeringBoundaryOutcome(tabId, messages, onUpdate, mode, provider, costState, runId, runOptions) { + const refresh = await this._revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions); + if (!refresh?.gate) return refresh; + if (!refresh.gate.proceed) return { + changed: true, terminal: true, content: refresh.gate.message || 'More information is required.', + status: refresh.gate.reason || 'cancelled', + }; + if (refresh.gate.responseOnly) { + const capture = {}; + const response = await this._completeResponseOnlyTurn( + tabId, messages, onUpdate, provider, costState, runId, runOptions, refresh.enriched, null, capture, + ); + // A correction received during a context-only answer supersedes that + // answer exactly like an ordinary in-flight model response. + if (this._hasPendingSteering(tabId)) { + onUpdate('text', { content: '', replace: true }); + return { changed: true }; + } + return { changed: true, terminal: true, ...response, capture }; + } + return { changed: true }; + } + _finishSteeringRun(tabId) { const run = this._steeringRuns.get(tabId); this._steeringRuns.delete(tabId); @@ -12672,7 +12826,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d reason: 'submit the configured Chrome Web Store release for review', }; } - const workflowPreSubmitBlock = await this._conditionalGithubCommitTransition(tabId, fnName, fnArgs, detectedSubmitAction) + const workflowPreSubmitBlock = this._steeringPriorSubmissionBlock(tabId, fnName, fnArgs, detectedSubmitAction) + || await this._conditionalGithubCommitTransition(tabId, fnName, fnArgs, detectedSubmitAction) || await this._workflowPreSubmitDispatchBlock( tabId, fnName, fnArgs, detectedSubmitAction, provider, ); @@ -20323,20 +20478,46 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // boundaries unambiguous when this composite is later embedded in the // trusted recovery anchor or hashed as execution evidence authority, and // the flat shape keeps it bounded across an arbitrarily long chain. - const requestText = this._progressTaskTextKey(taskText).slice(0, 1600); + const requestText = carriesPriorBinding && priorBinding.updates + ? taskText : this._progressTaskTextKey(taskText).slice(0, 1600); const answers = [...carriedAnswers, answerText] .map(answer => this._progressTaskTextKey(answer).slice(0, CLARIFICATION_ANSWER_CHARS)) .filter(Boolean) .slice(-CLARIFICATION_ANSWER_LIMIT); + const updates = carriesPriorBinding && priorBinding.updates + ? [...priorBinding.updates, { kind: 'clarification', text: answerText }] + : null; const text = `User task clarified by the latest answer: ${JSON.stringify({ request: requestText, answers, + ...(updates ? { updates } : {}), })}`; const pinnedIndices = [...carriedPinnedIndices, taskIndex, clarificationIndex, answerIndex] .filter(index => index >= 0) .filter((index, position, all) => all.indexOf(index) === position) .sort((a, b) => a - b); - return { index: answerIndex, taskIndex, clarificationIndex, text, requestText, answers, pinnedIndices }; + return { index: answerIndex, taskIndex, clarificationIndex, text, requestText, answers, pinnedIndices, + ...(updates ? { updates } : {}), + ...(carriesPriorBinding && priorBinding.priorSubmission ? { priorSubmission: priorBinding.priorSubmission } : {}) }; + } + + _steeringTaskBinding(messages, index) { + const message = messages[index]; + const metadata = message?.webbrainSteering; + if (!metadata?.requestId || !metadata.messageId || !Number.isInteger(metadata.revision) + || metadata.revision < 1 || !metadata.parentTaskKey) return null; + const prior = this._activeTaskBinding(messages.slice(0, index)); + if (!prior.text || metadata.parentTaskKey !== this._progressTaskKeyForText(prior.text)) return null; + const updates = [...(prior.updates || (prior.answers || []).map(text => ({ kind: 'clarification', text }))), + { kind: 'steering', text: this._plannerUserAuthoredText(message) }]; + const requestText = prior.requestText || prior.text; + return { + index, taskIndex: prior.taskIndex, clarificationIndex: -1, requestText, + answers: prior.answers || [], updates, + priorSubmission: metadata.priorSubmission?.dispatched === true ? metadata.priorSubmission : prior.priorSubmission || null, + text: `User task revised by steering: ${JSON.stringify({ request: requestText, updates })}`, + pinnedIndices: [...prior.pinnedIndices, index], + }; } // Return the latest genuine task-bearing user turn, not a synthetic runtime @@ -20352,6 +20533,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (this._isAgentInjectedUserMessage(message)) continue; const text = this._plannerUserAuthoredText(message); if (!text) continue; + const steered = this._steeringTaskBinding(messages, i); + if (steered) return steered; const clarification = this._plannerClarificationTaskBinding(messages, i); if (clarification) return clarification; const normalized = text.toLowerCase(); @@ -20698,7 +20881,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Wikipedia passages through rawModelMessagesForRun. runOptions._standalonePersistedUserMessage = submittedUserMessage; } - messages.push(submittedUserMessage); + // Steering's actual user messages are already durable. Its composite is + // planner input only; recording it as another user turn would replace the + // original request and break the revision's authority chain. + if (runOptions.steeringRevision == null) messages.push(submittedUserMessage); if (runOptions?.selectionGroundingScopeStarted === true) { this._finalizeSelectionGroundingScope(tabId, messages, submittedUserMessage); } @@ -20718,7 +20904,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } return { proceed: true, requestKind: 'execute', requiresStateChange: false }; } - const fastPathPlan = this._recommendedActionFastPathPlan(runOptions); + const fastPathPlan = runOptions.steeringRevision == null + ? this._recommendedActionFastPathPlan(runOptions) : null; if (fastPathPlan) { this.plannerFollowUpSkipTabs.delete(tabId); const approvedScratchpadText = this._formatRecommendedActionFastPathScratchpad(fastPathPlan); @@ -20749,7 +20936,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const historyDigest = this._buildPlannerHistoryDigest(priorMessages); const followUpContext = this._buildPlannerFollowUpContext(priorMessages); let gate; - if (this._shouldSkipPlannerForShortFollowUp(tabId, priorMessages, enriched, plannerMode)) { + if (runOptions.steeringRevision == null && this._shouldSkipPlannerForShortFollowUp(tabId, priorMessages, enriched, plannerMode)) { this.plannerFollowUpSkipTabs.delete(tabId); gate = await this._runPlannerIntentGate( tabId, enriched, onUpdate, costState, runId, historyDigest, tabInfo, mode, runOptions, followUpContext, @@ -20764,6 +20951,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d tabId, enriched, onUpdate, costState, runId, historyDigest, tabInfo, mode, runOptions, followUpContext, ); } + if (this._steeringGateSuperseded(tabId, runOptions)) return { proceed: false, steeringSuperseded: true }; + if (runOptions.steeringRevision == null && this._hasPendingSteering(tabId)) { + const refresh = await this._revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions); + if (refresh?.gate) return refresh.gate; + } if (runOptions?.trustedContinuation === true && gate?.proceed && (gate.requestKind === 'execute' || gate.plannerFailedContinueAct === true) @@ -20809,7 +21001,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } if (gate.approvedScratchpadText) { - const scratchResult = this._scratchpadWrite(tabId, { text: gate.approvedScratchpadText }); + let scratchText = gate.approvedScratchpadText; + if (runOptions.steeringRevision != null) { + const scratchIdx = this._findScratchpadIndex(messages); + const previousPlan = this._planExecutionGuards.get(tabId)?.approvedPlanText || ''; + if (scratchIdx >= 0) scratchText = + this._extractScratchpadBody(messages[scratchIdx].content).replace(previousPlan, '').trim() + + '\n\n' + scratchText; + } + const scratchResult = this._scratchpadWrite(tabId, { + text: scratchText, replace: runOptions.steeringRevision != null, + }); if (!scratchResult?.success) { onUpdate('warning', { message: scratchResult?.error || 'Could not pin plan to scratchpad.' }); } else { @@ -21072,6 +21274,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } _plannerActContinuation(runOptions, onUpdate, runId = null, reason = 'invalid_output') { + if (runOptions.steeringRevision != null) return { + proceed: false, reason: 'planner_error', requestKind: 'respond', + message: `WebBrain could not revalidate the revised task (${reason}). No further actions were dispatched.`, + }; const message = this._plannerIntentFailureMessage(runOptions); onUpdate('warning', { code: 'planner_failed_continue_act', @@ -21125,14 +21331,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } - _plannerRequestFailure(error, onUpdate, provider = null) { + _plannerRequestFailure(error, onUpdate, provider = null, steering = false) { const detail = sanitizePlannerText( formatErrorMessage(error, { fallback: 'Unknown planner request error.' }), 500, { collapseWhitespace: true }, ); const detailSentence = /[.!?]$/.test(detail) ? detail : `${detail}.`; - const message = `Planner request failed before a valid response was available: ${detailSentence} No tools ran.`; + const message = `Planner request failed before a valid response was available: ${detailSentence} ${steering ? 'No further actions were dispatched after steering.' : 'No tools ran.'}`; const failureKind = plannerRequestFailureKind(detail); onUpdate('warning', { code: 'planner_request_failed', @@ -21154,14 +21360,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } - _plannerPostValidationFailure(error, onUpdate) { + _plannerPostValidationFailure(error, onUpdate, steering = false) { const detail = sanitizePlannerText( formatErrorMessage(error, { fallback: 'Unknown plan processing error.' }), 500, { collapseWhitespace: true }, ); const detailSentence = /[.!?]$/.test(detail) ? detail : `${detail}.`; - const message = `A valid plan was produced, but WebBrain could not safely finish plan review or prepare the plan for execution: ${detailSentence} No tools ran.`; + const message = `A valid plan was produced, but WebBrain could not safely finish plan review or prepare the plan for execution: ${detailSentence} ${steering ? 'No further actions were dispatched after steering.' : 'No tools ran.'}`; onUpdate('warning', { code: 'planner_processing_failed', message, @@ -21584,6 +21790,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const recheckOnly = runOptions?.plannerIntentRecheckOnly === true; const provider = this._activeProvider(tabId); const plannerMessages = buildPlannerIntentMessages(enriched, tabUrl, tabTitle, historyDigest, { + steering: runOptions.steeringRevision != null, noThink: this._plannerPrefersNoThinkPrompt(provider), scheduledResume: runOptions?.scheduledResume === true, locale, @@ -21697,6 +21904,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ? this._plannerIntentRecheckFallback() : this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } await this._tracePlannerCompletionRequirementCorrection(runId, plan, 'intent'); if (plan.request_kind === 'respond') { return { @@ -21785,9 +21995,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } catch {} } - return this._plannerPostValidationFailure(e, onUpdate); + return this._plannerPostValidationFailure(e, onUpdate, runOptions.steeringRevision != null); } await this._tracePlannerAttemptFailure(runId, 'intent', 2, e); + if (runOptions.steeringRevision != null) return this._plannerRequestFailure(e, onUpdate, provider, true); return this._plannerActContinuation(runOptions, onUpdate, runId, 'request_error'); } } @@ -21814,6 +22025,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const tier = this._resolvePromptTier(); const skillCatalog = this._skillCatalog(conversationMode, tier); const plannerMessages = buildPlannerMessages(enriched, tabUrl, tabTitle, historyDigest, { + steering: runOptions.steeringRevision != null, noThink: this._plannerPrefersNoThinkPrompt(provider), scheduledResume: runOptions?.scheduledResume === true, allowApi: this.isApiMutationsAllowed(tabId), @@ -21877,6 +22089,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (this._checkAbort(tabId)) { return { proceed: false, message: '[Stopped by user]' }; } + if (this._steeringGateSuperseded(tabId, runOptions)) return { proceed: false, steeringSuperseded: true }; let consistencyRepairKind = null; let plan = parsePlanFromContent(result.content, plannerParseOptions); // Retry whenever the first attempt yields no parseable plan — empty @@ -21938,16 +22151,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return { proceed: false, message: '[Stopped by user]' }; } if (!plan) { + if (runOptions.steeringRevision != null) return this._plannerActContinuation(runOptions, onUpdate, runId, 'invalid_output'); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? this._strictPlannerFailure(onUpdate) : this._plannerActContinuation(runOptions, onUpdate, runId, 'invalid_output'); } hasValidPlannerResponse = true; if (this._plannerIntentUnresolvedConsistencyIssue(plan, consistencyRepairKind, followUpContext)) { + if (runOptions.steeringRevision != null) return this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? this._strictPlannerFailure(onUpdate) : this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } await this._tracePlannerCompletionRequirementCorrection(runId, plan, 'planner'); if (this._shouldRecheckReadOnlyFollowUpIntent(plan, historyDigest, followUpContext)) { const intentGate = await this._runPlannerIntentGate( @@ -22058,11 +22276,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ...this._plannerProgressLedgerGateFields(plan), }; } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } const choice = await this._waitForPlanReview(tabId, planId, plan, markdown, onUpdate, verboseMarkdown); if (this._checkAbort(tabId)) { return { proceed: false, message: '[Stopped by user]' }; } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } if (choice?.cancelled || choice?.action === 'reject') { return { proceed: false, message: 'Task cancelled — plan was not approved.' }; } @@ -22180,9 +22404,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } catch {} } - return this._plannerPostValidationFailure(e, onUpdate); + return this._plannerPostValidationFailure(e, onUpdate, runOptions.steeringRevision != null); } await this._tracePlannerAttemptFailure(runId, 'planner', 2, e); + if (runOptions.steeringRevision != null) return this._plannerRequestFailure(e, onUpdate, provider, true); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? (plannerRequestInFlight ? this._plannerRequestFailure(e, onUpdate, provider) @@ -25678,6 +25903,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai { form: tier === 'compact' ? 'brief' : 'full' }, ); if (workflowPolicy) prompt += `\n\n${workflowPolicy}`; + if (this._planExecutionGuards.get(tabId)?.steeringPriorSubmission) { + prompt += '\n\n[The user revised this task after a submission or publication attempt. Inspect and report its existing effect before continuing. Do not repeat or replace that submission; report partial or failed if revised work cannot be completed without another submission.]'; + } } if (this.researchEscalationEnabled) { @@ -30146,6 +30374,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai approvedPlanText, evidenceTaskKey: carryMatches ? carried.evidenceTaskKey : '', taskDrifted: false, + steeringPriorSubmission: carryMatches && carried.steeringPriorSubmission + ? structuredClone(carried.steeringPriorSubmission) + : structuredClone(this._activeTaskBinding(this.conversations.get(tabId) || []).priorSubmission || null), approvedPlan: this._hasApprovedExecutionPlan(this.conversations.get(tabId) || []), // Only the app-owned Continue action can carry verified evidence from // the immediately preceding run; ordinary user turns always start at 0. @@ -30533,6 +30764,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai || guard.successfulConsequentialToolCalls > 0 || guard.pendingDownloadIds.length > 0 || guard.verifiedSubmissionEvidence === true + || !!guard.steeringPriorSubmission || !!guard.workflowTerminalEvidence || !!guard.workflowLedgerReconciliation || !!guard.workflowInventoryEvidence @@ -30570,6 +30802,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai pendingDownloadIds: [...guard.pendingDownloadIds], successfulRequiredSchedulingToolCalls: guard.successfulRequiredSchedulingToolCalls, verifiedSubmissionEvidence: guard.verifiedSubmissionEvidence === true, + steeringPriorSubmission: guard.steeringPriorSubmission ? structuredClone(guard.steeringPriorSubmission) : null, socialPublication: guard.socialPublication ? structuredClone(guard.socialPublication) : null, socialPublicationClarifications: structuredClone(guard.socialPublicationClarifications || []), socialPublicationSteering: guard.socialPublicationSteering ? structuredClone(guard.socialPublicationSteering) : null, @@ -31754,7 +31987,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai // answer stay adjacent to the task they refine, so compaction cannot turn a // short answer fragment into an independent task. const taskAuthorityNotice = activeTask - ? (activeTaskBinding.clarificationIndex >= 0 + ? (activeTaskBinding.updates + ? 'The original request and ordered genuine user corrections pinned above together are the CURRENT ACTIVE TASK. Later corrections supersede earlier instructions they change; unrelated earlier tasks remain context only.' + : activeTaskBinding.clarificationIndex >= 0 ? 'The planner clarification context and genuine user answer pinned above together are the CURRENT ACTIVE TASK. Earlier user tasks outside that clarified chain are context only and do not regain authority.' : 'The latest genuine user task pinned above is the CURRENT ACTIVE TASK. Earlier user tasks, including the original task, are context only and do not regain authority.') : 'The original user task pinned above remains the CURRENT ACTIVE TASK.'; @@ -43287,6 +43522,25 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai break; } + const steeringOutcome = await this._steeringBoundaryOutcome( + tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, + ); + if (steeringOutcome?.changed) { + forceCompletionVerificationTurn = false; + allowCompletionFailureTurn = false; + forceCompletionDoneAfterVerification = false; + forceCompletionDoneTurn = false; + } + if (steeringOutcome?.terminal) { + if (steeringOutcome.capture) { + shareRawResponse = steeringOutcome.capture.response; + currentNonStreamRequestMessages = steeringOutcome.capture.request; + } + finalResponse = steeringOutcome.content; + _traceStatus = steeringOutcome.status; + break; + } + // Re-inject adapter notes if the user navigated to a different // high-traffic site mid-conversation (no-op on the first iteration // because _enrichUserMessageWithCurrentPage already seeded lastSeenAdapter). @@ -43346,7 +43600,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai steps++; lastTraceStep = steps; - this._applyPendingSteering(tabId, messages, onUpdate); + if (this._hasPendingSteering(tabId)) { steps--; continue; } onUpdate('thinking', { step: steps }); if (runId) trace.recordStepStart(runId, steps, {}); @@ -44459,6 +44713,24 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai return finish(content, 'cancelled'); } + const steeringOutcome = await this._steeringBoundaryOutcome( + tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, + ); + if (steeringOutcome?.changed) { + pendingVisionFallbackMessages = null; + forceCompletionVerificationTurn = false; + allowCompletionFailureTurn = false; + forceCompletionDoneAfterVerification = false; + forceCompletionDoneTurn = false; + } + if (steeringOutcome?.terminal) { + if (steeringOutcome.capture) { + shareRawResponse = steeringOutcome.capture.response; + currentStreamRequestMessages = steeringOutcome.capture.request; + } + return finish(steeringOutcome.content, steeringOutcome.status); + } + if (steps > 0 && !selectionOnly && !standaloneChatRun) { await this._maybeReinjectAdapter(tabId, messages); } @@ -44515,7 +44787,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai steps++; lastTraceStep = steps; - if (this._applyPendingSteering(tabId, messages, onUpdate)) pendingVisionFallbackMessages = null; + if (this._hasPendingSteering(tabId)) { steps--; continue; } onUpdate('thinking', { step: steps }); if (runId) trace.recordStepStart(runId, steps, {}); let traceStepClosed = false; diff --git a/src/chrome/src/agent/conversation-persistence.js b/src/chrome/src/agent/conversation-persistence.js index 134c2f5ce2..2477a8b392 100644 --- a/src/chrome/src/agent/conversation-persistence.js +++ b/src/chrome/src/agent/conversation-persistence.js @@ -62,6 +62,9 @@ function sanitizeContent(message, state, caps) { state.compacted = true; return { type: 'text', text: attachmentPlaceholder(message, 'document') }; } + if (caps.textChars === Infinity && block?.type === 'text' && typeof block.text === 'string') { + return { ...sanitizeValue({ ...block, text: '' }, state), text: capText(block.text, caps.textChars, '[large content truncated for session recovery]', state) }; + } return sanitizeValue(block, state); }); } @@ -87,7 +90,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) const out = messages.map(message => ({ ...message })); const preserved = new Set( Array.isArray(preserveMessageIndices) - ? preserveMessageIndices.filter(index => Number.isInteger(index) && index >= 0 && index < out.length) + ? preserveMessageIndices.filter(index => Number.isInteger(index) && index >= 0 && index < out.length).map(index => out[index]) : [], ); let keepRecentFrom = Math.max(1, out.length - 14); @@ -98,7 +101,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) // messages intact even when a long tool loop pushes them outside the // ordinary recent-message window; otherwise a worker restart can promote // only the latest answer fragment as a new task. - if (!message || message.role === 'system' || preserved.has(index)) continue; + if (!message || message.role === 'system' || preserved.has(message)) continue; state.compacted = true; out[index] = { role: message.role, @@ -136,7 +139,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) } for (let index = keepRecentFrom; index < out.length && byteLength(out) > maxBytes; index++) { const message = out[index]; - if (!message || typeof message.content !== 'string' || message.content.length <= 4_000) continue; + if (!message || preserved.has(message) || typeof message.content !== 'string' || message.content.length <= 4_000) continue; state.compacted = true; out[index] = { ...message, content: `${message.content.slice(0, 3_900)}\n[content truncated for session recovery]` }; } @@ -176,7 +179,14 @@ export function serializeConversationForSession(messages, options = {}) { ? { textChars: 16_000, toolChars: 8_000, toolArgsChars: 8_000 } : { textChars: 96_000, toolChars: 32_000, toolArgsChars: 24_000 }; const state = { compacted: false }; - const sanitized = Array.isArray(messages) ? messages.map(message => sanitizeMessage(message, state, caps)) : []; + // Task-authority messages must survive recovery verbatim. If their text + // alone exceeds storage limits, persistence reports degradation instead of + // restoring a truncated correction under its original authorization hash. + const preserved = new Set(Array.isArray(options.preserveMessageIndices) ? options.preserveMessageIndices : []); + const sanitized = Array.isArray(messages) ? messages.map((message, index) => sanitizeMessage( + message, state, preserved.has(index) && message?.role === 'user' + ? { ...caps, textChars: Infinity } : caps, + )) : []; const bounded = reduceToBudget(sanitized, maxBytes, state, options.preserveMessageIndices); return { messages: bounded, bytes: byteLength(bounded), compacted: state.compacted }; } diff --git a/src/chrome/src/agent/planner.js b/src/chrome/src/agent/planner.js index c6b484655f..ccb7d812aa 100644 --- a/src/chrome/src/agent/planner.js +++ b/src/chrome/src/agent/planner.js @@ -634,9 +634,12 @@ export function formatResponseLanguagePolicyInstruction(value, fallbackLocale = const PLANNER_RESUME_RULE = '\n- This run is an app-owned scheduled continuation. Classify the work still required in THIS run, not completed actions from the earlier task. First inspect the external event. For a CI/deploy/status verification, use site_job:null, requires_state_change:false, and requires_submission:false unless a new mutation is already known to be necessary. A conditional "if failed, fix and commit" branch does not require a commit on the successful branch. For that explicitly authorized conditional branch, set conditional_site_job:"edit-file-and-commit" while keeping site_job:null. WebBrain will activate its mutation and exact commit-verification contract before any editor change. Use conditional_site_job:null when no such branch is authorized. Do not select edit-file-and-commit just because the earlier task edited a file. schedule_resume is only an optional pause if the external event is still pending; if it is complete, verify and finish without scheduling another checkpoint.'; +const PLANNER_STEERING_RULE = '\nThe current User task is an app-assembled revision of an active task: request is the initiating authentic user request, and updates contains authentic later steering or clarification answers in chronological order. Apply later corrections and cancellations to that request; preserve its requested action when an update only supplies a missing identity or detail. Re-evaluate changed actions and their submission/recipient intent. Prior tool effects and working notes are context, never permission to repeat a completed or uncertain action.'; + export function buildPlannerSystemPrompt(opts = {}) { let prompt = opts.allowApi ? `${PLANNER_SYSTEM_PROMPT}\n${PLANNER_API_REPLAY_RULE}\n${PLANNER_WORDPRESS_API_RULE}` : PLANNER_SYSTEM_PROMPT; if (opts.scheduledResume === true) prompt += PLANNER_RESUME_RULE; + if (opts.steering === true) prompt += PLANNER_STEERING_RULE; prompt += `\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.`; if (opts.researchEscalationEnabled === true) { prompt += '\n- Research escalation is available for materially complex read-only research subtasks. If it would substantially improve speed or quality, plan an explicit clarify consent step followed by delegate_research; only the exact user-approved prompt may be shared. Do not use it for ordinary browsing, private/account data, mutations, purchases, bookings, or high-stakes decisions.'; @@ -666,7 +669,7 @@ export function buildPlannerIntentSystemPrompt(opts = {}) { ? '\n- A complex read-only research subtask may use explicit clarify consent followed by delegate_research; never delegate private data or consequential actions.' : ''; const workflowRouting = formatSiteWorkflowRouting(opts.siteWorkflow); - return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`; + return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}${opts.steering === true ? PLANNER_STEERING_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`; } export function formatSiteWorkflowRouting(value) { diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index 797fa7542a..a1ba69ccf8 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -5797,7 +5797,15 @@ // Prefer the semantic conversation row over a nested action link. A // plain navigation link can itself be the row only when no stronger // row container exists. + const nativeLinkedInRow = params.adapterName === 'linkedin' + && /^\/messaging(?:\/|$)/.test(location.pathname) + ? clicked.closest?.('li') : null; + const nativeConversationRow = nativeLinkedInRow + && nativeLinkedInRow.parentElement?.matches( + 'ul.msg-conversations-container__conversations-list,ul[aria-label="Conversation List"],ol[aria-label="Conversation List"]', + ) ? nativeLinkedInRow : null; const row = clicked.closest?.(semanticRowSelector) + || nativeConversationRow || clicked.closest?.('a[href]') || null; if (!row || !visible(row) || editable(row)) return false; @@ -5819,9 +5827,18 @@ if (nestedAction && nestedAction !== row && row.contains?.(nestedAction)) return false; } + if (row === nativeConversationRow) { + const rect = clicked.getBoundingClientRect(); + const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); + if (!_isComposedAncestor(clicked, hit)) return false; + for (let node = hit; node && node !== clicked; node = _composedParent(node)) { + if (_isInteractive(node) || node.matches?.('label,option,summary,[contenteditable], [role=checkbox],[role=radio],[role=switch]')) return false; + } + } + const role = String(row.getAttribute?.('role') || '').toLowerCase(); const tag = String(row.tagName || '').toLowerCase(); - const hasRowSemantics = /^(option|listitem|treeitem|tab)$/.test(role) + const hasRowSemantics = row === nativeConversationRow || /^(option|listitem|treeitem|tab)$/.test(role) || row.hasAttribute?.('aria-selected') || row.hasAttribute?.('aria-current') || ['data-conversation-id', 'data-thread-id', 'data-chat-id'] @@ -5843,6 +5860,36 @@ && railRect.right <= composerRect.left + 64; }; + const verifiedLinkedInImagePreview = clicked => { + if (params.adapterName !== 'linkedin') return false; + const button = _composedClosestElement(clicked, 'button,[role="button"]'); + if (!button || compact(button.getAttribute('aria-label')).toLowerCase() + !== 'click or press enter to display in the image preview') return false; + const painted = el => { + if (!visible(el) || !_hasVisibleBox(el)) return false; + for (let node = el; node?.nodeType === 1; node = _composedParent(node)) { + if (getComputedStyle(node).opacity === '0') return false; + } + return true; + }; + if (!button || !painted(button) || button.disabled || button.matches(':disabled') + || button.getAttribute('aria-disabled') === 'true' + || String(button.getAttribute('type') || '').toLowerCase() !== 'button' + || button.form || button.hasAttribute('form') || button.hasAttribute('download') + || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; + // Attachment previews live in history, not in the composer. Proximity + // to the composer cannot turn this verified viewer into a send; labels + // alone cannot exempt a message commit or an unrelated image button. + if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; + if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] + .some(_hasMessageCommitName)) return false; + if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false; + const rect = button.getBoundingClientRect(); + return _isComposedAncestor(button, _shadowAwareElementFromPoint( + rect.left + rect.width / 2, rect.top + rect.height / 2, + )); + }; + const classifyLinkedInNavigation = (clicked, blockingModal = null) => { if (params.adapterName !== 'linkedin' || !clicked) return 'none'; const link = _composedClosestElement(clicked, 'a[href]'); @@ -6199,6 +6246,9 @@ if (!visible(control) || (modal && !_isComposedAncestor(modal, target))) { return { success: true, messageSend: null, conclusive: false, identityCandidates: [] }; } + if (verifiedLinkedInImagePreview(target)) { + return { success: true, messageSend: false, conclusive: true, imagePreview: true, identityCandidates: [] }; + } if (verifiedTwitterNavigation(target)) { return { success: true, messageSend: false, conclusive: true, navigation: true, identityCandidates: [] }; } diff --git a/src/firefox/src/agent/adapters.js b/src/firefox/src/agent/adapters.js index 2658783ffa..0ab90121fc 100644 --- a/src/firefox/src/agent/adapters.js +++ b/src/firefox/src/agent/adapters.js @@ -16322,6 +16322,8 @@ const ADAPTERS = [ - The post composer ("Start a post") opens a dialog rendered inside an open shadow root (#interop-outlet). If it is missing from get_accessibility_tree / click({text}) results even though the screenshot shows it open, do NOT coordinate-click inside it — clicks often hit the dialog backdrop and CLOSE it (the next screenshot shows the regular feed again). Reliable path: click({text: "Start a post"}) to open it, then type_text({selector: '[contenteditable]:not([contenteditable="false"])', text: "...", clear: true}) — this focuses the editor directly without a click. Then click the Post button (find it via get_interactive_elements or shadow_dom_query({selector: "button", shadowPath: "div#interop-outlet"}) if a plain text click matches "Post impressions" instead). - "Connect" button on profiles often has a "Send without a note" prompt — read it before clicking. - Messages are at /messaging — the message composer is a contenteditable with image/file upload icons. +- For a task referring to a named sender's message or event, inspect that sender's messages in the current conversation first. Scroll the message-history pane upward to earlier messages; do not scroll the thread heading or switch to unrelated conversations. Verify the sender beside each attachment before treating it as the event source. +- Open a message image with its image-preview control and inspect the preview. If the event's date, time, time zone, or other required details remain missing, call clarify instead of guessing or claiming completion. - In Messaging, after filling the composer, the reliable send path is usually Enter. If the composer footer says "Press Enter to Send" or the send-options popover shows "Press Enter to Send", call press_keys({key:"Enter"}) from the composer. Do NOT keep scrolling to find a Send button that is already visible/implicit. - The three-dot / send-options control near the composer opens send preferences ("Press Enter to Send" vs "Click Send"); it is not the Send action. If you opened that popover by mistake, choose/keep "Press Enter to Send", close it if needed, then press Enter to send the focused composer. - Search has filters (People, Posts, Jobs, Companies) as tabs at the top.`, diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 6994e310ec..460176a968 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -5772,6 +5772,8 @@ export class Agent extends LoopDetector { requestId: String(runOptions.detachedRequestId || ''), messages: [], acceptedIds: new Set(), + revision: 0, + authorizedRevision: 0, onUpdate, }); } @@ -5791,6 +5793,9 @@ export class Agent extends LoopDetector { if (!run.acceptedIds.has(id)) { run.acceptedIds.add(id); run.messages.push({ id, text: text.trim() }); + // A reviewed plan belongs to the task before this correction. Wake the + // planner so it can review the new revision without approving stale work. + this._cancelPendingPlans(tabId, 'superseded by steering'); // A cached fast-path action was prepared before this correction. const session = this._jevSessions?.get(tabId); if (session) { session.disabled = true; session.queue = []; } @@ -5808,14 +5813,163 @@ export class Agent extends LoopDetector { // Only consume at model/tool boundaries, after all tool results are present. // Human corrections stay ordinary user messages; no page context is added. for (const item of run.messages.splice(0)) { + const priorBinding = this._activeTaskBinding(messages); + const parentTaskKey = this._progressTaskKeyForText(priorBinding.text); + const guard = this._planExecutionGuards.get(tabId); + const submitted = this._completionSubmitStates.get(tabId); + const priorSubmission = guard?.steeringPriorSubmission || priorBinding.priorSubmission + || (submitted?.dispatched ? submitted : null) + || (Object.keys(guard?.socialPublication?.outcomes || {}).length + ? { dispatched: true, publicationOutcomes: guard.socialPublication.outcomes } : null); this._recordSocialPublicationSteering(tabId, item.text); - messages.push({ role: 'user', content: item.text }); + messages.push({ role: 'user', content: item.text, webbrainSteering: { + requestId: run.requestId, messageId: item.id, revision: ++run.revision, parentTaskKey, + // A durable attempt receipt is separate from revised completion proof. + ...(priorSubmission ? { priorSubmission: structuredClone(priorSubmission) } : {}), + } }); onUpdate('steering_applied', item); + this._traceSteeringRevision(tabId, 'steering_applied', { revision: run.revision, messageId: item.id }); } this._persist(tabId); return true; } + _traceSteeringRevision(tabId, note, extra) { + const runId = this.currentRunId.get(tabId); + if (runId) { + try { Promise.resolve(trace.recordNote(runId, 0, note, extra)).catch(() => {}); } catch {} + } + } + + _steeringGateSuperseded(tabId, runOptions) { + return runOptions.steeringRevision != null && ( + this._steeringRuns.get(tabId)?.revision !== runOptions.steeringRevision + || this._hasPendingSteering(tabId) + ); + } + + async _revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions = {}) { + const run = this._steeringRuns.get(tabId); + if (!run) return null; + this._applyPendingSteering(tabId, messages, onUpdate); + if (run.revision === run.authorizedRevision) return null; + if (!this._isActionMode(mode) || this._isStandaloneChatRun(runOptions)) { + run.authorizedRevision = run.revision; + return { changed: true }; + } + // A revision never inherits success counters or dispatch authorization. + // Keep actual attempts separately: resetting a guard must not make a sent + // message, an uncertain submit, or a publication eligible to run again. + const previous = this._planExecutionGuards.get(tabId); + const submitted = this._completionSubmitStates.get(tabId); + const priorSubmission = previous?.steeringPriorSubmission + || (submitted?.dispatched ? submitted : null) + || this._activeTaskBinding(messages).priorSubmission || null; + const attemptedPublication = previous?.socialPublication + && Object.keys(previous.socialPublication.outcomes || {}).length + ? previous.socialPublication : null; + while (!this._checkAbort(tabId)) { + this._applyPendingSteering(tabId, messages, onUpdate); + const revision = run.revision; + const enriched = { role: 'user', content: this._activeTaskBinding(messages).text }; + const revisionOptions = { + ...runOptions, steeringRevision: revision, trustedContinuation: false, + recommendedAction: null, + }; + const readState = this.readCompletenessStates.get(tabId); + if (readState) { + const pageUrl = await this._currentUrl(tabId); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + const adapterName = getActiveAdapter(pageUrl)?.name || ''; + // Let this revision's planner establish its own read obligation. + // Keep run ownership for teardown, but discard earlier coverage and + // obligations, including ones armed by a superseded planner result. + this.readCompletenessStates.set(tabId, createReadCompletenessState( + readState.runToken, false, isCommunicationThreadContext(pageUrl, adapterName), adapterName, + )); + } + const gate = await this._maybeRunPlannerGate( + tabId, messages, enriched, onUpdate, mode, costState, runId, null, revisionOptions, + ); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + run.authorizedRevision = revision; + this._traceSteeringRevision(tabId, 'steering_revalidated', { + revision, requestKind: gate.requestKind, proceed: gate.proceed, reason: gate.reason || null, + }); + if (!gate.proceed) return { changed: true, gate, enriched }; + this._completionSubmitStates.delete(tabId); + const guard = this._startPlanExecutionGuard(tabId, mode, gate, revisionOptions); + guard.steeringPriorSubmission = priorSubmission; + if (priorSubmission || attemptedPublication) guard.evidenceTaskKey = guard.taskKey; + guard.socialPublicationSteering = previous?.socialPublicationSteering || null; + guard.socialPublicationClarifications = previous?.socialPublicationClarifications || []; + if (attemptedPublication) { + guard.socialPublication = attemptedPublication; + this._invalidateSocialPublicationContract(guard); + } + this._setResponseLanguagePolicy(tabId, gate.responseLanguagePolicy, runOptions.locale || 'en', { + approvedPlanLanguageOverride: gate.responseLanguageApprovedPlanOverride === true, + }); + if (!gate.responseOnly) await this._ensureProgressSessionForCurrentTask(tabId, { + provider: this._activeProvider(tabId), costState, + progressLedgerPolicy: gate.progressLedgerPolicy, progressAction: gate.progressAction, + expectedItems: gate.expectedItems, + }); + // The system prompt is rebuilt by the new guard. Clear the old approved + // plan even when only intent classification is enabled for this revision. + if (!gate.approvedScratchpadText && previous?.approvedPlanText) { + const scratchIdx = this._findScratchpadIndex(messages); + if (scratchIdx >= 0) this._scratchpadWrite(tabId, { + replace: true, + text: this._extractScratchpadBody(messages[scratchIdx].content) + .replace(previous.approvedPlanText, '').trim(), + }); + } + this._persist(tabId); + return { changed: true, gate, enriched }; + } + return { changed: true, gate: { proceed: false, reason: 'cancelled', message: '[Stopped by user]' } }; + } + + _steeringPriorSubmissionBlock(tabId, name, args, detectedSubmit) { + if (!this._planExecutionGuards.get(tabId)?.steeringPriorSubmission) return null; + const resolvedNonSubmit = detectedSubmit?.resolvedNavigationTarget === true + || detectedSubmit?.resolvedEditableTarget === true + || detectedSubmit?.resolvedNonSubmitTarget === true; + if (!isNetworkMutation(name, args) && name !== 'chrome_web_store_publish' + && !(detectedSubmit?.isSubmit || (!resolvedNonSubmit && this._isFormValidationCandidate(name, args)))) return null; + return { + success: false, blocked: true, noDispatch: true, dispatched: false, + steeringPriorSubmission: true, + error: 'A submission was already attempted before the user revised this task. Inspect its result before proceeding; do not repeat or replace that submission in this run. Report the verified effect and any remaining revised work with outcome partial or failed.', + }; + } + + async _steeringBoundaryOutcome(tabId, messages, onUpdate, mode, provider, costState, runId, runOptions) { + const refresh = await this._revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions); + if (!refresh?.gate) return refresh; + if (!refresh.gate.proceed) return { + changed: true, terminal: true, content: refresh.gate.message || 'More information is required.', + status: refresh.gate.reason || 'cancelled', + }; + if (refresh.gate.responseOnly) { + const capture = {}; + const response = await this._completeResponseOnlyTurn( + tabId, messages, onUpdate, provider, costState, runId, runOptions, refresh.enriched, null, capture, + ); + // A correction received during a context-only answer supersedes that + // answer exactly like an ordinary in-flight model response. + if (this._hasPendingSteering(tabId)) { + onUpdate('text', { content: '', replace: true }); + return { changed: true }; + } + return { changed: true, terminal: true, ...response, capture }; + } + return { changed: true }; + } + _finishSteeringRun(tabId) { const run = this._steeringRuns.get(tabId); this._steeringRuns.delete(tabId); @@ -11307,7 +11461,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d reason: 'submit the configured Chrome Web Store release for review', }; } - const workflowPreSubmitBlock = await this._conditionalGithubCommitTransition(tabId, fnName, fnArgs, detectedSubmitAction) + const workflowPreSubmitBlock = this._steeringPriorSubmissionBlock(tabId, fnName, fnArgs, detectedSubmitAction) + || await this._conditionalGithubCommitTransition(tabId, fnName, fnArgs, detectedSubmitAction) || await this._workflowPreSubmitDispatchBlock( tabId, fnName, fnArgs, detectedSubmitAction, provider, ); @@ -18169,20 +18324,46 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // boundaries unambiguous when this composite is later embedded in the // trusted recovery anchor or hashed as execution evidence authority, and // the flat shape keeps it bounded across an arbitrarily long chain. - const requestText = this._progressTaskTextKey(taskText).slice(0, 1600); + const requestText = carriesPriorBinding && priorBinding.updates + ? taskText : this._progressTaskTextKey(taskText).slice(0, 1600); const answers = [...carriedAnswers, answerText] .map(answer => this._progressTaskTextKey(answer).slice(0, CLARIFICATION_ANSWER_CHARS)) .filter(Boolean) .slice(-CLARIFICATION_ANSWER_LIMIT); + const updates = carriesPriorBinding && priorBinding.updates + ? [...priorBinding.updates, { kind: 'clarification', text: answerText }] + : null; const text = `User task clarified by the latest answer: ${JSON.stringify({ request: requestText, answers, + ...(updates ? { updates } : {}), })}`; const pinnedIndices = [...carriedPinnedIndices, taskIndex, clarificationIndex, answerIndex] .filter(index => index >= 0) .filter((index, position, all) => all.indexOf(index) === position) .sort((a, b) => a - b); - return { index: answerIndex, taskIndex, clarificationIndex, text, requestText, answers, pinnedIndices }; + return { index: answerIndex, taskIndex, clarificationIndex, text, requestText, answers, pinnedIndices, + ...(updates ? { updates } : {}), + ...(carriesPriorBinding && priorBinding.priorSubmission ? { priorSubmission: priorBinding.priorSubmission } : {}) }; + } + + _steeringTaskBinding(messages, index) { + const message = messages[index]; + const metadata = message?.webbrainSteering; + if (!metadata?.requestId || !metadata.messageId || !Number.isInteger(metadata.revision) + || metadata.revision < 1 || !metadata.parentTaskKey) return null; + const prior = this._activeTaskBinding(messages.slice(0, index)); + if (!prior.text || metadata.parentTaskKey !== this._progressTaskKeyForText(prior.text)) return null; + const updates = [...(prior.updates || (prior.answers || []).map(text => ({ kind: 'clarification', text }))), + { kind: 'steering', text: this._plannerUserAuthoredText(message) }]; + const requestText = prior.requestText || prior.text; + return { + index, taskIndex: prior.taskIndex, clarificationIndex: -1, requestText, + answers: prior.answers || [], updates, + priorSubmission: metadata.priorSubmission?.dispatched === true ? metadata.priorSubmission : prior.priorSubmission || null, + text: `User task revised by steering: ${JSON.stringify({ request: requestText, updates })}`, + pinnedIndices: [...prior.pinnedIndices, index], + }; } // Return the latest genuine task-bearing user turn, not a synthetic runtime @@ -18198,6 +18379,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (this._isAgentInjectedUserMessage(message)) continue; const text = this._plannerUserAuthoredText(message); if (!text) continue; + const steered = this._steeringTaskBinding(messages, i); + if (steered) return steered; const clarification = this._plannerClarificationTaskBinding(messages, i); if (clarification) return clarification; const normalized = text.toLowerCase(); @@ -18535,7 +18718,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d : null; const submittedUserMessage = this._standalonePersistedUserMessage(enriched, runOptions); if (submittedUserMessage !== enriched) runOptions._standalonePersistedUserMessage = submittedUserMessage; - messages.push(submittedUserMessage); + // Steering's actual user messages are already durable. Its composite is + // planner input only; recording it as another user turn would replace the + // original request and break the revision's authority chain. + if (runOptions.steeringRevision == null) messages.push(submittedUserMessage); if (runOptions?.selectionGroundingScopeStarted === true) { this._finalizeSelectionGroundingScope(tabId, messages, submittedUserMessage); } @@ -18555,7 +18741,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } return { proceed: true, requestKind: 'execute', requiresStateChange: false }; } - const fastPathPlan = this._recommendedActionFastPathPlan(runOptions); + const fastPathPlan = runOptions.steeringRevision == null + ? this._recommendedActionFastPathPlan(runOptions) : null; if (fastPathPlan) { this.plannerFollowUpSkipTabs.delete(tabId); const approvedScratchpadText = this._formatRecommendedActionFastPathScratchpad(fastPathPlan); @@ -18586,7 +18773,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const historyDigest = this._buildPlannerHistoryDigest(priorMessages); const followUpContext = this._buildPlannerFollowUpContext(priorMessages); let gate; - if (this._shouldSkipPlannerForShortFollowUp(tabId, priorMessages, enriched, plannerMode)) { + if (runOptions.steeringRevision == null && this._shouldSkipPlannerForShortFollowUp(tabId, priorMessages, enriched, plannerMode)) { this.plannerFollowUpSkipTabs.delete(tabId); gate = await this._runPlannerIntentGate( tabId, enriched, onUpdate, costState, runId, historyDigest, tabInfo, mode, runOptions, followUpContext, @@ -18601,6 +18788,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d tabId, enriched, onUpdate, costState, runId, historyDigest, tabInfo, mode, runOptions, followUpContext, ); } + if (this._steeringGateSuperseded(tabId, runOptions)) return { proceed: false, steeringSuperseded: true }; + if (runOptions.steeringRevision == null && this._hasPendingSteering(tabId)) { + const refresh = await this._revalidatePendingSteering(tabId, messages, onUpdate, mode, costState, runId, runOptions); + if (refresh?.gate) return refresh.gate; + } if (runOptions?.trustedContinuation === true && gate?.proceed && (gate.requestKind === 'execute' || gate.plannerFailedContinueAct === true) @@ -18646,7 +18838,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } if (gate.approvedScratchpadText) { - const scratchResult = this._scratchpadWrite(tabId, { text: gate.approvedScratchpadText }); + let scratchText = gate.approvedScratchpadText; + if (runOptions.steeringRevision != null) { + const scratchIdx = this._findScratchpadIndex(messages); + const previousPlan = this._planExecutionGuards.get(tabId)?.approvedPlanText || ''; + if (scratchIdx >= 0) scratchText = + this._extractScratchpadBody(messages[scratchIdx].content).replace(previousPlan, '').trim() + + '\n\n' + scratchText; + } + const scratchResult = this._scratchpadWrite(tabId, { + text: scratchText, replace: runOptions.steeringRevision != null, + }); if (!scratchResult?.success) { onUpdate('warning', { message: scratchResult?.error || 'Could not pin plan to scratchpad.' }); } else { @@ -18908,6 +19110,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } _plannerActContinuation(runOptions, onUpdate, runId = null, reason = 'invalid_output') { + if (runOptions.steeringRevision != null) return { + proceed: false, reason: 'planner_error', requestKind: 'respond', + message: `WebBrain could not revalidate the revised task (${reason}). No further actions were dispatched.`, + }; const message = this._plannerIntentFailureMessage(runOptions); onUpdate('warning', { code: 'planner_failed_continue_act', @@ -18961,14 +19167,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } - _plannerRequestFailure(error, onUpdate, provider = null) { + _plannerRequestFailure(error, onUpdate, provider = null, steering = false) { const detail = sanitizePlannerText( formatErrorMessage(error, { fallback: 'Unknown planner request error.' }), 500, { collapseWhitespace: true }, ); const detailSentence = /[.!?]$/.test(detail) ? detail : `${detail}.`; - const message = `Planner request failed before a valid response was available: ${detailSentence} No tools ran.`; + const message = `Planner request failed before a valid response was available: ${detailSentence} ${steering ? 'No further actions were dispatched after steering.' : 'No tools ran.'}`; const failureKind = plannerRequestFailureKind(detail); onUpdate('warning', { code: 'planner_request_failed', @@ -18990,14 +19196,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } - _plannerPostValidationFailure(error, onUpdate) { + _plannerPostValidationFailure(error, onUpdate, steering = false) { const detail = sanitizePlannerText( formatErrorMessage(error, { fallback: 'Unknown plan processing error.' }), 500, { collapseWhitespace: true }, ); const detailSentence = /[.!?]$/.test(detail) ? detail : `${detail}.`; - const message = `A valid plan was produced, but WebBrain could not safely finish plan review or prepare the plan for execution: ${detailSentence} No tools ran.`; + const message = `A valid plan was produced, but WebBrain could not safely finish plan review or prepare the plan for execution: ${detailSentence} ${steering ? 'No further actions were dispatched after steering.' : 'No tools ran.'}`; onUpdate('warning', { code: 'planner_processing_failed', message, @@ -19420,6 +19626,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const recheckOnly = runOptions?.plannerIntentRecheckOnly === true; const provider = this.providerManager.getActive(); const plannerMessages = buildPlannerIntentMessages(enriched, tabUrl, tabTitle, historyDigest, { + steering: runOptions.steeringRevision != null, noThink: this._plannerPrefersNoThinkPrompt(provider), scheduledResume: runOptions?.scheduledResume === true, locale, @@ -19533,6 +19740,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ? this._plannerIntentRecheckFallback() : this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } await this._tracePlannerCompletionRequirementCorrection(runId, plan, 'intent'); if (plan.request_kind === 'respond') { return { @@ -19621,9 +19831,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } catch {} } - return this._plannerPostValidationFailure(e, onUpdate); + return this._plannerPostValidationFailure(e, onUpdate, runOptions.steeringRevision != null); } await this._tracePlannerAttemptFailure(runId, 'intent', 2, e); + if (runOptions.steeringRevision != null) return this._plannerRequestFailure(e, onUpdate, provider, true); return this._plannerActContinuation(runOptions, onUpdate, runId, 'request_error'); } } @@ -19646,6 +19857,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const tier = this._resolvePromptTier(); const skillCatalog = this._skillCatalog(conversationMode, tier); const plannerMessages = buildPlannerMessages(enriched, tabUrl, tabTitle, historyDigest, { + steering: runOptions.steeringRevision != null, noThink: this._plannerPrefersNoThinkPrompt(provider), scheduledResume: runOptions?.scheduledResume === true, allowApi: this.isApiMutationsAllowed(tabId), @@ -19709,6 +19921,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (this._checkAbort(tabId)) { return { proceed: false, message: '[Stopped by user]' }; } + if (this._steeringGateSuperseded(tabId, runOptions)) return { proceed: false, steeringSuperseded: true }; let consistencyRepairKind = null; let plan = parsePlanFromContent(result.content, plannerParseOptions); // Retry whenever the first attempt yields no parseable plan — empty @@ -19770,16 +19983,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return { proceed: false, message: '[Stopped by user]' }; } if (!plan) { + if (runOptions.steeringRevision != null) return this._plannerActContinuation(runOptions, onUpdate, runId, 'invalid_output'); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? this._strictPlannerFailure(onUpdate) : this._plannerActContinuation(runOptions, onUpdate, runId, 'invalid_output'); } hasValidPlannerResponse = true; if (this._plannerIntentUnresolvedConsistencyIssue(plan, consistencyRepairKind, followUpContext)) { + if (runOptions.steeringRevision != null) return this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? this._strictPlannerFailure(onUpdate) : this._plannerActContinuation(runOptions, onUpdate, runId, 'inconsistent_intent'); } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } await this._tracePlannerCompletionRequirementCorrection(runId, plan, 'planner'); if (this._shouldRecheckReadOnlyFollowUpIntent(plan, historyDigest, followUpContext)) { const intentGate = await this._runPlannerIntentGate( @@ -19890,11 +20108,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ...this._plannerProgressLedgerGateFields(plan), }; } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } const choice = await this._waitForPlanReview(tabId, planId, plan, markdown, onUpdate, verboseMarkdown); if (this._checkAbort(tabId)) { return { proceed: false, message: '[Stopped by user]' }; } + if (this._hasPendingSteering(tabId) || this._steeringGateSuperseded(tabId, runOptions)) { + return { proceed: false, steeringSuperseded: true }; + } if (choice?.cancelled || choice?.action === 'reject') { return { proceed: false, message: 'Task cancelled — plan was not approved.' }; } @@ -20012,9 +20236,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } catch {} } - return this._plannerPostValidationFailure(e, onUpdate); + return this._plannerPostValidationFailure(e, onUpdate, runOptions.steeringRevision != null); } await this._tracePlannerAttemptFailure(runId, 'planner', 2, e); + if (runOptions.steeringRevision != null) return this._plannerRequestFailure(e, onUpdate, provider, true); return this._normalizePlanBeforeActMode(plannerMode) === 'strict' ? (plannerRequestInFlight ? this._plannerRequestFailure(e, onUpdate, provider) @@ -23321,6 +23546,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d { form: tier === 'compact' ? 'brief' : 'full' }, ); if (workflowPolicy) prompt += `\n\n${workflowPolicy}`; + if (this._planExecutionGuards.get(tabId)?.steeringPriorSubmission) { + prompt += '\n\n[The user revised this task after a submission or publication attempt. Inspect and report its existing effect before continuing. Do not repeat or replace that submission; report partial or failed if revised work cannot be completed without another submission.]'; + } } if (this.researchEscalationEnabled) { @@ -27765,6 +27993,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d approvedPlanText, evidenceTaskKey: carryMatches ? carried.evidenceTaskKey : '', taskDrifted: false, + steeringPriorSubmission: carryMatches && carried.steeringPriorSubmission + ? structuredClone(carried.steeringPriorSubmission) + : structuredClone(this._activeTaskBinding(this.conversations.get(tabId) || []).priorSubmission || null), approvedPlan: this._hasApprovedExecutionPlan(this.conversations.get(tabId) || []), // Only the app-owned Continue action can carry verified evidence from // the immediately preceding run; ordinary user turns always start at 0. @@ -28152,6 +28383,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d || guard.successfulConsequentialToolCalls > 0 || guard.pendingDownloadIds.length > 0 || guard.verifiedSubmissionEvidence === true + || !!guard.steeringPriorSubmission || !!guard.workflowTerminalEvidence || !!guard.workflowLedgerReconciliation || !!guard.workflowInventoryEvidence @@ -28189,6 +28421,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d pendingDownloadIds: [...guard.pendingDownloadIds], successfulRequiredSchedulingToolCalls: guard.successfulRequiredSchedulingToolCalls, verifiedSubmissionEvidence: guard.verifiedSubmissionEvidence === true, + steeringPriorSubmission: guard.steeringPriorSubmission ? structuredClone(guard.steeringPriorSubmission) : null, socialPublication: guard.socialPublication ? structuredClone(guard.socialPublication) : null, socialPublicationClarifications: structuredClone(guard.socialPublicationClarifications || []), socialPublicationSteering: guard.socialPublicationSteering ? structuredClone(guard.socialPublicationSteering) : null, @@ -29355,7 +29588,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // answer stay adjacent to the task they refine, so compaction cannot turn a // short answer fragment into an independent task. const taskAuthorityNotice = activeTask - ? (activeTaskBinding.clarificationIndex >= 0 + ? (activeTaskBinding.updates + ? 'The original request and ordered genuine user corrections pinned above together are the CURRENT ACTIVE TASK. Later corrections supersede earlier instructions they change; unrelated earlier tasks remain context only.' + : activeTaskBinding.clarificationIndex >= 0 ? 'The planner clarification context and genuine user answer pinned above together are the CURRENT ACTIVE TASK. Earlier user tasks outside that clarified chain are context only and do not regain authority.' : 'The latest genuine user task pinned above is the CURRENT ACTIVE TASK. Earlier user tasks, including the original task, are context only and do not regain authority.') : 'The original user task pinned above remains the CURRENT ACTIVE TASK.'; @@ -36239,6 +36474,25 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d break; } + const steeringOutcome = await this._steeringBoundaryOutcome( + tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, + ); + if (steeringOutcome?.changed) { + forceCompletionVerificationTurn = false; + allowCompletionFailureTurn = false; + forceCompletionDoneAfterVerification = false; + forceCompletionDoneTurn = false; + } + if (steeringOutcome?.terminal) { + if (steeringOutcome.capture) { + shareRawResponse = steeringOutcome.capture.response; + currentNonStreamRequestMessages = steeringOutcome.capture.request; + } + finalResponse = steeringOutcome.content; + _traceStatus = steeringOutcome.status; + break; + } + if (steps > 0 && !selectionOnly && !standaloneChatRun) { await this._maybeReinjectAdapter(tabId, messages); } @@ -36294,7 +36548,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d steps++; lastTraceStep = steps; - this._applyPendingSteering(tabId, messages, onUpdate); + if (this._hasPendingSteering(tabId)) { steps--; continue; } onUpdate('thinking', { step: steps }); if (runId) trace.recordStepStart(runId, steps, {}); @@ -37241,6 +37495,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return finish(content, 'cancelled'); } + const steeringOutcome = await this._steeringBoundaryOutcome( + tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, + ); + if (steeringOutcome?.changed) { + pendingVisionFallbackMessages = null; + forceCompletionVerificationTurn = false; + allowCompletionFailureTurn = false; + forceCompletionDoneAfterVerification = false; + forceCompletionDoneTurn = false; + } + if (steeringOutcome?.terminal) { + if (steeringOutcome.capture) { + shareRawResponse = steeringOutcome.capture.response; + currentStreamRequestMessages = steeringOutcome.capture.request; + } + return finish(steeringOutcome.content, steeringOutcome.status); + } + if (steps > 0 && !selectionOnly && !standaloneChatRun) { await this._maybeReinjectAdapter(tabId, messages); } @@ -37296,7 +37568,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d steps++; lastTraceStep = steps; - this._applyPendingSteering(tabId, messages, onUpdate); + if (this._hasPendingSteering(tabId)) { steps--; continue; } onUpdate('thinking', { step: steps }); if (runId) trace.recordStepStart(runId, steps, {}); let traceStepClosed = false; diff --git a/src/firefox/src/agent/conversation-persistence.js b/src/firefox/src/agent/conversation-persistence.js index 134c2f5ce2..2477a8b392 100644 --- a/src/firefox/src/agent/conversation-persistence.js +++ b/src/firefox/src/agent/conversation-persistence.js @@ -62,6 +62,9 @@ function sanitizeContent(message, state, caps) { state.compacted = true; return { type: 'text', text: attachmentPlaceholder(message, 'document') }; } + if (caps.textChars === Infinity && block?.type === 'text' && typeof block.text === 'string') { + return { ...sanitizeValue({ ...block, text: '' }, state), text: capText(block.text, caps.textChars, '[large content truncated for session recovery]', state) }; + } return sanitizeValue(block, state); }); } @@ -87,7 +90,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) const out = messages.map(message => ({ ...message })); const preserved = new Set( Array.isArray(preserveMessageIndices) - ? preserveMessageIndices.filter(index => Number.isInteger(index) && index >= 0 && index < out.length) + ? preserveMessageIndices.filter(index => Number.isInteger(index) && index >= 0 && index < out.length).map(index => out[index]) : [], ); let keepRecentFrom = Math.max(1, out.length - 14); @@ -98,7 +101,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) // messages intact even when a long tool loop pushes them outside the // ordinary recent-message window; otherwise a worker restart can promote // only the latest answer fragment as a new task. - if (!message || message.role === 'system' || preserved.has(index)) continue; + if (!message || message.role === 'system' || preserved.has(message)) continue; state.compacted = true; out[index] = { role: message.role, @@ -136,7 +139,7 @@ function reduceToBudget(messages, maxBytes, state, preserveMessageIndices = []) } for (let index = keepRecentFrom; index < out.length && byteLength(out) > maxBytes; index++) { const message = out[index]; - if (!message || typeof message.content !== 'string' || message.content.length <= 4_000) continue; + if (!message || preserved.has(message) || typeof message.content !== 'string' || message.content.length <= 4_000) continue; state.compacted = true; out[index] = { ...message, content: `${message.content.slice(0, 3_900)}\n[content truncated for session recovery]` }; } @@ -176,7 +179,14 @@ export function serializeConversationForSession(messages, options = {}) { ? { textChars: 16_000, toolChars: 8_000, toolArgsChars: 8_000 } : { textChars: 96_000, toolChars: 32_000, toolArgsChars: 24_000 }; const state = { compacted: false }; - const sanitized = Array.isArray(messages) ? messages.map(message => sanitizeMessage(message, state, caps)) : []; + // Task-authority messages must survive recovery verbatim. If their text + // alone exceeds storage limits, persistence reports degradation instead of + // restoring a truncated correction under its original authorization hash. + const preserved = new Set(Array.isArray(options.preserveMessageIndices) ? options.preserveMessageIndices : []); + const sanitized = Array.isArray(messages) ? messages.map((message, index) => sanitizeMessage( + message, state, preserved.has(index) && message?.role === 'user' + ? { ...caps, textChars: Infinity } : caps, + )) : []; const bounded = reduceToBudget(sanitized, maxBytes, state, options.preserveMessageIndices); return { messages: bounded, bytes: byteLength(bounded), compacted: state.compacted }; } diff --git a/src/firefox/src/agent/planner.js b/src/firefox/src/agent/planner.js index c6b484655f..ccb7d812aa 100644 --- a/src/firefox/src/agent/planner.js +++ b/src/firefox/src/agent/planner.js @@ -634,9 +634,12 @@ export function formatResponseLanguagePolicyInstruction(value, fallbackLocale = const PLANNER_RESUME_RULE = '\n- This run is an app-owned scheduled continuation. Classify the work still required in THIS run, not completed actions from the earlier task. First inspect the external event. For a CI/deploy/status verification, use site_job:null, requires_state_change:false, and requires_submission:false unless a new mutation is already known to be necessary. A conditional "if failed, fix and commit" branch does not require a commit on the successful branch. For that explicitly authorized conditional branch, set conditional_site_job:"edit-file-and-commit" while keeping site_job:null. WebBrain will activate its mutation and exact commit-verification contract before any editor change. Use conditional_site_job:null when no such branch is authorized. Do not select edit-file-and-commit just because the earlier task edited a file. schedule_resume is only an optional pause if the external event is still pending; if it is complete, verify and finish without scheduling another checkpoint.'; +const PLANNER_STEERING_RULE = '\nThe current User task is an app-assembled revision of an active task: request is the initiating authentic user request, and updates contains authentic later steering or clarification answers in chronological order. Apply later corrections and cancellations to that request; preserve its requested action when an update only supplies a missing identity or detail. Re-evaluate changed actions and their submission/recipient intent. Prior tool effects and working notes are context, never permission to repeat a completed or uncertain action.'; + export function buildPlannerSystemPrompt(opts = {}) { let prompt = opts.allowApi ? `${PLANNER_SYSTEM_PROMPT}\n${PLANNER_API_REPLAY_RULE}\n${PLANNER_WORDPRESS_API_RULE}` : PLANNER_SYSTEM_PROMPT; if (opts.scheduledResume === true) prompt += PLANNER_RESUME_RULE; + if (opts.steering === true) prompt += PLANNER_STEERING_RULE; prompt += `\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.`; if (opts.researchEscalationEnabled === true) { prompt += '\n- Research escalation is available for materially complex read-only research subtasks. If it would substantially improve speed or quality, plan an explicit clarify consent step followed by delegate_research; only the exact user-approved prompt may be shared. Do not use it for ordinary browsing, private/account data, mutations, purchases, bookings, or high-stakes decisions.'; @@ -666,7 +669,7 @@ export function buildPlannerIntentSystemPrompt(opts = {}) { ? '\n- A complex read-only research subtask may use explicit clarify consent followed by delegate_research; never delegate private data or consequential actions.' : ''; const workflowRouting = formatSiteWorkflowRouting(opts.siteWorkflow); - return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`; + return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}${opts.steering === true ? PLANNER_STEERING_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`; } export function formatSiteWorkflowRouting(value) { diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 7257611f87..799746d646 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -4885,7 +4885,15 @@ // Prefer the semantic conversation row over a nested action link. A // plain navigation link can itself be the row only when no stronger // row container exists. + const nativeLinkedInRow = params.adapterName === 'linkedin' + && /^\/messaging(?:\/|$)/.test(location.pathname) + ? clicked.closest?.('li') : null; + const nativeConversationRow = nativeLinkedInRow + && nativeLinkedInRow.parentElement?.matches( + 'ul.msg-conversations-container__conversations-list,ul[aria-label="Conversation List"],ol[aria-label="Conversation List"]', + ) ? nativeLinkedInRow : null; const row = clicked.closest?.(semanticRowSelector) + || nativeConversationRow || clicked.closest?.('a[href]') || null; if (!row || !visible(row) || editable(row)) return false; @@ -4907,9 +4915,18 @@ if (nestedAction && nestedAction !== row && row.contains?.(nestedAction)) return false; } + if (row === nativeConversationRow) { + const rect = clicked.getBoundingClientRect(); + const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); + if (!_isComposedAncestor(clicked, hit)) return false; + for (let node = hit; node && node !== clicked; node = _composedParent(node)) { + if (_isInteractive(node) || node.matches?.('label,option,summary,[contenteditable], [role=checkbox],[role=radio],[role=switch]')) return false; + } + } + const role = String(row.getAttribute?.('role') || '').toLowerCase(); const tag = String(row.tagName || '').toLowerCase(); - const hasRowSemantics = /^(option|listitem|treeitem|tab)$/.test(role) + const hasRowSemantics = row === nativeConversationRow || /^(option|listitem|treeitem|tab)$/.test(role) || row.hasAttribute?.('aria-selected') || row.hasAttribute?.('aria-current') || ['data-conversation-id', 'data-thread-id', 'data-chat-id'] @@ -4931,6 +4948,36 @@ && railRect.right <= composerRect.left + 64; }; + const verifiedLinkedInImagePreview = clicked => { + if (params.adapterName !== 'linkedin') return false; + const button = _composedClosestElement(clicked, 'button,[role="button"]'); + if (!button || compact(button.getAttribute('aria-label')).toLowerCase() + !== 'click or press enter to display in the image preview') return false; + const painted = el => { + if (!visible(el) || !_hasVisibleBox(el)) return false; + for (let node = el; node?.nodeType === 1; node = _composedParent(node)) { + if (getComputedStyle(node).opacity === '0') return false; + } + return true; + }; + if (!button || !painted(button) || button.disabled || button.matches(':disabled') + || button.getAttribute('aria-disabled') === 'true' + || String(button.getAttribute('type') || '').toLowerCase() !== 'button' + || button.form || button.hasAttribute('form') || button.hasAttribute('download') + || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; + // Attachment previews live in history, not in the composer. Proximity + // to the composer cannot turn this verified viewer into a send; labels + // alone cannot exempt a message commit or an unrelated image button. + if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; + if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] + .some(_hasMessageCommitName)) return false; + if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false; + const rect = button.getBoundingClientRect(); + return _isComposedAncestor(button, _shadowAwareElementFromPoint( + rect.left + rect.width / 2, rect.top + rect.height / 2, + )); + }; + const classifyLinkedInNavigation = (clicked, blockingModal = null) => { if (params.adapterName !== 'linkedin' || !clicked) return 'none'; const link = _composedClosestElement(clicked, 'a[href]'); @@ -5280,6 +5327,9 @@ if (!visible(control) || (modal && !_isComposedAncestor(modal, target))) { return { success: true, messageSend: null, conclusive: false, identityCandidates: [] }; } + if (verifiedLinkedInImagePreview(target)) { + return { success: true, messageSend: false, conclusive: true, imagePreview: true, identityCandidates: [] }; + } if (verifiedTwitterNavigation(target)) { return { success: true, messageSend: false, conclusive: true, navigation: true, identityCandidates: [] }; } diff --git a/test/chat-steering-ui.mjs b/test/chat-steering-ui.mjs index 1c3f1281da..3187a78fa0 100644 --- a/test/chat-steering-ui.mjs +++ b/test/chat-steering-ui.mjs @@ -128,7 +128,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) { const deliverySetting = page.locator('#select-composer-delivery-mode'); await page.waitForFunction(() => composerDeliveryMode === 'steer' && composerDeliveryModeSelect.value === 'steer'); - assert.equal(await page.locator('#btn-send').getAttribute('title'), 'sp.steer.title', + assert.equal(await page.locator('#btn-send').getAttribute('title'), 'sp.steer.title · sp.queue.send (Alt+Shift+Enter)', 'Saved preference hydrates both settings and composer'); assert.equal(await page.locator('#btn-steer').count(), 0, 'No separate composer Steer button'); await deliverySetting.selectOption('queue'); diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index e36581ff94..7cfe9f8a68 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import vm from 'node:vm'; import { test } from 'node:test'; +import { serializeConversationForSession } from '../src/chrome/src/agent/conversation-persistence.js'; const area = { get: async () => ({}), set: async () => {}, remove: async () => {} }; const api = { @@ -57,6 +58,21 @@ function allowBatchPreparation(agent) { agent._shouldAutoScreenshot = () => false; } +function actSetup(Agent, implementation = {}, classify = async () => ({ + proceed: true, requestKind: 'execute', requiresStateChange: true, requiresSubmission: false, +})) { + const agent = setup(Agent, implementation); + delete agent._maybeRunPlannerGate; + agent._plannerMode = () => 'off'; + agent._runPlannerIntentGate = classify; + agent._persistSubmittedTurn = async () => {}; + agent._getTabUrlTitle = async () => ({ tabUrl: 'https://www.linkedin.com/messaging/thread/example/', tabTitle: 'Messaging' }); + agent._ensureProgressSessionForCurrentTask = async () => {}; + allowBatchPreparation(agent); + agent._completionDoneBlock = () => null; + return agent; +} + function assertPairedTools(messages) { let pending = new Set(); for (const message of messages) { @@ -92,7 +108,7 @@ for (const build of ['chrome', 'firefox']) { assert.equal(await agent._maybeJevFastTurn(tabId, 'Original task', [], 'act', new Set(), {}, null), null); const messages = []; assert.ok(agent._applyPendingSteering(tabId, messages, (type, data) => updates.push({ type, data }))); - assert.deepEqual(messages, [{ role: 'user', content: 'Use blue' }, { role: 'user', content: 'Keep the logo' }]); + assert.deepEqual(messages.map(({ role, content }) => ({ role, content })), [{ role: 'user', content: 'Use blue' }, { role: 'user', content: 'Keep the logo' }]); assert.equal(updates.filter(update => update.type === 'steering_applied').length, 2); agent._finishSteeringRun(tabId); assert.equal(steer(agent, 'Too late').accepted, false); @@ -100,6 +116,96 @@ for (const build of ['chrome', 'firefox']) { }); for (const streaming of [false, true]) { + test(`${build}: ${streaming ? 'stream' : 'chat'} reauthorizes Ipek steering with the real Act guard`, async () => { + const entered = deferred(), release = deferred(); + const plans = [], dispatched = [], requests = []; + let calls = 0; + const next = async messages => { + requests.push(structuredClone(messages)); + if (++calls === 1) { + entered.resolve(); await release.promise; + return { toolCalls: [{ id: 'old', function: { name: 'navigate', arguments: '{"url":"https://example.com/old"}' } }] }; + } + if (calls === 2) return { toolCalls: [{ id: 'calendar', function: { name: 'navigate', arguments: '{"url":"https://calendar.google.com/"}' } }] }; + return { toolCalls: [{ id: 'done', function: { name: 'done', arguments: '{"summary":"Revised task executed","outcome":"success"}' } }] }; + }; + const agent = actSetup(Agent, { + chat: next, + async *chatStream(messages) { const result = await next(messages); yield { type: 'tool_call', content: result.toolCalls }; yield { type: 'done' }; }, + }, async (_tab, enriched) => { + plans.push(enriched.content); + return { proceed: true, requestKind: 'execute', requiresStateChange: true, requiresSubmission: false }; + }); + agent.executeTool = async (_tab, name, args) => { + if (name === 'done') { + const guard = agent._planExecutionGuards.get(tabId); + assert.equal(guard.taskDrifted, false); + assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null); + return { done: true, summary: args.summary, outcome: args.outcome }; + } + dispatched.push(args.url); + return { success: true, url: args.url }; + }; + const run = streaming + ? agent.processMessageStream(tabId, 'add this event to my google calendar', () => {}, 'act', options) + : agent.processMessage(tabId, 'add this event to my google calendar', () => {}, 'act', [], options); + await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]); + steer(agent, 'I mean Ipek’s message'); release.resolve(); + assert.equal(await run, 'Revised task executed'); + assert.equal(plans.length, 2); + assert.match(plans[1], /add this event to my google calendar/); + assert.match(plans[1], /Ipek/); + assert.deepEqual(dispatched, ['https://calendar.google.com/']); + assertPairedTools(requests.at(-1)); + }); + + for (const replacement of [false, true]) { + test(`${build}: ${streaming ? 'stream' : 'chat'} steering ${replacement ? 'replaces' : 'cancels'} the active Act task`, async () => { + const entered = deferred(), release = deferred(), dispatched = []; + let modelCalls = 0, planCalls = 0; + const next = async () => { + if (++modelCalls === 1) { + entered.resolve(); await release.promise; + return { toolCalls: [{ id: 'stale', function: { name: 'navigate', arguments: '{"url":"https://calendar.google.com/"}' } }] }; + } + if (!replacement) return { content: 'Calendar task cancelled.' }; + if (modelCalls === 2) return { toolCalls: [{ id: 'replacement', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }] }; + return { toolCalls: [{ id: 'done', function: { name: 'done', arguments: '{"summary":"Dashboard opened","outcome":"success"}' } }] }; + }; + const agent = actSetup(Agent, { + chat: next, + async *chatStream() { + const result = await next(); + if (result.content) yield { type: 'text', content: result.content }; + if (result.toolCalls) yield { type: 'tool_call', content: result.toolCalls }; + yield { type: 'done' }; + }, + }, async (_tab, enriched) => { + if (++planCalls > 1) { + assert.match(enriched.content, /Cancel the calendar task/); + if (!replacement) return { proceed: true, responseOnly: true, requestKind: 'respond' }; + } + return { proceed: true, requestKind: 'execute', requiresStateChange: true, requiresSubmission: false }; + }); + agent.executeTool = async (_tab, name, args) => { + if (name === 'done') { + assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null); + return { done: true, summary: args.summary, outcome: args.outcome }; + } + dispatched.push(args.url); return { success: true, url: args.url }; + }; + const run = streaming + ? agent.processMessageStream(tabId, 'Add the event to my calendar', () => {}, 'act', options) + : agent.processMessage(tabId, 'Add the event to my calendar', () => {}, 'act', [], options); + await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]); + steer(agent, replacement ? 'Cancel the calendar task; open my dashboard instead' : 'Cancel the calendar task; just confirm cancellation'); + release.resolve(); + assert.equal(await run, replacement ? 'Dashboard opened' : 'Calendar task cancelled.'); + assert.equal(planCalls, 2); + assert.deepEqual(dispatched, replacement ? ['https://example.com/dashboard'] : []); + }); + } + test(`${build}: ${streaming ? 'stream' : 'chat'} corrects an in-flight response before dispatch`, async () => { const entered = deferred(); const release = deferred(); @@ -169,6 +275,308 @@ for (const build of ['chrome', 'firefox']) { }); } + test(`${build}: revisions preserve authority through recovery and reset completion evidence`, async () => { + const plans = []; + const agent = actSetup(Agent, {}, async (_tab, enriched) => { + plans.push(enriched.content); + return { proceed: true, requestKind: 'execute', requiresStateChange: true }; + }); + await agent._claimRunEntry(tabId, 'interactive', options); + agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event to my calendar.' }]; + agent.conversations.set(tabId, messages); + const original = agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + agent._markPlanExecutionToolCall(tabId, 'click', { success: true }, { consequential: true }); + assert.equal(agent._executionEvidenceSatisfied(original), true); + steer(agent, 'Use Ipek’s message', 'a'); steer(agent, 'Use the work calendar', 'b'); steer(agent, 'Use the work calendar', 'b'); + const refreshed = await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + assert.equal(refreshed.gate.proceed, true); + assert.equal(plans.length, 1); + const guard = agent._planExecutionGuards.get(tabId); + assert.equal(guard.successfulConsequentialToolCalls, 0); + assert.equal(agent._executionEvidenceSatisfied(guard), false); + assert.notEqual(guard.taskKey, original.taskKey); + const binding = agent._activeTaskBinding(messages); + assert.deepEqual(binding.updates.map(update => update.text), ['Use Ipek’s message', 'Use the work calendar']); + messages.push(...Array.from({ length: 70 }, () => ({ role: 'assistant', content: 'Long tool-loop context '.repeat(1000) }))); + const recovered = serializeConversationForSession(messages, { maxBytes: 40000, preserveMessageIndices: binding.pinnedIndices }).messages; + assert.equal(agent._activeTaskBinding(recovered).text, binding.text); + recovered.push({ role: 'user', content: 'New task: inspect the dashboard.' }); + assert.equal(agent._activeTaskBinding(recovered).text, 'New task: inspect the dashboard.'); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + test(`${build}: steering during planning discards the superseded intent`, async () => { + const entered = deferred(), release = deferred(), plans = []; + const agent = actSetup(Agent, {}, async (_tab, enriched) => { + plans.push(enriched.content); + if (plans.length === 1) { + agent._armReadCompletenessFromPlan(tabId, { request_kind: 'execute', read_scope: 'complete_thread' }); + entered.resolve(); await release.promise; + } + return { proceed: true, requestKind: 'execute', requiresStateChange: true }; + }); + delete agent._beginReadCompleteness; + agent._currentUrl = async () => 'https://mail.google.com/mail/u/0/#inbox/FMfc123'; + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + const token = await agent._beginReadCompleteness(tabId, messages[1].content, options); + steer(agent, 'From Ipek', 'a'); + const refresh = agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + await entered.promise; + assert.ok(agent._readCompletenessBlock(tabId), 'Superseded planner armed a read obligation'); + steer(agent, 'Tomorrow instead', 'b'); release.resolve(); + await refresh; + assert.equal(plans.length, 2); + assert.match(plans[1], /Ipek.*Tomorrow/); + assert.equal(agent._steeringRuns.get(tabId).authorizedRevision, 2); + assert.equal(agent._readCompletenessBlock(tabId), null, 'Superseded planner obligation cannot leak into the latest revision'); + agent._clearReadCompleteness(tabId, token); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + for (const plannerMode of ['off', 'try']) { + test(`${build}: ${plannerMode} steering replaces read obligations and discards old coverage`, async () => { + const agent = actSetup(Agent); + delete agent._beginReadCompleteness; + delete agent._runPlannerIntentGate; + agent._plannerMode = () => plannerMode; + agent.setPlanReviewSettings({ mode: 'never' }); + let pageUrl = 'https://mail.google.com/mail/u/0/#inbox/FMfc123'; + agent._currentUrl = async () => pageUrl; + agent._getTabUrlTitle = async () => ({ tabUrl: pageUrl, tabTitle: 'Current page' }); + const plan = { + request_kind: 'execute', requires_state_change: true, requires_submission: false, + read_scope: 'none', summary: 'Open the dashboard', confidence: 0.99, + steps: [{ id: '1', action: 'Open the dashboard', tools: ['navigate'] }], + memory: { use_scratchpad: false, use_progress_ledger: false }, risks: [], + }; + agent._chatWithCostAllowance = async () => ({ content: JSON.stringify(plan) }); + await agent._claimRunEntry(tabId, 'interactive', options); + agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Summarize this Gmail thread.' }]; + agent.conversations.set(tabId, messages); + agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: false }); + const originalOptions = { ...options, recommendedAction: { id: 'summarize-thread' } }; + const token = await agent._beginReadCompleteness(tabId, messages[1].content, originalOptions); + assert.ok(agent._readCompletenessBlock(tabId), 'Original thread read is unfinished'); + + steer(agent, 'Cancel the thread summary; open my dashboard instead', 'dashboard'); + const refresh = () => agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, originalOptions); + assert.equal((await refresh()).gate.proceed, true); + assert.equal(agent._readCompletenessBlock(tabId), null, 'Cancelled read must not block the replacement task'); + assert.equal(agent.readCompletenessStates.get(tabId).runToken, token); + + agent._armReadCompletenessFromPlan(tabId, { request_kind: 'execute', read_scope: 'complete_thread' }); + agent._recordReadCompleteness(tabId, 'get_accessibility_tree', { + filter: 'all', maxDepth: 15, maxChars: 12000, ref_id: 'thread', page: 1, + }, { + pageContent: 'Earlier thread', conversationRootRefId: 'thread', conversationExpansionState: 'expanded', + treeRevision: 'earlier-thread', page: 1, hasMore: false, truncated: false, + }); + assert.equal(agent.readCompletenessStates.get(tabId).complete, true, 'Fixture has genuine terminal read coverage'); + plan.read_scope = 'complete_thread'; plan.summary = 'Read the latest reply'; + plan.steps = [{ id: '1', action: 'Read the complete thread', tools: ['get_accessibility_tree'] }]; + steer(agent, 'Read the thread again including the latest reply', 'latest'); + assert.equal((await refresh()).gate.proceed, true); + assert.ok(agent._readCompletenessBlock(tabId), 'Revised full-thread read needs fresh coverage'); + assert.deepEqual(agent.readCompletenessStates.get(tabId).treePages, []); + assert.equal(agent.readCompletenessStates.get(tabId).expansionConfirmed, false); + + pageUrl = 'https://example.com/dashboard'; plan.read_scope = 'none'; + steer(agent, 'Stay on the dashboard', 'stay'); + assert.equal((await refresh()).gate.proceed, true); + assert.equal(agent.readCompletenessStates.get(tabId).communicationThread, false, 'Classification uses the current page'); + agent._clearReadCompleteness(tabId, token); + assert.equal(agent.readCompletenessStates.has(tabId), false, 'Run teardown still owns the revised state'); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + + test(`${build}: prior submissions permit verified recovery controls but block ambiguous or consequential actions`, () => { + const agent = actSetup(Agent); + delete agent._isFormValidationCandidate; + agent.conversations.set(tabId, [{ role: 'user', content: 'Inspect the previous publication' }]); + const guard = agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + guard.steeringPriorSubmission = { dispatched: true, observedAfterSubmit: false }; + const safe = { isSubmit: false, resolvedNonSubmitTarget: true }; + for (const [name, args] of [ + ['click', { text: 'Close' }], ['click_ax', { ref_id: 'cancel' }], + ['iframe_click', { selector: '#keep-editing' }], + ]) { + assert.equal(agent._steeringPriorSubmissionBlock(tabId, name, args, safe), null); + assert.equal(agent._steeringPriorSubmissionBlock(tabId, name, args, null)?.noDispatch, true); + assert.equal(agent._steeringPriorSubmissionBlock(tabId, name, args, { ...safe, isSubmit: true })?.noDispatch, true); + } + assert.equal(agent._steeringPriorSubmissionBlock(tabId, 'fetch_url', { url: 'https://example.com/', method: 'POST' }, safe)?.noDispatch, true); + assert.equal(agent._steeringPriorSubmissionBlock(tabId, 'chrome_web_store_publish', {}, safe)?.noDispatch, true); + }); + + test(`${build}: revalidation retains uncertain submissions and publication attempts`, async () => { + const agent = actSetup(Agent); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Publish the announcement.' }]; + agent.conversations.set(tabId, messages); + const guard = agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true, requiresSubmission: true }); + guard.socialPublication = { outcomes: { announcement: { status: 'pending' } } }; + agent._completionSubmitStates.set(tabId, { dispatched: true, observedAfterSubmit: false }); + steer(agent, 'Use the updated title'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + assert.equal(agent._completionSubmitStates.has(tabId), false); + assert.equal(agent._planExecutionGuards.get(tabId).socialPublication.outcomes.announcement.status, 'pending'); + assert.equal(agent._steeringPriorSubmissionBlock(tabId, 'click', { text: 'Publish' }, { isSubmit: true }).noDispatch, true); + assert.equal(agent._steeringPriorSubmissionBlock(tabId, 'read_page', {}, null), null); + agent._storeContinuationExecutionEvidence(tabId); + const carried = agent._startPlanExecutionGuard(tabId, 'act', { + requiresStateChange: true, requiresSubmission: false, + }, { trustedContinuation: true }); + assert.equal(carried.steeringPriorSubmission.dispatched, true); + assert.equal(carried.successfulConsequentialToolCalls, 0); + const restored = actSetup(Agent); + restored.conversations.set(tabId, serializeConversationForSession(messages, { + preserveMessageIndices: agent._activeTaskBinding(messages).pinnedIndices, + }).messages); + const restoredGuard = restored._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + assert.equal(restoredGuard.steeringPriorSubmission.dispatched, true); + assert.equal(restoredGuard.successfulConsequentialToolCalls, 0); + assert.equal(restored._steeringPriorSubmissionBlock(tabId, 'click', { text: 'Publish' }, { isSubmit: true }).noDispatch, true); + assert.match(restored.conversations.get(tabId)[0].content, /Inspect and report its existing effect/); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + test(`${build}: long steering authority survives actual compaction and bounded recovery`, async () => { + const agent = actSetup(Agent); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Original calendar details '.repeat(5000) }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'Ipek’s corrected event details '.repeat(1000)); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + const original = agent._activeTaskBinding(messages); + messages.push(...Array.from({ length: 70 }, () => ({ role: 'assistant', content: 'Earlier observation '.repeat(1000) }))); + delete agent._manageContext; + await agent._manageContext(tabId, messages, () => {}, null, { force: true }); + assert.equal(agent._activeTaskBinding(messages).text, original.text); + assert.match(messages.find(message => message.content?.includes?.('Context window was trimmed'))?.content || '', /original request and ordered genuine user corrections/); + const restored = serializeConversationForSession(messages, { maxBytes: 450000, preserveMessageIndices: agent._activeTaskBinding(messages).pinnedIndices }); + assert.equal(agent._activeTaskBinding(restored.messages).text, original.text); + messages.push({ role: 'assistant', content: 'Which calendar?', webbrainPlannerClarification: { + taskKey: agent._progressTaskKeyForText(original.text), taskText: original.text.slice(0, 1600), + } }, { role: 'user', content: 'The work calendar' }); + assert.equal(agent._activeTaskBinding(messages).requestText, original.requestText); + steer(agent, 'Use October 8', 'second-correction'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + const clarified = agent._activeTaskBinding(messages); + assert.equal(clarified.requestText, original.requestText); + assert.deepEqual(clarified.updates.slice(-2).map(update => update.text), ['The work calendar', 'Use October 8']); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + test(`${build}: steering invalidates a real plan review and requires review of the latest revision`, async () => { + const agent = actSetup(Agent); + agent._plannerMode = () => 'try'; agent.setPlanReviewSettings({ mode: 'always' }); + agent._chatWithCostAllowance = async () => ({ content: JSON.stringify({ + request_kind: 'execute', requires_state_change: true, requires_submission: true, + read_scope: 'visible_page', summary: 'Add Ipek’s event to the calendar', confidence: 0.99, + steps: [{ id: '1', action: 'Inspect the invitation and save its event', tools: ['read_page', 'click'] }], + memory: { use_scratchpad: true, use_progress_ledger: false }, risks: [], + }) }); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + agent._scratchpadWrite(tabId, { text: 'Existing invitation downloadId=42' }); + steer(agent, 'From Ipek', 'a'); + const reviewed = deferred(), revisedReview = deferred(); + let firstPlanId, secondPlanId; + const refresh = agent._revalidatePendingSteering(tabId, messages, (type, data) => { + if (type === 'plan_review') { + if (!firstPlanId) { firstPlanId = data.planId; reviewed.resolve(); } + else { secondPlanId = data.planId; revisedReview.resolve(); } + } + }, 'act', null, null, options); + await Promise.race([reviewed.promise, refresh.then(result => assert.fail(`Review skipped: ${JSON.stringify(result)}`))]); + steer(agent, 'Use the work calendar', 'b'); + await Promise.race([revisedReview.promise, refresh.then(result => assert.fail(`Revised review skipped: ${JSON.stringify(result)}`))]); + assert.equal(agent.submitPlanResponse(tabId, firstPlanId, 'approve'), false); + assert.equal(agent._steeringRuns.get(tabId).authorizedRevision, 0); + assert.equal(agent.submitPlanResponse(tabId, secondPlanId, 'approve'), true); + assert.equal((await refresh).gate.proceed, true); + assert.equal(agent._steeringRuns.get(tabId).authorizedRevision, 2); + assert.match(agent._planExecutionGuards.get(tabId).taskText, /Ipek.*work calendar/); + assert.match(messages[agent._findScratchpadIndex(messages)].content, /Existing invitation downloadId=42/); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + for (const plannerMode of ['try', 'strict']) { + test(`${build}: ${plannerMode} full planner fails closed on malformed steering output`, async () => { + const agent = actSetup(Agent, { chat: async () => ({ content: 'not structured JSON' }) }); + agent._plannerMode = () => plannerMode; + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'From Ipek'); + const result = await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + assert.equal(result.gate.proceed, false); assert.equal(result.gate.reason, 'planner_error'); + assert.match(result.gate.message, /invalid_output/); assert.doesNotMatch(result.gate.message, /No tools ran|fresh run/); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + + for (const failure of ['transport', 'invalid-output']) { + test(`${build}: real intent revalidation fails closed on ${failure}`, async () => { + const agent = actSetup(Agent, { chat: async () => { + if (failure === 'transport') throw new Error('Provider unavailable for revision'); + return { content: 'not structured JSON' }; + } }); + delete agent._runPlannerIntentGate; + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'From Ipek'); + const result = await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + assert.equal(result.gate.proceed, false); + assert.equal(result.gate.reason, 'planner_error'); + assert.doesNotMatch(result.gate.message, /fresh run|No tools ran/); + assert.match(result.gate.message, failure === 'transport' ? /Provider unavailable/ : /invalid_output/); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + + test(`${build}: Stop during revision planning does not authorize the pending task`, async () => { + const entered = deferred(), release = deferred(); + const agent = actSetup(Agent, {}, async () => { + entered.resolve(); await release.promise; + return { proceed: true, requestKind: 'execute', requiresStateChange: true }; + }); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); const original = agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'From Ipek'); + const revision = agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + await entered.promise; agent.abort(tabId); release.resolve(); + assert.equal((await revision).gate.reason, 'cancelled'); + assert.equal(agent._planExecutionGuards.get(tabId), original); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + for (const kind of ['respond', 'clarify', 'plan_only', 'planner_error']) { + test(`${build}: revised ${kind} intent cannot dispatch the stale task`, async () => { + const agent = actSetup(Agent, {}, async () => kind === 'respond' + ? { proceed: true, responseOnly: true, requestKind: kind } + : { proceed: false, reason: kind, requestKind: kind === 'planner_error' ? 'respond' : kind, message: kind }); + agent._completeResponseOnlyTurn = async () => ({ content: 'Cancelled', status: 'done' }); + agent.executeTool = async () => assert.fail('Stale action dispatched'); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Add the event.' }]; + agent.conversations.set(tabId, messages); agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'Cancel that; just explain the event'); + const outcome = await agent._steeringBoundaryOutcome(tabId, messages, () => {}, 'act', agent._activeProvider(tabId), null, null, options); + assert.equal(outcome.terminal, true); + assert.equal(outcome.content, kind === 'respond' ? 'Cancelled' : kind); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + test(`${build}: steering during a tool call waits for its result and skips remaining calls`, async () => { const agent = setup(Agent); allowBatchPreparation(agent); diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjs index fa948800cf..e93bc7c7ee 100644 --- a/test/fixtures/message-recipient-navigation.mjs +++ b/test/fixtures/message-recipient-navigation.mjs @@ -52,6 +52,201 @@ export function registerMessageRecipientNavigationFixtures({ return { agent, guard, probe }; }; + const setupMessageHistory = async page => { + await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '' })); + await page.goto('https://www.linkedin.com/messaging/thread/example/'); + await setupContentHtml(page, `
+

Ipek

+ +
+

Conversation with Ipek

+ +
`, kind); + await page.evaluate(() => { + document.querySelector('#preview').addEventListener('click', () => { document.querySelector('#viewer').hidden = false; }); + document.querySelector('#close-viewer').addEventListener('click', () => { document.querySelector('#viewer').hidden = true; }); + window.sentMessages = 0; + document.querySelector('#send').addEventListener('click', () => { window.sentMessages++; }); + }); + const agent = new AgentClass({ getActive: () => ({ supportsVision: false }) }); + agent.conversations.set(1, [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Read Ipek’s event invitation.' }]); + agent._startPlanExecutionGuard(1, 'act', { requiresStateChange: false, requiresSubmission: false }); + agent._messageRecipientContentProbe = (_, params) => call(page, 'probe_message_recipient_guard', params); + return { agent, guard: (tool, args) => agent._messageRecipientGuardBlock(1, tool, args, page.url()) }; + }; + + register(`${kind}: steering after a submission permits verified social recovery controls`, async page => { + await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '' })); + for (const [url, label, publishId] of [ + ['https://x.com/compose/post', 'Close', 'tweetButton'], + ['https://bsky.app/', 'Cancel', 'composerPublishBtn'], + ['https://bsky.app/', 'Keep editing', 'composerPublishBtn'], + ]) { + await page.goto(url); + await setupContentHtml(page, `
+
Existing draft
+ + + +
`, kind); + await page.evaluate(() => { + window.recoveryClicks = 0; window.publishClicks = 0; + document.querySelector('#recover').onclick = () => window.recoveryClicks++; + document.querySelector('#publish').onclick = () => window.publishClicks++; + }); + const agent = new AgentClass({ getActive: () => ({ supportsVision: false }) }); + agent.conversations.set(1, [{ role: 'system', content: 'sys' }, { role: 'user', content: 'Inspect the previous publication.' }]); + const guard = agent._startPlanExecutionGuard(1, 'act', { requiresStateChange: true }); + guard.steeringPriorSubmission = { dispatched: true, observedAfterSubmit: false }; + const probe = (tool, args) => page.evaluate(({ source, tool, args }) => + Function('return (' + source + ')')()(tool, args), { source: AgentClass._submitActionProbe.toString(), tool, args }); + const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 }); + const ref = new RegExp(`button "${label}" \\[([^\\]]+)\\]`).exec(tree.pageContent)?.[1]; + assert.ok(ref, tree.pageContent); + const point = await page.locator('#recover').evaluate(el => { + const rect = el.getBoundingClientRect(); return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; + }); + for (const [tool, args] of [ + ['click', { selector: '#recover-label' }], ['click', { text: label, textMatch: 'exact' }], + ['click_ax', { ref_id: ref }], ['click', point], + ]) { + const detected = await probe(tool, args); + assert.equal(detected?.resolvedNonSubmitTarget, true, `${label}/${tool}: real target is safe`); + assert.equal(agent._steeringPriorSubmissionBlock(1, tool, args, detected), null, `${label}/${tool}`); + assert.equal((await call(page, tool, args)).success, true); + } + for (const selector of ['#publish', '#unknown', '#missing']) { + const args = { selector }; + assert.equal(agent._steeringPriorSubmissionBlock(1, 'click', args, await probe('click', args))?.noDispatch, true, selector); + } + assert.deepEqual(await page.evaluate(() => [window.recoveryClicks, window.publishClicks]), [4, 0]); + } + }); + + register(`${kind}: LinkedIn history previews and native conversation rows are not sends`, async page => { + const { guard } = await setupMessageHistory(page); + const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 }); + const previewRef = /button "Click or press enter to display in the image preview" \[([^\]]+)\]/.exec(tree.pageContent)?.[1]; + const sendRef = /button "Send" \[([^\]]+)\]/.exec(tree.pageContent)?.[1]; + assert.ok(previewRef, tree.pageContent); + assert.ok(sendRef, tree.pageContent); + const rect = await page.locator('#preview').evaluate(el => { const r=el.getBoundingClientRect();return {x:r.x+r.width/2,y:r.y+r.height/2}; }); + for (const [tool, args] of [ + ['click', { selector: '#preview' }], ['click', { text: 'Click or press enter to display in the image preview' }], + ['click_ax', { ref_id: previewRef }], ['click', rect], ['click', { selector: '#preview-image' }], + ['click', { selector: '#row' }], ['click', { selector: '#row-label' }], ['click', { text: 'Ipek conversation' }], + ]) assert.equal(await guard(tool, args), null, JSON.stringify({tool,args})); + for (const selector of ['#row-options', '#row-options-label', '#send']) { + assert.equal((await guard('click', { selector }))?.noDispatch, true, selector); + } + const sendPoint = await page.locator('#send').evaluate(el => {const r=el.getBoundingClientRect();return {x:r.x+r.width/2,y:r.y+r.height/2};}); + for (const [tool,args] of [['click',{text:'Send'}],['click_ax',{ref_id:sendRef}],['click',sendPoint]]) { + assert.equal((await guard(tool,args))?.noDispatch,true,JSON.stringify({tool,args})); + } + await page.locator('#row-options').evaluate(el => {el.style='position:absolute;left:0;top:0;width:270px;height:90px';}); + assert.equal((await guard('click',{selector:'#row'}))?.noDispatch,true,'row center over nested action'); + await page.locator('textarea').focus(); + assert.equal((await guard('press_keys', { key: 'Enter' }))?.noDispatch, true); + assert.equal(await page.evaluate(() => window.sentMessages), 0); + }); + + register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => { + const { guard } = await setupMessageHistory(page); + for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text']) { + await page.evaluate(change => { + const button=document.querySelector('#preview'); + window.originalPreview=button.cloneNode(true); + if(change==='no-image')button.querySelector('img').remove(); + if(change==='composer')document.querySelector('#composer').append(button); + if(change==='hidden')button.hidden=true; + if(change==='transparent')button.style.opacity='0'; + if(change==='ancestor-transparent')button.parentElement.style.opacity='0'; + if(change==='disabled')button.disabled=true; + if(change==='commit-label')button.title='Send'; + if(change==='commit-text'){button.title='View image';button.append('Send');} + if(change==='occluded') { + const cover=document.createElement('div');cover.id='cover'; + cover.style='position:fixed;left:350px;top:80px;width:400px;height:340px;z-index:20;background:white';document.body.append(cover); + } + }, change); + assert.equal((await guard('click', { selector:'#preview' }))?.noDispatch, true, change); + await page.evaluate(() => { + document.querySelector('#preview').remove(); document.querySelector('.msg-s-event-listitem').append(window.originalPreview); + document.querySelector('.msg-s-event-listitem').style.opacity=''; + document.querySelector('#cover')?.remove(); + }); + } + await page.evaluate(() => {document.querySelector('#viewer').hidden=false;}); + assert.equal((await guard('click', {selector:'#preview'}))?.noDispatch,true,'background image behind modal'); + }); + + register(`${kind}: Ipek steering replays a message invitation into exactly one calendar event`, async page => { + const { agent, guard } = await setupMessageHistory(page); + agent._persist = () => {}; agent._persistSubmittedTurn = async () => {}; + agent._getTabUrlTitle = async () => ({tabUrl:page.url(),tabTitle:'Messaging'}); + agent._ensureProgressSessionForCurrentTask = async () => {}; + agent._plannerMode = () => 'off'; + agent._runPlannerIntentGate = async (_tab,enriched) => { + assert.match(enriched.content,/Add this event to my Google Calendar/); assert.match(enriched.content,/Ipek/); + return {proceed:true,requestKind:'execute',requiresStateChange:true,requiresSubmission:true}; + }; + agent.conversations.set(1,[{role:'system',content:'sys'},{role:'user',content:'Add this event to my Google Calendar.'}]); + agent._startPlanExecutionGuard(1,'act',{requiresStateChange:true,requiresSubmission:true}); + agent._beginSteeringRun(1,()=>{},{detachedRequestId:'replay'}); + agent._runningTabs.add(1); + assert.equal(agent.steerMessage(1,'Use Ipek’s message',{requestId:'replay',messageId:'ipek'}).accepted,true); + const refresh=await agent._revalidatePendingSteering(1,agent.conversations.get(1),()=>{},'act',null,null,{}); + assert.equal(refresh.gate.proceed,true); + await page.evaluate(() => { + const recent=document.createElement('div');recent.style.height='500px';recent.textContent='Emre: a recent photo'; + document.querySelector('#history').append(recent); + document.querySelector('#history').scrollTop=10000; + }); + assert.equal((await guard('click',{selector:'#preview'}))?.noDispatch,true,'earlier invitation must be brought into view'); + const scrolled=await call(page,'scroll',{direction:'up',amount:700,x:550,y:250}); + assert.equal(scrolled.success,true,JSON.stringify(scrolled)); + assert.equal(await guard('click',{selector:'#preview'}),null); + assert.equal((await call(page,'click',{selector:'#preview'})).success,true); + const invitation=await call(page,'get_page_info_cdp',{}); + assert.match(JSON.stringify(invitation),/Browser workshop/); + assert.match(JSON.stringify(invitation),/October 8, 2026/); + assert.match(JSON.stringify(invitation),/Europe\/Istanbul/); + await page.goto('https://calendar.google.com/calendar/u/0/r'); + await setupContentHtml(page, `
+ + +
`,kind); + await page.evaluate(() => { + window.savedEvents=[]; + document.querySelector('#save').addEventListener('click',()=>{ + window.savedEvents.push(Object.fromEntries(['title','date','start','end','zone'].map(key=>[key,document.getElementById(key).value]))); + document.querySelector('#confirmation').textContent='Event saved'; + }); + }); + for(const [key,text] of Object.entries({title:'Browser workshop',date:'2026-10-08',start:'14:00',end:'15:00',zone:'Europe/Istanbul'})) { + assert.equal((await call(page,'type',{selector:'#'+key,text,clear:true})).success,true); + } + assert.equal((await call(page,'click',{selector:'#save'})).success,true); + assert.deepEqual(await page.evaluate(()=>window.savedEvents),[{title:'Browser workshop',date:'2026-10-08',start:'14:00',end:'15:00',zone:'Europe/Istanbul'}]); + assert.match(JSON.stringify(await call(page,'get_page_info_cdp',{})),/Event saved/); + assert.equal(agent._planExecutionGuards.get(1).taskDrifted,false); + agent._finishSteeringRun(1);agent._runningTabs.delete(1); + }); + register(`${kind}: X sent DM completes with the open composer and exact outgoing delivery evidence`, async (page) => { const { agent } = await setup(page); const url = 'https://x.com/i/chat/123-456'; From b07e6828adb6abb52802943cfec6c4e802c7a778 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 19:20:09 +0300 Subject: [PATCH 2/6] Preserve steering through emergency trims and initial responses --- src/chrome/src/agent/agent.js | 131 ++++++++++++++++++++++++--------- src/firefox/src/agent/agent.js | 131 ++++++++++++++++++++++++--------- test/chat-steering.mjs | 98 ++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 70 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index bf3eb083c4..d40e567de0 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -6264,6 +6264,27 @@ export class Agent extends LoopDetector { } } + async _completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gate, enriched, priorMessages) { + let revalidated = false; + while (gate.proceed && gate.responseOnly) { + const capture = {}; + const response = await this._completeResponseOnlyTurn( + tabId, messages, onUpdate, provider, costState, runId, runOptions, enriched, priorMessages, capture, + ); + if (!this._hasPendingSteering(tabId)) return { gate, response, capture, revalidated }; + onUpdate('text', { content: '', replace: true }); + const refresh = await this._revalidatePendingSteering( + tabId, messages, onUpdate, mode, costState, runId, runOptions, + ); + if (refresh?.gate) { + gate = refresh.gate; + revalidated = true; + } + enriched = refresh?.enriched || { role: 'user', content: this._activeTaskBinding(messages).text }; + } + return { gate, revalidated }; + } + activeRunState(tabId) { const persistenceState = this.persistenceDegradedTabs.get(tabId) || null; const state = { @@ -22837,6 +22858,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const stopped = this._consumeContextOnlyAbort(tabId, messages, onUpdate); if (stopped) return stopped; + // An accepted correction supersedes this answer before it reaches the + // transcript, UI, persistence, or voluntary generation sharing. + if (this._hasPendingSteering(tabId)) return { content: '', status: 'steering_superseded' }; if (!finalResponse) { status = 'empty_output'; finalResponse = 'I could not generate a usable response from the existing conversation context.'; @@ -32671,6 +32695,19 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai _emergencyTrimModelCopy(messages) { const modelCopy = messages.map(message => ({ ...message })); this._emergencyTrim(modelCopy); + // Linked corrections hash their parent task, so even wrapped source in + // that authority must stay verbatim across further overflow retries. + if (this._activeTaskBinding(messages).pinnedIndices.length > 1) return modelCopy; + // A selected-text request can itself contain the oversized source. + // Trim only wrapped page data in this disposable model view, retaining + // the surrounding user instructions and the complete saved transcript. + for (const message of modelCopy) { + if (message.role !== 'user' || typeof message.content !== 'string') continue; + message.content = message.content.replace( + /]*>[\s\S]*?<\/untrusted_page_content\b[^>]*>/g, + wrapper => wrapper.length > 5000 ? this._truncatePreservingUntrustedWrapper(wrapper, 5000) : wrapper, + ); + } return modelCopy; } @@ -32961,6 +32998,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai originalTask = m; break; } + const taskBinding = this._activeTaskBinding(messages); + // Authorized corrections must survive even outside the recent window, + // without the truncation applied to intermediate tool results. + const pinnedTasks = taskBinding.pinnedIndices.map(index => messages[index]).filter(Boolean); + if (originalTask && !pinnedTasks.includes(originalTask)) pinnedTasks.unshift(originalTask); + const pinnedTaskSet = new Set(pinnedTasks); const scheduledResumeIdx = this._findLatestScheduledResumeIndex(messages); const scheduledResumeMsg = scheduledResumeIdx >= 0 && messages[scheduledResumeIdx] !== originalTask ? messages[scheduledResumeIdx] @@ -32974,7 +33017,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai const progressIdx = this._findProgressLedgerIndex(messages); const progressMsg = progressIdx >= 0 ? messages[progressIdx] : null; const keepLast = 6; // keep only 6 most recent messages - const recent = messages.slice(-keepLast).filter(m => m !== scheduledResumeMsg && !this._isScheduledResumeTurn(m.content) && !this._isPinnedAgentStateMessage(m)); + const recent = messages.slice(-keepLast).filter(m => !pinnedTaskSet.has(m) && m !== scheduledResumeMsg && !this._isScheduledResumeTurn(m.content) && !this._isPinnedAgentStateMessage(m)); // Drop any leading `tool` messages whose requesting assistant turn fell // outside the kept window. Both OpenAI-compatible and Anthropic APIs reject @@ -33001,16 +33044,16 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai const notice = { role: 'user', - content: '[Context was too large for the model. Older intermediate steps were removed, but your ORIGINAL TASK is pinned above — keep working on it based on the most recent state you can see.]', + content: '[Context was too large for the model. Older intermediate steps were removed, but the active request and ordered genuine user corrections are pinned above. Apply the latest corrections and keep working from the most recent state.]', }; const ack = { role: 'assistant', - content: 'Understood. I\'ll continue the original task with the recent state.', + content: 'Understood. I\'ll continue the active request with its latest corrections and recent state.', }; messages.length = 0; messages.push(systemMsg); - if (originalTask) messages.push(originalTask); + messages.push(...pinnedTasks); if (scheduledResumeMsg) messages.push(scheduledResumeMsg); if (scratchpadMsg) messages.push(scratchpadMsg); if (memoryMsg) messages.push(memoryMsg); @@ -43233,7 +43276,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai ); } - const gateOutcome = await this._maybeRunPlannerGate( + let gateOutcome = await this._maybeRunPlannerGate( tabId, messages, enriched, onUpdate, mode, costState, runId, plannerTabInfo, runOptions, ); if (!gateOutcome.proceed) { @@ -43250,28 +43293,37 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); + let initialResponseRevalidated = false; if (gateOutcome.responseOnly === true) { - const responseOnlyShareCapture = {}; - const responseOnly = await this._completeResponseOnlyTurn( - tabId, messages, onUpdate, provider, costState, runId, - runOptions, enriched, sourceBoundPriorMessages, responseOnlyShareCapture, + const responseOutcome = await this._completeInitialResponseOnlyTurn( + tabId, messages, onUpdate, mode, provider, costState, runId, + runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); - finalResponse = responseOnly.content; - _traceStatus = responseOnly.status; - // Response-only turns do call the model (source-grounded + pruned - // inside _generateContextOnlyResponse). Only mark shareable on success, - // keeping the exact request (with the context-only system prompt). - if (responseOnly.status === 'done') { - shareHadProviderCompletion = true; - shareRawResponse = responseOnlyShareCapture.response; - if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { - currentNonStreamRequestMessages = responseOnlyShareCapture.request; + gateOutcome = responseOutcome.gate; + initialResponseRevalidated = responseOutcome.revalidated; + if (!gateOutcome.proceed) { + _traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; + return (finalResponse = gateOutcome.message || 'More information is required.'); + } + const { response: responseOnly, capture: responseOnlyShareCapture } = responseOutcome; + if (responseOnly) { + finalResponse = responseOnly.content; + _traceStatus = responseOnly.status; + // Response-only turns do call the model (source-grounded + pruned + // inside _generateContextOnlyResponse). Only mark shareable on success, + // keeping the exact request (with the context-only system prompt). + if (responseOnly.status === 'done') { + shareHadProviderCompletion = true; + shareRawResponse = responseOnlyShareCapture.response; + if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { + currentNonStreamRequestMessages = responseOnlyShareCapture.request; + } } + return finalResponse; } - return finalResponse; } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -43477,7 +43529,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai this._appendStandaloneUngroundedPolicy(enriched, standaloneGroundingGap); } - const recommendedFirstTool = await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { @@ -44581,7 +44633,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai ); } - const gateOutcome = await this._maybeRunPlannerGate( + let gateOutcome = await this._maybeRunPlannerGate( tabId, messages, enriched, onUpdate, mode, costState, runId, plannerTabInfo, runOptions, ); if (!gateOutcome.proceed) { @@ -44598,23 +44650,32 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); + let initialResponseRevalidated = false; if (gateOutcome.responseOnly === true) { - const responseOnlyShareCapture = {}; - const responseOnly = await this._completeResponseOnlyTurn( - tabId, messages, onUpdate, provider, costState, runId, - runOptions, enriched, sourceBoundPriorMessages, responseOnlyShareCapture, + const responseOutcome = await this._completeInitialResponseOnlyTurn( + tabId, messages, onUpdate, mode, provider, costState, runId, + runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); - if (responseOnly.status === 'done') { - shareHadProviderCompletion = true; - shareRawResponse = responseOnlyShareCapture.response; - if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { - currentStreamRequestMessages = responseOnlyShareCapture.request; + gateOutcome = responseOutcome.gate; + initialResponseRevalidated = responseOutcome.revalidated; + if (!gateOutcome.proceed) { + const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; + return finish(gateOutcome.message || 'More information is required.', status); + } + const { response: responseOnly, capture: responseOnlyShareCapture } = responseOutcome; + if (responseOnly) { + if (responseOnly.status === 'done') { + shareHadProviderCompletion = true; + shareRawResponse = responseOnlyShareCapture.response; + if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { + currentStreamRequestMessages = responseOnlyShareCapture.request; + } } + return finish(responseOnly.content, responseOnly.status); } - return finish(responseOnly.content, responseOnly.status); } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); let standaloneGroundingGap = this._standaloneOfflineGroundingGap(localWikipediaRag, runOptions); if (standaloneGroundingGap) { @@ -44678,7 +44739,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai // currentStreamRequestMessages is declared at the top of this function so // pre-loop exits (e.g. response-only turns) can capture their request. - const recommendedFirstTool = await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 460176a968..9704135f86 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -5978,6 +5978,27 @@ export class Agent extends LoopDetector { } } + async _completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gate, enriched, priorMessages) { + let revalidated = false; + while (gate.proceed && gate.responseOnly) { + const capture = {}; + const response = await this._completeResponseOnlyTurn( + tabId, messages, onUpdate, provider, costState, runId, runOptions, enriched, priorMessages, capture, + ); + if (!this._hasPendingSteering(tabId)) return { gate, response, capture, revalidated }; + onUpdate('text', { content: '', replace: true }); + const refresh = await this._revalidatePendingSteering( + tabId, messages, onUpdate, mode, costState, runId, runOptions, + ); + if (refresh?.gate) { + gate = refresh.gate; + revalidated = true; + } + enriched = refresh?.enriched || { role: 'user', content: this._activeTaskBinding(messages).text }; + } + return { gate, revalidated }; + } + activeRunState(tabId) { const persistenceState = this.persistenceDegradedTabs.get(tabId) || null; const state = { @@ -20633,6 +20654,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const stopped = this._consumeContextOnlyAbort(tabId, messages, onUpdate); if (stopped) return stopped; + // An accepted correction supersedes this answer before it reaches the + // transcript, UI, persistence, or voluntary generation sharing. + if (this._hasPendingSteering(tabId)) return { content: '', status: 'steering_superseded' }; if (!finalResponse) { status = 'empty_output'; finalResponse = 'I could not generate a usable response from the existing conversation context.'; @@ -30269,6 +30293,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d _emergencyTrimModelCopy(messages) { const modelCopy = messages.map(message => ({ ...message })); this._emergencyTrim(modelCopy); + // Linked corrections hash their parent task, so even wrapped source in + // that authority must stay verbatim across further overflow retries. + if (this._activeTaskBinding(messages).pinnedIndices.length > 1) return modelCopy; + // A selected-text request can itself contain the oversized source. + // Trim only wrapped page data in this disposable model view, retaining + // the surrounding user instructions and the complete saved transcript. + for (const message of modelCopy) { + if (message.role !== 'user' || typeof message.content !== 'string') continue; + message.content = message.content.replace( + /]*>[\s\S]*?<\/untrusted_page_content\b[^>]*>/g, + wrapper => wrapper.length > 5000 ? this._truncatePreservingUntrustedWrapper(wrapper, 5000) : wrapper, + ); + } return modelCopy; } @@ -30558,6 +30595,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d originalTask = m; break; } + const taskBinding = this._activeTaskBinding(messages); + // Authorized corrections must survive even outside the recent window, + // without the truncation applied to intermediate tool results. + const pinnedTasks = taskBinding.pinnedIndices.map(index => messages[index]).filter(Boolean); + if (originalTask && !pinnedTasks.includes(originalTask)) pinnedTasks.unshift(originalTask); + const pinnedTaskSet = new Set(pinnedTasks); const scheduledResumeIdx = this._findLatestScheduledResumeIndex(messages); const scheduledResumeMsg = scheduledResumeIdx >= 0 && messages[scheduledResumeIdx] !== originalTask ? messages[scheduledResumeIdx] @@ -30571,7 +30614,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const progressIdx = this._findProgressLedgerIndex(messages); const progressMsg = progressIdx >= 0 ? messages[progressIdx] : null; const keepLast = 6; // keep only 6 most recent messages - const recent = messages.slice(-keepLast).filter(m => m !== scheduledResumeMsg && !this._isScheduledResumeTurn(m.content) && !this._isPinnedAgentStateMessage(m)); + const recent = messages.slice(-keepLast).filter(m => !pinnedTaskSet.has(m) && m !== scheduledResumeMsg && !this._isScheduledResumeTurn(m.content) && !this._isPinnedAgentStateMessage(m)); // Drop any leading `tool` messages whose requesting assistant turn fell // outside the kept window. Both OpenAI-compatible and Anthropic APIs reject @@ -30598,16 +30641,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const notice = { role: 'user', - content: '[Context was too large for the model. Older intermediate steps were removed, but your ORIGINAL TASK is pinned above — keep working on it based on the most recent state you can see.]', + content: '[Context was too large for the model. Older intermediate steps were removed, but the active request and ordered genuine user corrections are pinned above. Apply the latest corrections and keep working from the most recent state.]', }; const ack = { role: 'assistant', - content: 'Understood, some earlier context was trimmed. I\'ll continue with what I have.', + content: 'Understood. I\'ll continue the active request with its latest corrections and recent state.', }; messages.length = 0; messages.push(systemMsg); - if (originalTask) messages.push(originalTask); + messages.push(...pinnedTasks); if (scheduledResumeMsg) messages.push(scheduledResumeMsg); if (scratchpadMsg) messages.push(scratchpadMsg); if (memoryMsg) messages.push(memoryMsg); @@ -36197,7 +36240,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); } - const gateOutcome = await this._maybeRunPlannerGate( + let gateOutcome = await this._maybeRunPlannerGate( tabId, messages, enriched, onUpdate, mode, costState, runId, plannerTabInfo, runOptions, ); if (!gateOutcome.proceed) { @@ -36214,28 +36257,37 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); + let initialResponseRevalidated = false; if (gateOutcome.responseOnly === true) { - const responseOnlyShareCapture = {}; - const responseOnly = await this._completeResponseOnlyTurn( - tabId, messages, onUpdate, provider, costState, runId, - runOptions, enriched, sourceBoundPriorMessages, responseOnlyShareCapture, + const responseOutcome = await this._completeInitialResponseOnlyTurn( + tabId, messages, onUpdate, mode, provider, costState, runId, + runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); - finalResponse = responseOnly.content; - _traceStatus = responseOnly.status; - // Response-only turns do call the model (source-grounded + pruned - // inside _generateContextOnlyResponse). Only mark shareable on success, - // keeping the exact request (with the context-only system prompt). - if (responseOnly.status === 'done') { - shareHadProviderCompletion = true; - shareRawResponse = responseOnlyShareCapture.response; - if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { - currentNonStreamRequestMessages = responseOnlyShareCapture.request; + gateOutcome = responseOutcome.gate; + initialResponseRevalidated = responseOutcome.revalidated; + if (!gateOutcome.proceed) { + _traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; + return (finalResponse = gateOutcome.message || 'More information is required.'); + } + const { response: responseOnly, capture: responseOnlyShareCapture } = responseOutcome; + if (responseOnly) { + finalResponse = responseOnly.content; + _traceStatus = responseOnly.status; + // Response-only turns do call the model (source-grounded + pruned + // inside _generateContextOnlyResponse). Only mark shareable on success, + // keeping the exact request (with the context-only system prompt). + if (responseOnly.status === 'done') { + shareHadProviderCompletion = true; + shareRawResponse = responseOnlyShareCapture.response; + if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { + currentNonStreamRequestMessages = responseOnlyShareCapture.request; + } } + return finalResponse; } - return finalResponse; } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -36430,7 +36482,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); } - const recommendedFirstTool = await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { @@ -37376,7 +37428,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); } - const gateOutcome = await this._maybeRunPlannerGate( + let gateOutcome = await this._maybeRunPlannerGate( tabId, messages, enriched, onUpdate, mode, costState, runId, plannerTabInfo, runOptions, ); if (!gateOutcome.proceed) { @@ -37393,23 +37445,32 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); + let initialResponseRevalidated = false; if (gateOutcome.responseOnly === true) { - const responseOnlyShareCapture = {}; - const responseOnly = await this._completeResponseOnlyTurn( - tabId, messages, onUpdate, provider, costState, runId, - runOptions, enriched, sourceBoundPriorMessages, responseOnlyShareCapture, + const responseOutcome = await this._completeInitialResponseOnlyTurn( + tabId, messages, onUpdate, mode, provider, costState, runId, + runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); - if (responseOnly.status === 'done') { - shareHadProviderCompletion = true; - shareRawResponse = responseOnlyShareCapture.response; - if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { - currentStreamRequestMessages = responseOnlyShareCapture.request; + gateOutcome = responseOutcome.gate; + initialResponseRevalidated = responseOutcome.revalidated; + if (!gateOutcome.proceed) { + const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; + return finish(gateOutcome.message || 'More information is required.', status); + } + const { response: responseOnly, capture: responseOnlyShareCapture } = responseOutcome; + if (responseOnly) { + if (responseOnly.status === 'done') { + shareHadProviderCompletion = true; + shareRawResponse = responseOnlyShareCapture.response; + if (Array.isArray(responseOnlyShareCapture.request) && responseOnlyShareCapture.request.length) { + currentStreamRequestMessages = responseOnlyShareCapture.request; + } } + return finish(responseOnly.content, responseOnly.status); } - return finish(responseOnly.content, responseOnly.status); } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -37460,7 +37521,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // currentStreamRequestMessages is declared at the top of this function so // pre-loop exits (e.g. response-only turns) can capture their request. - const recommendedFirstTool = await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index 7cfe9f8a68..cb376a124d 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -116,6 +116,60 @@ for (const build of ['chrome', 'firefox']) { }); for (const streaming of [false, true]) { + for (const mode of ['act', 'dev']) { + for (const revisedKind of ['respond', 'execute']) { + test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} initial response-only steering switches to ${revisedKind}`, async () => { + const entered = deferred(), release = deferred(), updates = [], persisted = [], dispatched = []; + let modelCalls = 0, plannerCalls = 0; + const next = async () => { + if (++modelCalls === 1) { + entered.resolve(); await release.promise; + return { content: 'Obsolete initial answer' }; + } + if (revisedKind === 'respond') return { content: 'Revised answer' }; + if (modelCalls === 2) return { toolCalls: [{ id: 'dashboard', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }] }; + return { toolCalls: [{ id: 'finished', function: { name: 'done', arguments: '{"summary":"Dashboard opened","outcome":"success"}' } }] }; + }; + const agent = actSetup(Agent, { + chat: next, + async *chatStream() { + const result = await next(); + if (result.content) yield { type: 'text', content: result.content }; + if (result.toolCalls) yield { type: 'tool_call', content: result.toolCalls }; + yield { type: 'done' }; + }, + }, async (_tab, enriched) => { + if (++plannerCalls > 1) assert.match(enriched.content, /Use the revised request/); + return plannerCalls > 1 && revisedKind === 'execute' + ? { proceed: true, requestKind: 'execute', requiresStateChange: true, requiresSubmission: false } + : { proceed: true, responseOnly: true, requestKind: 'respond', requiresStateChange: false, requiresSubmission: false }; + }); + agent._persist = () => persisted.push(structuredClone(agent.conversations.get(tabId) || [])); + if (revisedKind === 'execute') agent._maybeExecuteRecommendedActionFirstTool = async () => assert.fail('Superseded initial recommendations must not run'); + agent.executeTool = async (_tab, name, args) => { + if (name === 'done') { + assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null); + return { done: true, summary: args.summary, outcome: args.outcome }; + } + dispatched.push(args.url); return { success: true, url: args.url }; + }; + const update = (type, data) => updates.push({ type, data }); + const run = streaming + ? agent.processMessageStream(tabId, 'Explain the event', update, mode, options) + : agent.processMessage(tabId, 'Explain the event', update, mode, [], options); + await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]); + assert.equal(steer(agent, 'Use the revised request').accepted, true); + release.resolve(); + assert.equal(await run, revisedKind === 'respond' ? 'Revised answer' : 'Dashboard opened'); + assert.equal(plannerCalls, 2); + assert.equal(updates.some(update => update.type === 'steering_queued'), false); + assert.equal(updates.some(update => update.type === 'text' && update.data.content === 'Obsolete initial answer'), false); + assert.equal(persisted.some(messages => messages.some(message => message.role === 'assistant' && message.content === 'Obsolete initial answer')), false); + assert.deepEqual(dispatched, revisedKind === 'execute' ? ['https://example.com/dashboard'] : []); + }); + } + } + test(`${build}: ${streaming ? 'stream' : 'chat'} reauthorizes Ipek steering with the real Act guard`, async () => { const entered = deferred(), release = deferred(); const plans = [], dispatched = [], requests = []; @@ -472,6 +526,50 @@ for (const build of ['chrome', 'firefox']) { agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); }); + test(`${build}: emergency trims preserve authorized steering and attempt receipts verbatim`, async () => { + const agent = actSetup(Agent); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const task = `Add the event to my calendar.\n${agent._wrapUntrusted('read_page', 'Invitation source '.repeat(500))}`; + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: task }]; + agent.conversations.set(tabId, messages); + const guard = agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + guard.steeringPriorSubmission = { dispatched: true, observedAfterSubmit: false }; + steer(agent, 'Use Ipek’s corrected details '.repeat(400), 'early'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + for (let i = 0; i < 8; i++) { + messages.push({ role: 'assistant', tool_calls: [{ id: `read-${i}`, function: { name: 'read_page', arguments: '{}' } }] }, + { role: 'tool', tool_call_id: `read-${i}`, content: agent._wrapUntrusted('read_page', 'Earlier page observation '.repeat(300)) }); + } + steer(agent, 'Use the work calendar '.repeat(400), 'recent'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + const binding = agent._activeTaskBinding(messages); + const pinned = binding.pinnedIndices.map(index => structuredClone(messages[index])); + const before = structuredClone(messages); + const modelCopy = agent._emergencyTrimModelCopy(messages); + assert.deepEqual(messages, before, 'Model-copy trim does not mutate the transcript'); + agent._emergencyTrim(messages); + for (const trimmed of [messages, modelCopy]) { + assert.equal(agent._activeTaskBinding(trimmed).text, binding.text); + assert.equal(agent._activeTaskBinding(trimmed).priorSubmission.dispatched, true); + for (const message of pinned) assert.equal(trimmed.filter(candidate => JSON.stringify(candidate) === JSON.stringify(message)).length, 1); + assertPairedTools(trimmed); + } + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + + test(`${build}: emergency model copies trim source data while preserving surrounding instructions`, () => { + const agent = setup(Agent); + const source = agent._wrapUntrusted('read_page', `${'Selected source '.repeat(600)}SOURCE_TAIL`); + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content: `Summarize only this source.\n${source}\nUse the specified output format.` }]; + const copy = agent._emergencyTrimModelCopy(messages); + const request = copy.find(message => message.role === 'user' && message.content.startsWith('Summarize only this source.')); + assert.ok(request); + assert.equal(agent._hasUntrustedWrapper(request.content), true); + assert.doesNotMatch(request.content, /SOURCE_TAIL/); + assert.ok(request.content.endsWith('Use the specified output format.')); + assert.match(messages[1].content, /SOURCE_TAIL/); + }); + test(`${build}: steering invalidates a real plan review and requires review of the latest revision`, async () => { const agent = actSetup(Agent); agent._plannerMode = () => 'try'; agent.setPlanReviewSettings({ mode: 'always' }); From 74219e56e5ba5eed189ae45bd51295b9becaa451 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 19:35:16 +0300 Subject: [PATCH 3/6] Preserve recovery task hashes and recognize localized LinkedIn previews --- .../src/agent/conversation-persistence.js | 3 ++ src/chrome/src/content/content.js | 22 +++++++---- .../src/agent/conversation-persistence.js | 3 ++ src/firefox/src/content/content.js | 22 +++++++---- test/chat-steering.mjs | 37 +++++++++++++++++++ .../fixtures/message-recipient-navigation.mjs | 33 ++++++++++++++++- 6 files changed, 103 insertions(+), 17 deletions(-) diff --git a/src/chrome/src/agent/conversation-persistence.js b/src/chrome/src/agent/conversation-persistence.js index 2477a8b392..d99033e872 100644 --- a/src/chrome/src/agent/conversation-persistence.js +++ b/src/chrome/src/agent/conversation-persistence.js @@ -8,6 +8,9 @@ function byteLength(value) { } function capText(value, maxChars, marker, state) { + // Authority-bearing text must retain the exact parent-task hash. Its data + // URLs are part of that text; quota handling may fail, but cannot rewrite it. + if (maxChars === Infinity) return String(value || ''); const sanitized = String(value || '').replace(DATA_URL_RE, () => { state.compacted = true; return '[embedded binary data omitted from session recovery]'; diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index a1ba69ccf8..e9fe24e0bd 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -5863,8 +5863,9 @@ const verifiedLinkedInImagePreview = clicked => { if (params.adapterName !== 'linkedin') return false; const button = _composedClosestElement(clicked, 'button,[role="button"]'); - if (!button || compact(button.getAttribute('aria-label')).toLowerCase() - !== 'click or press enter to display in the image preview') return false; + if (!button) return false; + // Reject other controls before preview-specific style and hit probes. + if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; const painted = el => { if (!visible(el) || !_hasVisibleBox(el)) return false; for (let node = el; node?.nodeType === 1; node = _composedParent(node)) { @@ -5876,18 +5877,23 @@ || button.getAttribute('aria-disabled') === 'true' || String(button.getAttribute('type') || '').toLowerCase() !== 'button' || button.form || button.hasAttribute('form') || button.hasAttribute('download') + || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '') || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; // Attachment previews live in history, not in the composer. Proximity // to the composer cannot turn this verified viewer into a send; labels // alone cannot exempt a message commit or an unrelated image button. - if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; - if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] + if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] .some(_hasMessageCommitName)) return false; - if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false; const rect = button.getBoundingClientRect(); - return _isComposedAncestor(button, _shadowAwareElementFromPoint( - rect.left + rect.width / 2, rect.top + rect.height / 2, - )); + const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); + // LinkedIn localizes the preview label. Verify the attachment itself + // at the dispatch point instead; small avatar/reaction/menu icons do + // not prove that a button opens an image viewer. + return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => { + const imageRect = image.getBoundingClientRect(); + return painted(image) && imageRect.width >= 64 && imageRect.height >= 64 + && _isComposedAncestor(image, hit); + }); }; const classifyLinkedInNavigation = (clicked, blockingModal = null) => { diff --git a/src/firefox/src/agent/conversation-persistence.js b/src/firefox/src/agent/conversation-persistence.js index 2477a8b392..d99033e872 100644 --- a/src/firefox/src/agent/conversation-persistence.js +++ b/src/firefox/src/agent/conversation-persistence.js @@ -8,6 +8,9 @@ function byteLength(value) { } function capText(value, maxChars, marker, state) { + // Authority-bearing text must retain the exact parent-task hash. Its data + // URLs are part of that text; quota handling may fail, but cannot rewrite it. + if (maxChars === Infinity) return String(value || ''); const sanitized = String(value || '').replace(DATA_URL_RE, () => { state.compacted = true; return '[embedded binary data omitted from session recovery]'; diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 799746d646..38a0e2216f 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -4951,8 +4951,9 @@ const verifiedLinkedInImagePreview = clicked => { if (params.adapterName !== 'linkedin') return false; const button = _composedClosestElement(clicked, 'button,[role="button"]'); - if (!button || compact(button.getAttribute('aria-label')).toLowerCase() - !== 'click or press enter to display in the image preview') return false; + if (!button) return false; + // Reject other controls before preview-specific style and hit probes. + if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; const painted = el => { if (!visible(el) || !_hasVisibleBox(el)) return false; for (let node = el; node?.nodeType === 1; node = _composedParent(node)) { @@ -4964,18 +4965,23 @@ || button.getAttribute('aria-disabled') === 'true' || String(button.getAttribute('type') || '').toLowerCase() !== 'button' || button.form || button.hasAttribute('form') || button.hasAttribute('download') + || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '') || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; // Attachment previews live in history, not in the composer. Proximity // to the composer cannot turn this verified viewer into a send; labels // alone cannot exempt a message commit or an unrelated image button. - if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false; - if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] + if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent] .some(_hasMessageCommitName)) return false; - if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false; const rect = button.getBoundingClientRect(); - return _isComposedAncestor(button, _shadowAwareElementFromPoint( - rect.left + rect.width / 2, rect.top + rect.height / 2, - )); + const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); + // LinkedIn localizes the preview label. Verify the attachment itself + // at the dispatch point instead; small avatar/reaction/menu icons do + // not prove that a button opens an image viewer. + return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => { + const imageRect = image.getBoundingClientRect(); + return painted(image) && imageRect.width >= 64 && imageRect.height >= 64 + && _isComposedAncestor(image, hit); + }); }; const classifyLinkedInNavigation = (clicked, blockingModal = null) => { diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index cb376a124d..c5b8046f01 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -360,6 +360,43 @@ for (const build of ['chrome', 'firefox']) { agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); }); + for (const contentKind of ['string', 'text-block']) { + test(`${build}: ${contentKind} authority data URLs keep the steering chain after recovery`, async () => { + const { serializeConversationForSession: serialize } = await import(`../src/${build}/src/agent/conversation-persistence.js`); + const agent = actSetup(Agent); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const root = 'Add this invitation to my calendar: data:application/pdf;base64,QUJDRA==\nUse the meeting time in it.'; + const content = contentKind === 'string' ? root : [{ type: 'text', text: root }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,QUJDRA==' } }]; + const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content }]; + agent.conversations.set(tabId, messages); + agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + steer(agent, 'Use Ipek’s message: data:image/png;base64,QUJDRA==', 'image-correction'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + steer(agent, 'Use the work calendar', 'calendar-correction'); + await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options); + const live = agent._activeTaskBinding(messages); + messages.push({ role: 'assistant', content: 'Observed binary: data:image/png;base64,QUJDRA==' }); + messages.push(...Array.from({ length: 70 }, () => ({ role: 'assistant', content: 'Earlier observation '.repeat(1000) }))); + for (const maxBytes of [1_500_000, 450_000]) { + const snapshot = serialize(messages, { maxBytes, preserveMessageIndices: live.pinnedIndices }); + const restored = actSetup(Agent); + restored.conversations.set(tabId, snapshot.messages); + assert.equal(restored._activeTaskBinding(snapshot.messages).text, live.text); + assert.equal(restored._progressTaskKeyHash(tabId), agent._progressTaskKeyHash(tabId)); + assert.equal(restored._plannerUserAuthoredText(snapshot.messages[1]), root); + assert.equal(snapshot.messages[live.pinnedIndices[1]].content, messages[live.pinnedIndices[1]].content); + assert.ok(!snapshot.messages.some(message => message.role === 'assistant' && String(message.content).includes('data:image')), + 'Non-authority binary text must still be omitted'); + if (contentKind === 'text-block') assert.equal(snapshot.messages[1].content[1].type, 'text', 'Binary attachment blocks remain omitted'); + restored._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true }); + restored._markPlanExecutionToolCall(tabId, 'read_page', { success: true }); + assert.equal(restored._planExecutionGuards.get(tabId).taskDrifted, false); + } + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + test(`${build}: steering during planning discards the superseded intent`, async () => { const entered = deferred(), release = deferred(), plans = []; const agent = actSetup(Agent, {}, async (_tab, enriched) => { diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjs index e93bc7c7ee..a48a8edd58 100644 --- a/test/fixtures/message-recipient-navigation.mjs +++ b/test/fixtures/message-recipient-navigation.mjs @@ -164,9 +164,33 @@ export function registerMessageRecipientNavigationFixtures({ assert.equal(await page.evaluate(() => window.sentMessages), 0); }); + register(`${kind}: localized LinkedIn attachment viewers open across selector, text, AX, and coordinate targeting`, async page => { + const { guard } = await setupMessageHistory(page); + for (const label of ['Görsel önizlemesinde görüntülemek için tıklayın veya Enter tuşuna basın', + 'Cliquez ou appuyez sur Entrée pour afficher un aperçu de l’image', + 'Clique ou pressione Enter para exibir na visualização da imagem']) { + await page.locator('#preview').evaluate((el, label) => el.setAttribute('aria-label', label), label); + const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 }); + const ref = tree.pageContent.split('\n').find(line => line.includes(`button "${label}"`))?.match(/\[([^\]]+)\]/)?.[1]; + assert.ok(ref, tree.pageContent); + const point = await page.locator('#preview').evaluate(el => { + const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; + }); + for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: label, textMatch: 'exact' }], + ['click_ax', { ref_id: ref }], ['click', point], ['click', { selector: '#preview-image' }]]) { + assert.equal(await guard(tool, args), null, `${label}/${tool}`); + assert.equal((await call(page, tool, args)).success, true); + assert.equal(await page.locator('#viewer').isVisible(), true); + await page.locator('#close-viewer').click(); + } + } + assert.equal(await page.evaluate(() => window.sentMessages), 0); + }); + register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => { const { guard } = await setupMessageHistory(page); - for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text']) { + for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text', + 'localized-send', 'submit', 'external-form', 'outside-history', 'small-icon', 'menu', 'image-hidden']) { await page.evaluate(change => { const button=document.querySelector('#preview'); window.originalPreview=button.cloneNode(true); @@ -178,6 +202,13 @@ export function registerMessageRecipientNavigationFixtures({ if(change==='disabled')button.disabled=true; if(change==='commit-label')button.title='Send'; if(change==='commit-text'){button.title='View image';button.append('Send');} + if(change==='localized-send')button.setAttribute('aria-label','Gönder'); + if(change==='submit')button.type='submit'; + if(change==='external-form')button.setAttribute('form','composer'); + if(change==='outside-history')document.body.append(button); + if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px'; + if(change==='menu')button.setAttribute('aria-haspopup','menu'); + if(change==='image-hidden')button.querySelector('img').style.visibility='hidden'; if(change==='occluded') { const cover=document.createElement('div');cover.id='cover'; cover.style='position:fixed;left:350px;top:80px;width:400px;height:340px;z-index:20;background:white';document.body.append(cover); From 827371b3b9512bd7fdfc51eb2a21fbb7b895dabf Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 19:48:18 +0300 Subject: [PATCH 4/6] Skip superseded planner recommendations and allow form-free image previews --- src/chrome/src/agent/agent.js | 18 ++++---- src/chrome/src/content/content.js | 2 +- src/firefox/src/agent/agent.js | 18 ++++---- src/firefox/src/content/content.js | 2 +- test/chat-steering.mjs | 43 +++++++++++++++++++ .../fixtures/message-recipient-navigation.mjs | 33 +++++++++++++- 6 files changed, 97 insertions(+), 19 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index d40e567de0..1949d1c292 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -43293,14 +43293,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); - let initialResponseRevalidated = false; + // The initial planner can authorize steering before response-only handling. + let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0; if (gateOutcome.responseOnly === true) { const responseOutcome = await this._completeInitialResponseOnlyTurn( tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); gateOutcome = responseOutcome.gate; - initialResponseRevalidated = responseOutcome.revalidated; + initialTaskRevalidated ||= responseOutcome.revalidated; if (!gateOutcome.proceed) { _traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; return (finalResponse = gateOutcome.message || 'More information is required.'); @@ -43323,7 +43324,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai } } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -43529,7 +43530,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai this._appendStandaloneUngroundedPolicy(enriched, standaloneGroundingGap); } - const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { @@ -44650,14 +44651,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); - let initialResponseRevalidated = false; + // The initial planner can authorize steering before response-only handling. + let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0; if (gateOutcome.responseOnly === true) { const responseOutcome = await this._completeInitialResponseOnlyTurn( tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); gateOutcome = responseOutcome.gate; - initialResponseRevalidated = responseOutcome.revalidated; + initialTaskRevalidated ||= responseOutcome.revalidated; if (!gateOutcome.proceed) { const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; return finish(gateOutcome.message || 'More information is required.', status); @@ -44675,7 +44677,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai } } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); let standaloneGroundingGap = this._standaloneOfflineGroundingGap(localWikipediaRag, runOptions); if (standaloneGroundingGap) { @@ -44739,7 +44741,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai // currentStreamRequestMessages is declared at the top of this function so // pre-loop exits (e.g. response-only turns) can capture their request. - const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index e9fe24e0bd..ec4d911a96 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -5875,7 +5875,7 @@ }; if (!button || !painted(button) || button.disabled || button.matches(':disabled') || button.getAttribute('aria-disabled') === 'true' - || String(button.getAttribute('type') || '').toLowerCase() !== 'button' + || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button') || button.form || button.hasAttribute('form') || button.hasAttribute('download') || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '') || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 9704135f86..fdb610644d 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -36257,14 +36257,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); - let initialResponseRevalidated = false; + // The initial planner can authorize steering before response-only handling. + let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0; if (gateOutcome.responseOnly === true) { const responseOutcome = await this._completeInitialResponseOnlyTurn( tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); gateOutcome = responseOutcome.gate; - initialResponseRevalidated = responseOutcome.revalidated; + initialTaskRevalidated ||= responseOutcome.revalidated; if (!gateOutcome.proceed) { _traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; return (finalResponse = gateOutcome.message || 'More information is required.'); @@ -36287,7 +36288,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -36482,7 +36483,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); } - const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { @@ -37445,14 +37446,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d || gateOutcome.responseLanguageApprovedPlanOverride === true, trustedContinuation: runOptions?.trustedContinuation === true, }); - let initialResponseRevalidated = false; + // The initial planner can authorize steering before response-only handling. + let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0; if (gateOutcome.responseOnly === true) { const responseOutcome = await this._completeInitialResponseOnlyTurn( tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, gateOutcome, enriched, sourceBoundPriorMessages, ); gateOutcome = responseOutcome.gate; - initialResponseRevalidated = responseOutcome.revalidated; + initialTaskRevalidated ||= responseOutcome.revalidated; if (!gateOutcome.proceed) { const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled'; return finish(gateOutcome.message || 'More information is required.', status); @@ -37470,7 +37472,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId); - if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); + if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions); if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) { await this._ensureProgressSessionForCurrentTask(tabId, { @@ -37521,7 +37523,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // currentStreamRequestMessages is declared at the top of this function so // pre-loop exits (e.g. response-only turns) can capture their request. - const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( + const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool( tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas, ); if (recommendedFirstTool?.action === 'return') { diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 38a0e2216f..f20c87713d 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -4963,7 +4963,7 @@ }; if (!button || !painted(button) || button.disabled || button.matches(':disabled') || button.getAttribute('aria-disabled') === 'true' - || String(button.getAttribute('type') || '').toLowerCase() !== 'button' + || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button') || button.form || button.hasAttribute('form') || button.hasAttribute('download') || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '') || _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false; diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index c5b8046f01..fc546dce2a 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -117,6 +117,49 @@ for (const build of ['chrome', 'firefox']) { for (const streaming of [false, true]) { for (const mode of ['act', 'dev']) { + for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) { + test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => { + const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = []; + let modelCalls = 0; + const next = async () => ++modelCalls === 1 + ? { toolCalls: [{ id: 'dashboard', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }] } + : { toolCalls: [{ id: 'finished', function: { name: 'done', arguments: '{"summary":"Dashboard opened","outcome":"success"}' } }] }; + const agent = actSetup(Agent, { + supportsVision: true, + chat: next, + async *chatStream() { yield { type: 'tool_call', content: (await next()).toolCalls }; yield { type: 'done' }; }, + }, async (_tab, enriched) => { + plans.push(enriched.content); + if (plans.length === 1) { entered.resolve(); await release.promise; } + return { proceed: true, requestKind: 'execute', requiresStateChange: plans.length > 1, requiresSubmission: false }; + }); + agent.executeTool = async (_tab, name, args) => { + if (name === 'done') { + assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null); + return { done: true, summary: args.summary, outcome: args.outcome }; + } + dispatched.push(name); return { success: true, url: args.url }; + }; + const runOptions = { ...options, recommendedAction: { id, tool, autoExecute: true } }; + assert.ok(agent._recommendedActionFirstTool(runOptions), 'Fixture must activate the real recommended first tool'); + const firstTool = agent._maybeExecuteRecommendedActionFirstTool.bind(agent); + agent._maybeExecuteRecommendedActionFirstTool = async (...args) => { + recommendedAttempts.push(args[1]?.recommendedAction?.tool); + return firstTool(...args); + }; + const run = streaming + ? agent.processMessageStream(tabId, 'Inspect this page', () => {}, mode, runOptions) + : agent.processMessage(tabId, 'Inspect this page', () => {}, mode, [], runOptions); + await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]); + assert.equal(steer(agent, 'Cancel the page inspection; open my dashboard instead').accepted, true); + release.resolve(); + assert.equal(await run, 'Dashboard opened'); + assert.equal(plans.length, 2); + assert.match(plans[1], /Cancel the page inspection; open my dashboard instead/); + assert.deepEqual(recommendedAttempts, [], 'Steering must suppress the stale recommendation before any preparation or dispatch'); + assert.deepEqual(dispatched, ['navigate'], 'The original recommended read/screenshot must never dispatch'); + }); + } for (const revisedKind of ['respond', 'execute']) { test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} initial response-only steering switches to ${revisedKind}`, async () => { const entered = deferred(), release = deferred(), updates = [], persisted = [], dispatched = []; diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjs index a48a8edd58..972b4d3277 100644 --- a/test/fixtures/message-recipient-navigation.mjs +++ b/test/fixtures/message-recipient-navigation.mjs @@ -187,6 +187,37 @@ export function registerMessageRecipientNavigationFixtures({ assert.equal(await page.evaluate(() => window.sentMessages), 0); }); + register(`${kind}: form-free LinkedIn previews need no explicit button type`, async page => { + for (const tag of ['button', 'div']) { + const { guard } = await setupMessageHistory(page); + await page.evaluate(tag => { + const original = document.querySelector('#preview'); + original.removeAttribute('type'); + if (tag === 'div') { + const control = document.createElement('div'); + for (const attribute of original.attributes) control.setAttribute(attribute.name, attribute.value); + control.setAttribute('role', 'button'); control.tabIndex = 0; + control.append(...original.childNodes); original.replaceWith(control); + control.onclick = () => { document.querySelector('#viewer').hidden = false; }; + } + }, tag); + const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 }); + const ref = /button "Click or press enter to display in the image preview" \[([^\]]+)\]/.exec(tree.pageContent)?.[1]; + assert.ok(ref, tree.pageContent); + const point = await page.locator('#preview').evaluate(el => { + const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; + }); + for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: 'Click or press enter to display in the image preview' }], + ['click_ax', { ref_id: ref }], ['click', point]]) { + assert.equal(await guard(tool, args), null, `${tag}/${tool}`); + assert.equal((await call(page, tool, args)).success, true); + assert.equal(await page.locator('#viewer').isVisible(), true); + await page.locator('#close-viewer').click(); + } + assert.equal(await page.evaluate(() => window.sentMessages), 0); + } + }); + register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => { const { guard } = await setupMessageHistory(page); for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text', @@ -204,7 +235,7 @@ export function registerMessageRecipientNavigationFixtures({ if(change==='commit-text'){button.title='View image';button.append('Send');} if(change==='localized-send')button.setAttribute('aria-label','Gönder'); if(change==='submit')button.type='submit'; - if(change==='external-form')button.setAttribute('form','composer'); + if(change==='external-form'){button.removeAttribute('type');button.setAttribute('form','composer');} if(change==='outside-history')document.body.append(button); if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px'; if(change==='menu')button.setAttribute('aria-haspopup','menu'); From f9cdafc74a80616b0736a858427d5f7d6bd527d8 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 20:03:47 +0300 Subject: [PATCH 5/6] Reset superseded skills before steering reauthorization --- src/chrome/src/agent/agent.js | 16 +++++++++--- src/firefox/src/agent/agent.js | 16 +++++++++--- test/chat-steering.mjs | 47 ++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index bb7d373e25..1674a9b9d5 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -6166,11 +6166,19 @@ export class Agent extends LoopDetector { recommendedAction: null, }; const readState = this.readCompletenessStates.get(tabId); + const pageUrl = await this._currentUrl(tabId); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + const adapterName = getActiveAdapter(pageUrl)?.name || ''; + // A plan or load_skill activates tools for that revision only. Rebuild + // the current site's automatic baseline, excluding the old task's + // recommended action, before the revised planner can choose new skills. + this._resetActiveSkillsForRun(tabId, { refreshPrompt: false }); + this.lastSeenAdapter.set(tabId, adapterName); + this._preactivateNyTimesSkillForRun(tabId, mode); + this._preactivateHumanizerSkillForRun(tabId, mode); + this._refreshSystemPromptForTab(tabId, mode); if (readState) { - const pageUrl = await this._currentUrl(tabId); - if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; - if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; - const adapterName = getActiveAdapter(pageUrl)?.name || ''; // Let this revision's planner establish its own read obligation. // Keep run ownership for teardown, but discard earlier coverage and // obligations, including ones armed by a superseded planner result. diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 70739b9d3f..c73c837a2d 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -5880,11 +5880,19 @@ export class Agent extends LoopDetector { recommendedAction: null, }; const readState = this.readCompletenessStates.get(tabId); + const pageUrl = await this._currentUrl(tabId); + if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; + if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; + const adapterName = getActiveAdapter(pageUrl)?.name || ''; + // A plan or load_skill activates tools for that revision only. Rebuild + // the current site's automatic baseline, excluding the old task's + // recommended action, before the revised planner can choose new skills. + this._resetActiveSkillsForRun(tabId, { refreshPrompt: false }); + this.lastSeenAdapter.set(tabId, adapterName); + this._preactivateNyTimesSkillForRun(tabId, mode); + this._preactivateHumanizerSkillForRun(tabId, mode); + this._refreshSystemPromptForTab(tabId, mode); if (readState) { - const pageUrl = await this._currentUrl(tabId); - if (this._checkAbort(tabId) || this._steeringRuns.get(tabId) !== run) break; - if (run.revision !== revision || this._hasPendingSteering(tabId)) continue; - const adapterName = getActiveAdapter(pageUrl)?.name || ''; // Let this revision's planner establish its own read obligation. // Keep run ownership for teardown, but discard earlier coverage and // obligations, including ones armed by a superseded planner result. diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index fc546dce2a..17e0adc187 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -440,6 +440,53 @@ for (const build of ['chrome', 'firefox']) { }); } + for (const activatedBy of ['plan', 'load_skill']) { + test(`${build}: steering replaces skills activated by ${activatedBy} and refreshes the site baseline`, async () => { + let phase = 'original', pageUrl = 'https://mail.google.com/mail/u/0/#inbox'; + const agent = actSetup(Agent, {}, async () => { + if (phase !== 'original') { + assert.deepEqual([...(agent.activeSkillIds.get(tabId) || [])], phase === 'dashboard' ? [] : ['humanizer'], + 'Only the current site baseline may reach the revised planner'); + } + return { proceed: true, requestKind: 'execute', requiresStateChange: false, + skillIds: phase === 'original' ? ['freeskillz-xyz'] : phase === 'weather' ? ['open-meteo-weather'] : [] }; + }); + const records = [['freeskillz-xyz', 'FreeSkillz.xyz'], ['humanizer', 'Humanizer'], ['open-meteo-weather', 'Open-Meteo weather']] + .map(([id, name]) => ({ id, name, sourceType: 'built-in', sourceUrl: `skills/${id}.md`, content: read(build, `../skills/${id}.md`), createdAt: 0 })); + records[0].content += '\nSUPERSEDED_SKILL_BODY_SENTINEL\n'; + agent.setCustomSkills(records); + agent.conversationModes.set(tabId, 'act'); agent.lastSeenAdapter.set(tabId, 'gmail'); + agent._currentUrl = async () => pageUrl; + agent._getTabUrlTitle = async () => ({ tabUrl: pageUrl, tabTitle: 'Current page' }); + await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options); + const messages = [{ role: 'system', content: 'sys' }]; agent.conversations.set(tabId, messages); + assert.equal(agent._preactivateHumanizerSkillForRun(tabId, 'act'), true); + if (activatedBy === 'plan') { + await agent._maybeRunPlannerGate(tabId, messages, { role: 'user', content: 'Download this media.' }, () => {}, 'act', null, null, null, options); + } else { + messages.push({ role: 'user', content: 'Download this media.' }); + assert.equal(agent._loadSkillForRun(tabId, { skill_id: 'freeskillz-xyz' }).success, true); + } + agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: false }); + assert.match(messages[0].content, /SUPERSEDED_SKILL_BODY_SENTINEL/); + assert.ok(agent._activeSkillToolForName(tabId, 'download_public_media'), 'Old consequential tool must really be active'); + const refresh = async text => { + assert.equal(steer(agent, text, phase).accepted, true); + assert.equal((await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options)).gate.proceed, true); + assert.equal(agent._activeSkillToolForName(tabId, 'download_public_media'), null); + assert.doesNotMatch(messages[0].content, /SUPERSEDED_SKILL_BODY_SENTINEL/); + }; + phase = 'weather'; await refresh('Cancel the download; check the weather instead.'); + assert.deepEqual([...(agent.activeSkillIds.get(tabId) || [])], ['humanizer', 'open-meteo-weather']); + assert.ok(agent._skillToolDefinitions(tabId, 'act', 'full').length, 'The revised plan must activate its own tool'); + phase = 'mail'; await refresh('Cancel the weather request; inspect this email instead.'); + assert.deepEqual([...(agent.activeSkillIds.get(tabId) || [])], ['humanizer']); + pageUrl = 'https://example.com/dashboard'; phase = 'dashboard'; await refresh('Use the dashboard instead.'); + assert.equal(agent.activeSkillIds.has(tabId), false, 'A baseline from the old site cannot survive navigation'); + agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId); + }); + } + test(`${build}: steering during planning discards the superseded intent`, async () => { const entered = deferred(), release = deferred(), plans = []; const agent = actSetup(Agent, {}, async (_tab, enriched) => { From 990bf57a93aa0118af8c5de97835a6bf556cafae Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 6 Oct 2026 20:20:56 +0300 Subject: [PATCH 6/6] Consume final-step steering before terminating a run --- src/chrome/src/agent/agent.js | 28 ++++++++++---- src/firefox/src/agent/agent.js | 28 ++++++++++---- test/chat-steering.mjs | 69 ++++++++++++++++++++++++++++++---- 3 files changed, 104 insertions(+), 21 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 1674a9b9d5..6cc04ce0ca 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -6096,6 +6096,11 @@ export class Agent extends LoopDetector { return !!this._steeringRuns.get(tabId)?.messages.length; } + _hasUnvalidatedSteering(tabId) { + const run = this._steeringRuns.get(tabId); + return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision); + } + _applyPendingSteering(tabId, messages, onUpdate) { const run = this._steeringRuns.get(tabId); if (!run?.messages.length || this._checkAbort(tabId)) return false; @@ -9141,7 +9146,12 @@ export class Agent extends LoopDetector { const toolCounts = new Map(); let lastAssistantText = ''; let lastToolCall = null; - for (const m of messages) { + const binding = this._activeTaskBinding(messages); + // Earlier attempts remain in history, but this handoff describes the + // latest authorized revision rather than quoting its superseded answer. + const summaryMessages = binding.updates?.some(update => update.kind === 'steering') + ? messages.slice(binding.index + 1) : messages; + for (const m of summaryMessages) { if (m.role === 'assistant') { if (Array.isArray(m.tool_calls)) { for (const tc of m.tool_calls) { @@ -43584,7 +43594,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai // consider this run. Failures/cancels are still excluded via _traceStatus. shareHadProviderCompletion = true; - while (steps < this.maxSteps) { + while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) { // Each turn's raw provider text belongs to that turn only: reset so a // non-terminal final-text pass can never pair stale output with a later // tool-batch result in the shared record. @@ -43602,6 +43612,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, ); if (steeringOutcome?.changed) { + // Replace the superseded final turn with one freshly authorized turn. + if (steps >= this.maxSteps) steps--; forceCompletionVerificationTurn = false; allowCompletionFailureTurn = false; forceCompletionDoneAfterVerification = false; @@ -43874,7 +43886,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai break; } - if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) { + if (this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; } @@ -43978,7 +43990,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai const batchResult = await this._executeToolBatch( tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas ); - if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps + if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; @@ -44789,7 +44801,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai // Past local-only fast paths: the streaming loop below calls the provider. shareHadProviderCompletion = true; - while (steps < this.maxSteps) { + while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) { // See the non-streaming loop: raw provider text must not survive into // a later turn's shared record. shareRawResponse = null; @@ -44805,6 +44817,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, ); if (steeringOutcome?.changed) { + // Replace the superseded final turn with one freshly authorized turn. + if (steps >= this.maxSteps) steps--; pendingVisionFallbackMessages = null; forceCompletionVerificationTurn = false; allowCompletionFailureTurn = false; @@ -45024,7 +45038,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai toolCalls: streamedToolCalls, })); - if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) { + if (this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; } @@ -45114,7 +45128,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai const batchResult = await this._executeToolBatch( tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas ); - if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps + if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index c73c837a2d..90b8e23667 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -5810,6 +5810,11 @@ export class Agent extends LoopDetector { return !!this._steeringRuns.get(tabId)?.messages.length; } + _hasUnvalidatedSteering(tabId) { + const run = this._steeringRuns.get(tabId); + return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision); + } + _applyPendingSteering(tabId, messages, onUpdate) { const run = this._steeringRuns.get(tabId); if (!run?.messages.length || this._checkAbort(tabId)) return false; @@ -8973,7 +8978,12 @@ export class Agent extends LoopDetector { const toolCounts = new Map(); let lastAssistantText = ''; let lastToolCall = null; - for (const m of messages) { + const binding = this._activeTaskBinding(messages); + // Earlier attempts remain in history, but this handoff describes the + // latest authorized revision rather than quoting its superseded answer. + const summaryMessages = binding.updates?.some(update => update.kind === 'steering') + ? messages.slice(binding.index + 1) : messages; + for (const m of summaryMessages) { if (m.role === 'assistant') { if (Array.isArray(m.tool_calls)) { for (const tc of m.tool_calls) { @@ -36537,7 +36547,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // consider this run. Failures/cancels are still excluded via _traceStatus. shareHadProviderCompletion = true; - while (steps < this.maxSteps) { + while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) { // Each turn's raw provider text belongs to that turn only: reset so a // non-terminal final-text pass can never pair stale output with a later // tool-batch result in the shared record. @@ -36554,6 +36564,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, ); if (steeringOutcome?.changed) { + // Replace the superseded final turn with one freshly authorized turn. + if (steps >= this.maxSteps) steps--; forceCompletionVerificationTurn = false; allowCompletionFailureTurn = false; forceCompletionDoneAfterVerification = false; @@ -36801,7 +36813,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d break; } - if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) { + if (this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; } @@ -36857,7 +36869,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const batchResult = await this._executeToolBatch( tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas ); - if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps + if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; @@ -37571,7 +37583,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Past local-only fast paths: the streaming loop below calls the provider. shareHadProviderCompletion = true; - while (steps < this.maxSteps) { + while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) { // See the non-streaming loop: raw provider text must not survive into // a later turn's shared record. shareRawResponse = null; @@ -37587,6 +37599,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d tabId, messages, onUpdate, mode, provider, costState, runId, runOptions, ); if (steeringOutcome?.changed) { + // Replace the superseded final turn with one freshly authorized turn. + if (steps >= this.maxSteps) steps--; pendingVisionFallbackMessages = null; forceCompletionVerificationTurn = false; allowCompletionFailureTurn = false; @@ -37802,7 +37816,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d toolCalls: streamedToolCalls, })); - if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) { + if (this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; } @@ -37845,7 +37859,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const batchResult = await this._executeToolBatch( tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas ); - if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps + if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && this._applyPendingSteering(tabId, messages, onUpdate)) { onUpdate('text', { content: '', replace: true }); continue; diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs index 17e0adc187..b3c6d249b7 100644 --- a/test/chat-steering.mjs +++ b/test/chat-steering.mjs @@ -117,6 +117,59 @@ for (const build of ['chrome', 'firefox']) { for (const streaming of [false, true]) { for (const mode of ['act', 'dev']) { + for (const phase of ['model', 'tool']) { + for (const revisedKind of ['respond', 'execute']) { + test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} final ${phase} step revalidates steering to ${revisedKind}`, async () => { + const entered = deferred(), release = deferred(), plans = [], updates = [], dispatched = []; + let calls = 0; + const next = async () => { + if (++calls === 1) { + if (phase === 'model') { entered.resolve(); await release.promise; } + return { content: 'Obsolete final-step answer', toolCalls: [ + { id: 'old-read', function: { name: 'get_accessibility_tree', arguments: '{}' } }, + { id: 'old-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/old"}' } }, + ] }; + } + return revisedKind === 'respond' ? { content: 'Revised answer' } : { toolCalls: [ + { id: 'revised-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }, + ] }; + }; + const agent = actSetup(Agent, { + chat: next, + async *chatStream() { const r = await next(); if (r.content) yield { type: 'text', content: r.content }; if (r.toolCalls) yield { type: 'tool_call', content: r.toolCalls.map((call, index) => ({ ...call, index })) }; yield { type: 'done' }; }, + }, async (_tab, enriched) => { + plans.push(enriched.content); + return { proceed: true, requestKind: plans.length > 1 ? revisedKind : 'execute', + responseOnly: plans.length > 1 && revisedKind === 'respond', requiresStateChange: plans.length > 1 && revisedKind === 'execute' }; + }); + agent.maxSteps = 1; + agent.executeTool = async (_tab, name, args) => { + if (name === 'done') return { done: true, summary: args.summary, outcome: args.outcome }; + dispatched.push(args.url || name); + if (name === 'get_accessibility_tree') { entered.resolve(); await release.promise; return { success: true, pageContent: 'Completed old observation' }; } + return { success: true, url: args.url }; + }; + const update = (type, data) => updates.push({ type, data }); + const run = streaming ? agent.processMessageStream(tabId, 'Inspect the page', update, mode, options) + : agent.processMessage(tabId, 'Inspect the page', update, mode, [], options); + await Promise.race([entered.promise, run.then(r => assert.fail(`Early completion: ${r}`))]); + assert.equal(steer(agent, 'Use the corrected task instead').accepted, true); release.resolve(); + const result = await run; + if (revisedKind === 'respond') assert.equal(result, 'Revised answer'); + else { + // The replacement turn keeps the normal step cap and completion verification. + assert.match(result, /Step limit reached after 1 steps/); + assert.doesNotMatch(result, /Obsolete final-step answer|example\.com\/old/); + assert.equal(calls, 2); + } + assert.equal(plans.length, 2); assert.match(plans[1], /Use the corrected task instead/); + assert.deepEqual(dispatched, [...(phase === 'tool' ? ['get_accessibility_tree'] : []), ...(revisedKind === 'execute' ? ['https://example.com/dashboard'] : [])]); + assert.equal(updates.some(u => u.type === 'steering_queued'), false); + assert.equal(agent.conversations.get(tabId).some(m => m.role === 'assistant' && m.content === 'Obsolete final-step answer' && !m.tool_calls), false); + assertPairedTools(agent.conversations.get(tabId)); + }); + } + } for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) { test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => { const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = []; @@ -349,11 +402,12 @@ for (const build of ['chrome', 'firefox']) { assert.equal(agent.isRunning(tabId), false); }); - test(`${build}: ${streaming ? 'stream' : 'chat'} returns unconsumed steering when the step budget ends`, async () => { + test(`${build}: ${streaming ? 'stream' : 'chat'} final Ask step consumes steering and replaces the obsolete answer`, async () => { const entered = deferred(); const release = deferred(); + let calls = 0; const agent = setup(Agent, { - chat: async () => { entered.resolve(); await release.promise; return { content: 'Finished' }; }, - async *chatStream() { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; yield { type: 'done' }; }, + chat: async () => { if (++calls === 1) { entered.resolve(); await release.promise; return { content: 'Finished' }; } return { content: 'Corrected answer' }; }, + async *chatStream() { if (++calls === 1) { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; } else yield { type: 'text', content: 'Corrected answer' }; yield { type: 'done' }; }, }); agent.maxSteps = 1; const updates = []; @@ -364,10 +418,11 @@ for (const build of ['chrome', 'firefox']) { await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]); assert.equal(steer(agent, 'Follow-up').accepted, true); release.resolve(); - await run; - assert.deepEqual(updates.find(update => update.type === 'steering_queued').data.messages, - [{ id: 'correction-1', text: 'Follow-up' }]); - assert.equal(updates.filter(update => update.type === 'steering_applied').length, 0); + assert.equal(await run, 'Corrected answer'); + assert.equal(calls, 2); + assert.equal(updates.some(update => update.type === 'steering_queued'), false); + assert.equal(updates.filter(update => update.type === 'steering_applied').length, 1); + assert.equal(agent.conversations.get(tabId).some(message => message.role === 'assistant' && message.content === 'Finished'), false); assert.equal(agent._steeringRuns.size, 0); }); }