diff --git a/packages/uipath/samples/ticket-triage-agent/.env.example b/packages/uipath/samples/ticket-triage-agent/.env.example new file mode 100644 index 000000000..0466f06b3 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/.env.example @@ -0,0 +1,9 @@ +UIPATH_URL=https://cloud.uipath.com/your_org/your_tenant +UIPATH_ACCESS_TOKEN=your_access_token_here + +# Required for the escalation path: Action Center tasks must be created in +# an Orchestrator folder. +UIPATH_FOLDER_PATH=Shared + +# Required for Tier 1 triage: API key for TypeSafe AI's Jev model. +TYPESAFE_API_KEY=your_typesafe_api_key_here diff --git a/packages/uipath/samples/ticket-triage-agent/README.md b/packages/uipath/samples/ticket-triage-agent/README.md new file mode 100644 index 000000000..309378f3a --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/README.md @@ -0,0 +1,155 @@ +# Ticket Triage Agent: System 1 / System 2 Pattern + +This sample demonstrates a two-tier triage pattern for support tickets: + +1. **Tier 1 (System 1) - fast structured triage.** A "System One" model + takes the ticket text and a set of typed questions, and returns + calibrated typed answers (a classification, a 0-1 probability, and a + score) instead of free text - orders of magnitude cheaper and faster than an + LLM call. This sample uses TypeSafe AI's `Jev` model via the + `typesafe-sdk` package. +2. **Tier 2 (System 2) - branch on the triage result.** + - If the ticket is urgent, the customer sounds frustrated, or the + department routing is low-confidence, the agent **escalates to a + human** via a UiPath Action Center QuickForm task, attaching the Tier 1 + decision as context so the reviewer isn't starting cold. + - Otherwise, the agent calls a **real LLM** (UiPath LLM Gateway) with a + department-specific system prompt to draft a reply, and returns it as + an auto-resolved ticket. No human, and no expensive LLM call for + routing, is needed for the common case. + +## About the "Jev" model (important - read this) + +This sample calls TypeSafe AI's real `typesafe-sdk` PyPI package, not a +mock. `typesafe-sdk` is a young package (first published 2026-09-09), so +before wiring it in we statically inspected the wheel's source (no +install/execution): it's a normal, apparently auto-generated API client +(the response schemas reference an OpenAPI spec) with no `eval`/`exec`/ +`subprocess` calls, and no exfiltration of environment variables. Its +`httpx2` dependency turned out to be an unrelated, independent package +(the pydantic org's next-generation `httpx`), not something specific to +TypeSafe AI. If you're pulling a new AI SDK into your own project, reading +the source before trusting it is a cheap, worthwhile step regardless of how +long the package has existed. + +```python +from typesafe_sdk import Choice, Noul, Score, TypeSafeClient + +client = TypeSafeClient() # reads TYPESAFE_API_KEY from the environment +response = client.system_one(state=..., questions={...}) +``` + +`pyproject.toml` pins `typesafe-sdk>=0.7.0` rather than leaving it +unbounded, since `uipath`'s own `uv` install already applies a +minimum-package-age safety check that skips the very newest release. + +## Prerequisites + +* [UV package manager](https://docs.astral.sh/uv/) installed +* A TypeSafe AI API key (`TYPESAFE_API_KEY`) for the Tier 1 triage step +* A UiPath Orchestrator tenant with: + * Access to the LLM Gateway (for the auto-reply path) + * Access to Action Center (for the escalation path) + +## Setup + +### Step 1: Create and activate a virtual environment + +```bash +uv venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +``` + +### Step 2: Install dependencies + +```bash +uv sync +``` + +### Step 3: Configure credentials + +Copy `.env.example` to `.env` and fill in your Orchestrator URL, access +token, and TypeSafe AI API key: + +```bash +cp .env.example .env +``` + +### Step 4: Initialize the agent + +```bash +uv run uipath init +``` + +### Step 5: Run the agent + +```bash +uipath run main --input-file input.json +``` + +Try editing `input.json` to see both branches: + +* A calm, clearly-worded billing request (like the default input) -> the + agent auto-drafts a reply via the LLM Gateway and returns it directly. +* An urgent or angry-sounding message (e.g. mentioning "urgent", "ASAP", or + "furious") -> the agent creates an Action Center task instead and returns + its task id. + +## How it works + +1. `triage_ticket` sends the ticket's subject/message to `Jev` (via + `TypeSafeClient.system_one`) as `state`, along with three typed + `questions` (`department: Choice`, `is_urgent: Noul`, + `frustration: Score`), and gets back a `TriageDecision`. +2. `needs_escalation` checks the triage output against fixed thresholds + (urgency, frustration, routing confidence) in `main.py`. +3. On escalation, `escalate_to_human` creates an Action Center QuickForm + task via `client.tasks.create_quickform(...)`, with the ticket and the + triage decision as task data for the reviewer. +4. Otherwise, `draft_auto_reply` calls `client.llm.chat_completions(...)` + (UiPath LLM Gateway) with a department-specific system prompt to draft a + reply. + +## Evaluations + +`evaluations/eval-sets/default.json` exercises the Tier 1 routing/escalation +logic against 11 tickets (billing, technical, and sales; calm and +auto-replied vs. urgent/angry/low-confidence and escalated), using two +evaluators: + +* `DepartmentRoutingEvaluator` (`evaluations/evaluators/department-routing.json`) + - a `uipath-multiclass-classification` evaluator checking + `triage.department` against the expected class. +* `EscalationDecisionEvaluator` (`evaluations/evaluators/escalation-decision.json`) + - a `uipath-binary-classification` evaluator checking the `escalated` + boolean against the expected outcome. + +Two cases (`sales-demo-request-wrong-department`, +`billing-refund-calm-wrong-escalation`) have deliberately wrong ground truth, +mirroring the pattern in `classification_agent`, to demonstrate the +evaluators catching a mismatch. + +Every case calls the real Jev API (`TYPESAFE_API_KEY`), and Tier 2 always +runs too: `draft_auto_reply` and `escalate_to_human` call real UiPath +services (LLM Gateway, Action Center), so the full eval set requires valid +credentials in `.env` for all 11 cases, not just the ones the two +evaluators grade. + +**Heads up:** none of the current evaluators grade `auto_reply` or +`action_task_id`, but the ~3 cases whose triage escalates still create a +real Action Center QuickForm task each run, and nothing in this sample +cleans them up. Expect leftover tasks in Action Center after repeated +`uipath eval` runs. + +```bash +uipath eval +``` + +## Publish your coded agent + +Once tested locally, publish the agent to Orchestrator: + +```bash +uipath pack +uipath publish +``` diff --git a/packages/uipath/samples/ticket-triage-agent/bindings.json b/packages/uipath/samples/ticket-triage-agent/bindings.json new file mode 100644 index 000000000..5e9beeb01 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/bindings.json @@ -0,0 +1,4 @@ +{ + "version": "2.0", + "resources": [] +} diff --git a/packages/uipath/samples/ticket-triage-agent/evaluations/eval-sets/default.json b/packages/uipath/samples/ticket-triage-agent/evaluations/eval-sets/default.json new file mode 100644 index 000000000..01213ee8c --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/evaluations/eval-sets/default.json @@ -0,0 +1,143 @@ +{ + "version": "1.0", + "id": "TicketTriageEval", + "name": "Ticket Triage Routing & Escalation Evaluation", + "evaluatorRefs": [ + "DepartmentRoutingEvaluator", + "EscalationDecisionEvaluator" + ], + "evaluations": [ + { + "id": "billing-refund-calm", + "name": "Billing - calm refund request (auto-reply)", + "inputs": { + "subject": "Duplicate charge on my account", + "message": "Hi, I was charged twice for order A-104. Could you please refund the duplicate charge? Thanks for your help." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "billing" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "billing-invoice-question", + "name": "Billing - invoice clarification (auto-reply)", + "inputs": { + "subject": "Question about my invoice", + "message": "I noticed my invoice this month includes a charge I do not recognize. Could you clarify the payment breakdown? Thanks." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "billing" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "technical-bug-report", + "name": "Technical - routine bug report (auto-reply)", + "inputs": { + "subject": "App crashes on login", + "message": "The app crashes every time I try to log in. This started after the last update. Please help me fix this bug." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "technical" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "technical-outage-urgent", + "name": "Technical - urgent outage (escalate: high urgency)", + "inputs": { + "subject": "URGENT: service down", + "message": "Our integration has been down for 3 days and this is unacceptable. We need this fixed immediately, it is critical." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "technical" }, + "EscalationDecisionEvaluator": { "expectedClass": "true" } + } + }, + { + "id": "sales-pricing-question", + "name": "Sales - pricing inquiry (auto-reply)", + "inputs": { + "subject": "Question about pricing plans", + "message": "Hi, I would like a quote for upgrading to your premium plan. Can you share pricing details?" + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "sales" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "sales-demo-request", + "name": "Sales - demo request (auto-reply)", + "inputs": { + "subject": "Requesting a demo", + "message": "We are considering your product and would like to schedule a demo to see the upgrade options." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "sales" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "billing-angry-customer", + "name": "Billing - angry customer (escalate: high frustration)", + "inputs": { + "subject": "This is ridiculous", + "message": "I have been charged the wrong amount again and I am furious, this is unacceptable and terrible service." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "billing" }, + "EscalationDecisionEvaluator": { "expectedClass": "true" } + } + }, + { + "id": "technical-calm-minor-bug", + "name": "Technical - minor, no-rush bug (auto-reply)", + "inputs": { + "subject": "Minor bug report", + "message": "Just a small bug: the export button is not working as expected. No rush, whenever you get a chance, thanks." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "technical" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "ambiguous-low-confidence", + "name": "Ambiguous ticket (escalate: low routing confidence)", + "inputs": { + "subject": "Quick question", + "message": "Can someone tell me more about how things work around here?" + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "billing" }, + "EscalationDecisionEvaluator": { "expectedClass": "true" } + } + }, + { + "id": "sales-demo-request-wrong-department", + "name": "Sales - demo request (DELIBERATELY WRONG: ground truth department set to 'technical')", + "inputs": { + "subject": "Requesting a demo", + "message": "We are considering your product and would like to schedule a demo to see the upgrade options." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "technical" }, + "EscalationDecisionEvaluator": { "expectedClass": "false" } + } + }, + { + "id": "billing-refund-calm-wrong-escalation", + "name": "Billing - calm refund request (DELIBERATELY WRONG: ground truth escalation set to 'true')", + "inputs": { + "subject": "Duplicate charge on my account", + "message": "Hi, I was charged twice for order A-104. Could you please refund the duplicate charge? Thanks for your help." + }, + "evaluationCriterias": { + "DepartmentRoutingEvaluator": { "expectedClass": "billing" }, + "EscalationDecisionEvaluator": { "expectedClass": "true" } + } + } + ] +} diff --git a/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/department-routing.json b/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/department-routing.json new file mode 100644 index 000000000..ef9723183 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/department-routing.json @@ -0,0 +1,17 @@ +{ + "version": "1.0", + "id": "DepartmentRoutingEvaluator", + "name": "DepartmentRoutingEvaluator", + "description": "Checks whether Tier 1 (Jev) routed the ticket to the expected department", + "evaluatorTypeId": "uipath-multiclass-classification", + "evaluatorConfig": { + "name": "DepartmentRoutingEvaluator", + "targetOutputKey": "triage.department", + "classes": ["billing", "technical", "sales"], + "metricType": "precision", + "averaging": "macro", + "defaultEvaluationCriteria": { + "expectedClass": "technical" + } + } +} diff --git a/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/escalation-decision.json b/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/escalation-decision.json new file mode 100644 index 000000000..857cf32d6 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/evaluations/evaluators/escalation-decision.json @@ -0,0 +1,16 @@ +{ + "version": "1.0", + "id": "EscalationDecisionEvaluator", + "name": "EscalationDecisionEvaluator", + "description": "Checks whether the agent escalated to a human (Action Center) exactly when it should have", + "evaluatorTypeId": "uipath-binary-classification", + "evaluatorConfig": { + "name": "EscalationDecisionEvaluator", + "targetOutputKey": "escalated", + "positiveClass": "true", + "metricType": "f-score", + "defaultEvaluationCriteria": { + "expectedClass": "false" + } + } +} diff --git a/packages/uipath/samples/ticket-triage-agent/input.json b/packages/uipath/samples/ticket-triage-agent/input.json new file mode 100644 index 000000000..0bf583fdf --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/input.json @@ -0,0 +1,4 @@ +{ + "subject": "Duplicate charge on my account", + "message": "Hi, I was charged twice for order A-104. Could you please refund the duplicate charge? Thanks for your help." +} diff --git a/packages/uipath/samples/ticket-triage-agent/main.py b/packages/uipath/samples/ticket-triage-agent/main.py new file mode 100644 index 000000000..080c6f881 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/main.py @@ -0,0 +1,241 @@ +"""Two-tier support-ticket triage agent. + +Demonstrates a "System 1 / System 2" pattern: + +1. Tier 1 (System 1) - a fast, cheap, structured-decision model triages the + ticket: which department it belongs to, whether it's urgent, and how + frustrated the customer sounds. Uses TypeSafe AI's `typesafe-sdk` and + its "Jev" System One model (requires TYPESAFE_API_KEY; see README.md). +2. Tier 2 (System 2) - branches on the triage result: + * High urgency/frustration, or low routing confidence -> escalate to a + human via a UiPath Action Center QuickForm task (HITL), with the + Tier 1 decision attached as context. + * Otherwise -> call a real LLM (UiPath LLM Gateway) to draft a reply + for the routed department and mark the ticket resolved. +""" + +from __future__ import annotations + +import asyncio +import os + +from pydantic import BaseModel, Field +from typesafe_sdk import Choice, Noul, Score, TypeSafeClient + +from uipath.platform import UiPath +from uipath.platform.chat import ChatModels +from uipath.tracing import traced + +# --- Tunable escalation thresholds ----------------------------------------- +URGENCY_THRESHOLD = 0.6 +FRUSTRATION_THRESHOLD_FRACTION = 0.5 # fraction of the frustration scale +ROUTING_CONFIDENCE_THRESHOLD = 0.45 + +# Fixed schema key for the QuickForm task this sample registers/reuses. +TRIAGE_TASK_SCHEMA_KEY = "5b6f7e2a-3c9d-4e11-9a2b-6d1f0c9a2e77" + +TRIAGE_TASK_SCHEMA = { + "id": TRIAGE_TASK_SCHEMA_KEY, + "fields": [ + {"id": "subject", "type": "text", "label": "Subject", "direction": "input"}, + {"id": "message", "type": "text", "label": "Message", "direction": "input"}, + { + "id": "department", + "type": "text", + "label": "Suggested department", + "direction": "input", + }, + { + "id": "urgency", + "type": "text", + "label": "Urgency score", + "direction": "input", + }, + { + "id": "frustration", + "type": "text", + "label": "Frustration score", + "direction": "input", + }, + { + "id": "reply", + "type": "text", + "label": "Reviewer reply", + "direction": "output", + }, + ], + "outcomes": [ + {"id": "resolve", "name": "Resolve", "type": "string", "isPrimary": True}, + ], +} + +DEPARTMENT_SYSTEM_PROMPTS = { + "billing": ( + "You are a billing support agent. Write a short, polite reply that " + "acknowledges the customer's billing issue and explains next steps." + ), + "technical": ( + "You are a technical support agent. Write a short, polite reply " + "acknowledging the technical issue and the troubleshooting steps " + "that will follow." + ), + "sales": ( + "You are a sales representative. Write a short, polite reply " + "addressing the customer's pricing or account question." + ), +} + + +class TicketInput(BaseModel): + """A support ticket to triage.""" + + subject: str = Field(description="Ticket subject line") + message: str = Field(description="Ticket body / customer message") + + +class TriageDecision(BaseModel): + """Tier 1 (Jev) structured triage output.""" + + department: str + department_confidence: float = Field(ge=0.0, le=1.0) + urgency: float = Field( + ge=0.0, le=1.0, description="Calibrated probability that the ticket is urgent" + ) + frustration_score: float = Field( + ge=0.0, description="Rubric-weighted frustration level, 0..frustration_scale" + ) + frustration_scale: int = Field( + ge=1, description="Maximum value frustration_score can take" + ) + + +class TicketOutput(BaseModel): + """Final agent output.""" + + triage: TriageDecision + escalated: bool + auto_reply: str | None = Field( + default=None, description="LLM-drafted reply, when auto-handled" + ) + action_task_id: int | None = Field( + default=None, description="Action Center task id, when escalated" + ) + + +@traced() +def triage_ticket(ticket: TicketInput) -> TriageDecision: + """Run the fast Tier 1 structured triage over the ticket.""" + client = TypeSafeClient() + response = client.system_one( + state={"subject": ticket.subject, "message": ticket.message}, + questions={ + "department": Choice( + instructions="Which team should handle this ticket?", + criteria={ + "billing": "Payments, invoicing, refunds, subscriptions", + "technical": "Bugs, outages, integrations, errors", + "sales": "Pricing, upgrades, new accounts, demos", + }, + ), + "is_urgent": Noul(instructions="Does this convey urgency?"), + "frustration": Score( + instructions="How frustrated does the customer sound?", + criteria=["Calm", "Frustrated", "Very angry"], + ), + }, + ) + department = response.answers["department"] + is_urgent = response.answers["is_urgent"] + frustration = response.answers["frustration"] + return TriageDecision( + department=department.choice, + department_confidence=department.confidence, + urgency=is_urgent.noul, + frustration_score=frustration.score, + frustration_scale=len(frustration.probabilities) - 1, + ) + + +def needs_escalation(triage: TriageDecision) -> bool: + """Decide whether the ticket should go to a human instead of auto-reply.""" + return ( + triage.urgency >= URGENCY_THRESHOLD + or (triage.frustration_score / triage.frustration_scale) + >= FRUSTRATION_THRESHOLD_FRACTION + or triage.department_confidence < ROUTING_CONFIDENCE_THRESHOLD + ) + + +@traced() +def escalate_to_human( + client: UiPath, ticket: TicketInput, triage: TriageDecision +) -> int: + """Create an Action Center QuickForm task for a human reviewer.""" + folder_path = os.environ.get("UIPATH_FOLDER_PATH", "").strip() + if not folder_path: + raise RuntimeError( + "UIPATH_FOLDER_PATH is not set. Action Center tasks must be " + "created in an Orchestrator folder; set it in .env (see " + ".env.example)." + ) + + task = client.tasks.create_quickform( + title=f"Review ticket: {ticket.subject}", + task_schema_key=TRIAGE_TASK_SCHEMA_KEY, + schema=TRIAGE_TASK_SCHEMA, + data={ + "subject": ticket.subject, + "message": ticket.message, + "department": f"{triage.department} ({triage.department_confidence:.0%} confidence)", + "urgency": f"{triage.urgency:.2f}", + "frustration": f"{triage.frustration_score:.2f} / {triage.frustration_scale}", + }, + priority="High" if triage.urgency >= URGENCY_THRESHOLD else "Medium", + folder_path=folder_path, + ) + if task.id is None: + raise RuntimeError( + "Action Center did not return a task id for the created task." + ) + return task.id + + +@traced() +async def draft_auto_reply( + client: UiPath, ticket: TicketInput, triage: TriageDecision +) -> str: + """Use a real LLM to draft a reply for a routine, non-urgent ticket.""" + system_prompt = DEPARTMENT_SYSTEM_PROMPTS.get( + triage.department, DEPARTMENT_SYSTEM_PROMPTS["technical"] + ) + result = await client.llm.chat_completions( + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": f"Subject: {ticket.subject}\n\n{ticket.message}", + }, + ], + model=ChatModels.gpt_4_1_mini_2025_04_14, + max_tokens=300, + temperature=0.3, + ) + if not result.choices or not (content := result.choices[0].message.content): + raise RuntimeError( + "LLM Gateway returned no reply text; refusing to auto-resolve this ticket." + ) + return content + + +@traced() +async def main(input: TicketInput) -> TicketOutput: + """Triage a ticket and either auto-reply or escalate to a human.""" + triage = await asyncio.to_thread(triage_ticket, input) + client = UiPath() + + if needs_escalation(triage): + task_id = await asyncio.to_thread(escalate_to_human, client, input, triage) + return TicketOutput(triage=triage, escalated=True, action_task_id=task_id) + + reply = await draft_auto_reply(client, input, triage) + return TicketOutput(triage=triage, escalated=False, auto_reply=reply) diff --git a/packages/uipath/samples/ticket-triage-agent/pyproject.toml b/packages/uipath/samples/ticket-triage-agent/pyproject.toml new file mode 100644 index 000000000..97296a704 --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "ticket-triage-agent" +version = "0.0.1" +description = "Two-tier support ticket triage: TypeSafe AI's Jev System One model for fast structured routing, escalating to Action Center HITL or an LLM-drafted auto-reply." +requires-python = ">=3.11" +dependencies = [ + "uipath", + "typesafe-sdk>=0.7.0", +] + +[dependency-groups] +dev = [ + "uipath-dev", +] diff --git a/packages/uipath/samples/ticket-triage-agent/uipath.json b/packages/uipath/samples/ticket-triage-agent/uipath.json new file mode 100644 index 000000000..63268db8b --- /dev/null +++ b/packages/uipath/samples/ticket-triage-agent/uipath.json @@ -0,0 +1,5 @@ +{ + "agents": { + "main": "main.py:main" + } +} \ No newline at end of file