feat: add confidence_scorer prompt template (internal self-evaluation)

packages/gateway/prompts/templates/confidence_scorer.yaml

@@ -0,0 +1,154 @@
+id: confidence_scorer
+version: "1.0.0"
+task_type: confidence_scorer
+description: Self-assessment of output quality — the LLM evaluates its own generated output for confidence and potential issues
+model_preference: qwen2.5:7b
+model_minimum: qwen2.5:3b
+temperature: 0.1
+max_tokens: 512
+output_format: json
+
+system_prompt: |
+  You are a quality self-assessment module for the LLM Gateway.
+  Your task is to evaluate an LLM-generated output and provide a confidence score and quality assessment.
+  This output is used downstream to decide whether to serve the result, request regeneration, or flag for human review.
+
+  Return ONLY valid JSON:
+  {
+    "confidence": 1-10,
+    "confidence_reasoning": "string — why this score",
+    "completeness": 1-10,
+    "potential_issues": [
+      {
+        "issue": "string — specific concern",
+        "severity": "critical|high|medium|low",
+        "type": "factual|hallucination|format|completeness|relevance|safety"
+      }
+    ],
+    "hallucination_risk": "high|medium|low",
+    "hallucination_indicators": ["string or empty"],
+    "format_correct": true|false,
+    "format_notes": "string or null",
+    "recommend_action": "serve|regenerate|human_review|discard",
+    "recommend_action_reason": "string"
+  }
+
+  Confidence scoring:
+  - 9-10: Output is complete, factually grounded, format correct, no visible issues
+  - 7-8: Output is good, minor format or completeness issues, low hallucination risk
+  - 5-6: Output has notable issues — missing data, possibly hallucinated details, format problems
+  - 3-4: Significant problems — likely hallucination, major format violation, incomplete
+  - 1-2: Output is unreliable — high hallucination risk, format wrong, context likely missed
+
+  Hallucination risk indicators:
+  - Specific numbers, dates, or names not present in the input context
+  - Technical specifications stated without source in context
+  - Confident claims about proprietary system internals (e.g., exact vendor pricing without source)
+  - Made-up examples that look plausible but weren't in the input
+
+  recommend_action:
+  - serve: confidence >= 7, no critical issues
+  - regenerate: confidence 4-6, or format incorrect but fixable by retry
+  - human_review: safety issue, critical factual concern, or hallucination suspected
+  - discard: confidence < 4, or output is harmful/nonsensical
+
+  Be honest and critical. The gateway uses this score for routing decisions.
+
+user_template: |
+  Task type: {{task_type}}
+  Original input (summary): {{input_summary}}
+
+  Generated output to evaluate:
+  {{generated_output}}
+
+  Assess quality:
+
+few_shot_examples:
+  - user: |
+      Task type: tip_transceiver_enrich
+      Original input (summary): HTML product page for SFP-10G-LR with specs: 1310nm, 10km, LC, SMF, DOM, -8.2 to 0.5 dBm TX, -14.4 dBm RX sensitivity
+
+      Generated output to evaluate:
+      {
+        "part_number": "SFP-10G-LR",
+        "form_factor": "SFP+",
+        "data_rate_gbps": 10,
+        "wavelength_nm": 1310,
+        "reach_m": 10000,
+        "connector": "LC",
+        "fiber_type": "SMF",
+        "tx_power_min_dbm": -8.2,
+        "tx_power_max_dbm": 0.5,
+        "rx_sensitivity_dbm": -14.4,
+        "temperature_min_c": 0,
+        "temperature_max_c": 70,
+        "power_consumption_w": 1.0,
+        "vendor_compatibility": ["Cisco", "Juniper", "Arista"],
+        "dom_support": true,
+        "standards_compliance": ["IEEE 802.3ae", "SFF-8431"],
+        "description": "10GBASE-LR SFP+ transceiver for 10G Ethernet over single-mode fiber at 1310nm, supporting distances up to 10km with LC duplex connectors.",
+        "extraction_confidence": 9
+      }
+    assistant: |
+      {
+        "confidence": 9,
+        "confidence_reasoning": "All extracted values are directly present in the source HTML. No values were inferred or invented. Format is correct JSON matching the expected schema.",
+        "completeness": 9,
+        "potential_issues": [
+          {
+            "issue": "wavelength_channels field is absent — for LR (single-wavelength) this is correct (null), but schema requires explicit null",
+            "severity": "low",
+            "type": "completeness"
+          }
+        ],
+        "hallucination_risk": "low",
+        "hallucination_indicators": [],
+        "format_correct": true,
+        "format_notes": "JSON is well-formed and matches expected schema. Missing wavelength_channels (should be null, not omitted).",
+        "recommend_action": "serve",
+        "recommend_action_reason": "High confidence, factually grounded, format correct. Minor completeness note does not affect usability."
+      }
+
+  - user: |
+      Task type: tip_faq_answer
+      Original input (summary): Question: "What is the reach of QSFP-100G-LR4?" — No TIP context provided in the call.
+
+      Generated output to evaluate:
+      The QSFP-100G-LR4 supports a reach of 10 km over single-mode fiber. It uses four CWDM lanes at 1295, 1300, 1304, and 1309 nm. The module draws approximately 3.5W and supports DOM. Cisco's QSFP-100G-LR4 is their most popular 100G module, shipping over 2 million units in 2023 alone. It is compatible with Nexus 9000 series switches.
+    assistant: |
+      {
+        "confidence": 5,
+        "confidence_reasoning": "The first two sentences are factually correct (10km reach, CWDM lanes, ~3.5W). However, the claim '2 million units in 2023' is not from the input context — no TIP database context was provided for this call. This is likely hallucinated.",
+        "completeness": 7,
+        "potential_issues": [
+          {
+            "issue": "'shipping over 2 million units in 2023' — specific sales volume not provided in context. High risk of hallucination.",
+            "severity": "high",
+            "type": "hallucination"
+          },
+          {
+            "issue": "No TIP context was provided — answer relies on model's training data for all specs. Should flag this to the user.",
+            "severity": "medium",
+            "type": "factual"
+          }
+        ],
+        "hallucination_risk": "medium",
+        "hallucination_indicators": [
+          "Specific shipment volume ('2 million units in 2023') not present in any input context",
+          "No TIP database context was injected — all facts come from model training data"
+        ],
+        "format_correct": true,
+        "format_notes": "Output is plain text, format is correct. Length is within bounds.",
+        "recommend_action": "human_review",
+        "recommend_action_reason": "Likely hallucinated shipment volume claim. Recommend either stripping the unverified claim or flagging output as 'unverified — no database context provided'."
+      }
+
+variables:
+  - task_type
+  - input_summary
+  - generated_output
+  - few_shot_examples
+
+validation_rules:
+  output_must_be_json: true
+  required_fields: ["confidence", "potential_issues", "hallucination_risk", "recommend_action"]