From 0803fdb7226b9c3b9c0ab2f84c4ba9f889da5f0b Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Thu, 2 Apr 2026 23:20:31 +0200 Subject: [PATCH] feat: add confidence_scorer prompt template (internal self-evaluation) --- .../prompts/templates/confidence_scorer.yaml | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 packages/gateway/prompts/templates/confidence_scorer.yaml diff --git a/packages/gateway/prompts/templates/confidence_scorer.yaml b/packages/gateway/prompts/templates/confidence_scorer.yaml new file mode 100644 index 0000000..eafa923 --- /dev/null +++ b/packages/gateway/prompts/templates/confidence_scorer.yaml @@ -0,0 +1,154 @@ +id: confidence_scorer +version: "1.0.0" +task_type: confidence_scorer +description: Self-assessment of output quality — the LLM evaluates its own generated output for confidence and potential issues +model_preference: qwen2.5:7b +model_minimum: qwen2.5:3b +temperature: 0.1 +max_tokens: 512 +output_format: json + +system_prompt: | + You are a quality self-assessment module for the LLM Gateway. + Your task is to evaluate an LLM-generated output and provide a confidence score and quality assessment. + This output is used downstream to decide whether to serve the result, request regeneration, or flag for human review. + + Return ONLY valid JSON: + { + "confidence": 1-10, + "confidence_reasoning": "string — why this score", + "completeness": 1-10, + "potential_issues": [ + { + "issue": "string — specific concern", + "severity": "critical|high|medium|low", + "type": "factual|hallucination|format|completeness|relevance|safety" + } + ], + "hallucination_risk": "high|medium|low", + "hallucination_indicators": ["string or empty"], + "format_correct": true|false, + "format_notes": "string or null", + "recommend_action": "serve|regenerate|human_review|discard", + "recommend_action_reason": "string" + } + + Confidence scoring: + - 9-10: Output is complete, factually grounded, format correct, no visible issues + - 7-8: Output is good, minor format or completeness issues, low hallucination risk + - 5-6: Output has notable issues — missing data, possibly hallucinated details, format problems + - 3-4: Significant problems — likely hallucination, major format violation, incomplete + - 1-2: Output is unreliable — high hallucination risk, format wrong, context likely missed + + Hallucination risk indicators: + - Specific numbers, dates, or names not present in the input context + - Technical specifications stated without source in context + - Confident claims about proprietary system internals (e.g., exact vendor pricing without source) + - Made-up examples that look plausible but weren't in the input + + recommend_action: + - serve: confidence >= 7, no critical issues + - regenerate: confidence 4-6, or format incorrect but fixable by retry + - human_review: safety issue, critical factual concern, or hallucination suspected + - discard: confidence < 4, or output is harmful/nonsensical + + Be honest and critical. The gateway uses this score for routing decisions. + +user_template: | + Task type: {{task_type}} + Original input (summary): {{input_summary}} + + Generated output to evaluate: + {{generated_output}} + + Assess quality: + +few_shot_examples: + - user: | + Task type: tip_transceiver_enrich + Original input (summary): HTML product page for SFP-10G-LR with specs: 1310nm, 10km, LC, SMF, DOM, -8.2 to 0.5 dBm TX, -14.4 dBm RX sensitivity + + Generated output to evaluate: + { + "part_number": "SFP-10G-LR", + "form_factor": "SFP+", + "data_rate_gbps": 10, + "wavelength_nm": 1310, + "reach_m": 10000, + "connector": "LC", + "fiber_type": "SMF", + "tx_power_min_dbm": -8.2, + "tx_power_max_dbm": 0.5, + "rx_sensitivity_dbm": -14.4, + "temperature_min_c": 0, + "temperature_max_c": 70, + "power_consumption_w": 1.0, + "vendor_compatibility": ["Cisco", "Juniper", "Arista"], + "dom_support": true, + "standards_compliance": ["IEEE 802.3ae", "SFF-8431"], + "description": "10GBASE-LR SFP+ transceiver for 10G Ethernet over single-mode fiber at 1310nm, supporting distances up to 10km with LC duplex connectors.", + "extraction_confidence": 9 + } + assistant: | + { + "confidence": 9, + "confidence_reasoning": "All extracted values are directly present in the source HTML. No values were inferred or invented. Format is correct JSON matching the expected schema.", + "completeness": 9, + "potential_issues": [ + { + "issue": "wavelength_channels field is absent — for LR (single-wavelength) this is correct (null), but schema requires explicit null", + "severity": "low", + "type": "completeness" + } + ], + "hallucination_risk": "low", + "hallucination_indicators": [], + "format_correct": true, + "format_notes": "JSON is well-formed and matches expected schema. Missing wavelength_channels (should be null, not omitted).", + "recommend_action": "serve", + "recommend_action_reason": "High confidence, factually grounded, format correct. Minor completeness note does not affect usability." + } + + - user: | + Task type: tip_faq_answer + Original input (summary): Question: "What is the reach of QSFP-100G-LR4?" — No TIP context provided in the call. + + Generated output to evaluate: + The QSFP-100G-LR4 supports a reach of 10 km over single-mode fiber. It uses four CWDM lanes at 1295, 1300, 1304, and 1309 nm. The module draws approximately 3.5W and supports DOM. Cisco's QSFP-100G-LR4 is their most popular 100G module, shipping over 2 million units in 2023 alone. It is compatible with Nexus 9000 series switches. + assistant: | + { + "confidence": 5, + "confidence_reasoning": "The first two sentences are factually correct (10km reach, CWDM lanes, ~3.5W). However, the claim '2 million units in 2023' is not from the input context — no TIP database context was provided for this call. This is likely hallucinated.", + "completeness": 7, + "potential_issues": [ + { + "issue": "'shipping over 2 million units in 2023' — specific sales volume not provided in context. High risk of hallucination.", + "severity": "high", + "type": "hallucination" + }, + { + "issue": "No TIP context was provided — answer relies on model's training data for all specs. Should flag this to the user.", + "severity": "medium", + "type": "factual" + } + ], + "hallucination_risk": "medium", + "hallucination_indicators": [ + "Specific shipment volume ('2 million units in 2023') not present in any input context", + "No TIP database context was injected — all facts come from model training data" + ], + "format_correct": true, + "format_notes": "Output is plain text, format is correct. Length is within bounds.", + "recommend_action": "human_review", + "recommend_action_reason": "Likely hallucinated shipment volume claim. Recommend either stripping the unverified claim or flagging output as 'unverified — no database context provided'." + } + +variables: + - task_type + - input_summary + - generated_output + - few_shot_examples + +validation_rules: + output_must_be_json: true + required_fields: ["confidence", "potential_issues", "hallucination_risk", "recommend_action"]