/**
 * Crawler LLM — TIPLLM Training Data Writer.
 *
 * Converts validated transceiver extractions and crawl events into SFT training
 * pairs, appends them to JSONL files in the local tip-training-data git clone,
 * and pushes to Gitea after each batch.
 *
 * Training pair types generated:
 *   spec_qa         — "What are the specs of [PID]?" → structured answer
 *   crawl_reasoning — "How did you extract X from this HTML?" → CoT trace
 *   validation      — "Is this spec physically plausible?" → yes/no + reasoning
 *   discovery       — "Where can I find [vendor]'s transceiver catalog?" → nav guidance
 *
 * Gitea repo: http://192.168.178.196:3000/rene/tip-training-data
 * Local clone: /tmp/tip-training-data (pre-cloned with token auth remote)
 */

import { execSync } from "child_process";
import { appendFileSync, mkdirSync, existsSync, writeFileSync } from "fs";
import { join } from "path";
import { createHash } from "crypto";

import type { ExtractedSpec } from "./spec-validator";
import type { CombinedValidationResult } from "./spec-validator";

// ─────────────────────────────────────────────────────────────────────────────
// Config
// ─────────────────────────────────────────────────────────────────────────────

const REPO_DIR = process.env.TIP_TRAINING_REPO || "/tmp/tip-training-data";
const GITEA_TOKEN = process.env.GITEA_TOKEN || "";
const GITEA_BASE = process.env.GITEA_BASE || "http://192.168.178.196:3000";
const GITEA_REPO = process.env.TIP_TRAINING_GITEA_REPO || "rene/tip-training-data";

// Minimum confidence for a spec to enter the high-quality training set
const MIN_CONFIDENCE_HIGH = 0.75;
const MIN_CONFIDENCE_LOW = 0.50;

const SYSTEM_PROMPT = `You are TIPLLM, an expert AI assistant for the Transceiver Intelligence Platform (TIP). \
You have deep knowledge of optical transceiver specifications, form factors, IEEE standards, \
vendor product catalogs, and fiber optic networking. You help engineers select, source, and \
validate transceivers. You provide precise, structured answers with confidence scores and \
always cite your reasoning.`;

// ─────────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────────

export interface SftMessage {
  role: "system" | "user" | "assistant";
  content: string;
}

export interface SftRecord {
  id: string;
  source: string;
  kind: "sft-jsonl";
  messages: SftMessage[];
}

export interface CrawlExtraction {
  url: string;
  vendor_slug: string;
  vendor_name: string;
  spec: ExtractedSpec;
  validation: CombinedValidationResult;
  raw_html_snippet?: string;   // first 2000 chars of cleaned HTML for CoT training
  crawled_at: string;          // ISO timestamp
}

export interface RobotExperience {
  event: "status" | "tipllm_plan" | "queue_plan" | "queue_enqueued" | "queue_rejected" | "crawler_result";
  observed_at: string;
  actor: string;
  profile?: string;
  wave?: string;
  vendor?: string;
  summary: string;
  input: Record<string, unknown>;
  decision: Record<string, unknown>;
  outcome?: Record<string, unknown>;
  safety_notes?: string[];
}

// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────

function makeId(prefix: string, input: string): string {
  const hash = createHash("sha256").update(input).digest("hex").slice(0, 12);
  return `${prefix}-${hash}`;
}

function specToMarkdown(spec: ExtractedSpec): string {
  const lines: string[] = [];
  if (spec.part_number)   lines.push(`- **Part Number**: ${spec.part_number}`);
  if (spec.form_factor)   lines.push(`- **Form Factor**: ${spec.form_factor}`);
  if (spec.speed_gbps)    lines.push(`- **Speed**: ${spec.speed_gbps}G`);
  if (spec.fiber_type)    lines.push(`- **Fiber Type**: ${spec.fiber_type}`);
  if (spec.connector)     lines.push(`- **Connector**: ${spec.connector}`);
  if (spec.wavelengths)   lines.push(`- **Wavelengths**: ${spec.wavelengths}`);
  if (spec.reach_meters)  lines.push(`- **Reach**: ${spec.reach_meters >= 1000 ? `${spec.reach_meters / 1000}km` : `${spec.reach_meters}m`}`);
  if (spec.ieee_standard) lines.push(`- **IEEE Standard**: ${spec.ieee_standard}`);
  if (spec.dom_support != null) lines.push(`- **DOM**: ${spec.dom_support ? "Yes" : "No"}`);
  return lines.join("\n");
}

function ensureDir(dir: string): void {
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
}

function ensureTrainingRepoLayout(): void {
  ensureDir(REPO_DIR);
  ensureDir(join(REPO_DIR, "qa-pairs"));
  ensureDir(join(REPO_DIR, "robot-experiences"));
  ensureDir(join(REPO_DIR, "crawl-extractions"));
  ensureDir(join(REPO_DIR, "validated-specs"));

  const readme = join(REPO_DIR, "README.md");
  if (!existsSync(readme)) {
    writeFileSync(
      readme,
      [
        "# TIP Training Data Pool",
        "",
        "This repository stores TIPLLM learning records produced by TIP crawlers, scrapers and verification robots.",
        "",
        "Primary lanes:",
        "",
        "- `qa-pairs/`: supervised fine-tuning JSONL records with chat messages.",
        "- `robot-experiences/`: raw operational experience logs for audit and future dataset builders.",
        "- `crawl-extractions/`: crawler extraction audit logs.",
        "- `validated-specs/`: validated structured transceiver specifications.",
        "",
        "Only TIPLLM should be used for crawler planning and extraction feedback in this lane.",
        "",
      ].join("\n"),
      "utf8",
    );
  }
}

function appendRecord(filePath: string, record: SftRecord): void {
  ensureTrainingRepoLayout();
  appendFileSync(filePath, JSON.stringify(record) + "\n", "utf8");
}

// ─────────────────────────────────────────────────────────────────────────────
// Training pair generators
// ─────────────────────────────────────────────────────────────────────────────

function makeSpecQaPair(extraction: CrawlExtraction): SftRecord | null {
  const { spec, vendor_name, url, validation } = extraction;
  if (!spec.part_number) return null;

  const specMd = specToMarkdown(spec);
  if (!specMd) return null;

  const confNote = validation.adjusted_confidence >= MIN_CONFIDENCE_HIGH
    ? "high confidence"
    : `confidence ${validation.adjusted_confidence.toFixed(2)}`;

  return {
    id: makeId("spec-qa", spec.part_number + url),
    source: `crawl:${extraction.vendor_slug}:${url}`,
    kind: "sft-jsonl",
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      {
        role: "user",
        content: `What are the technical specifications for transceiver ${spec.part_number}?`,
      },
      {
        role: "assistant",
        content: `**${spec.part_number}** (${vendor_name})\n\n${specMd}\n\n*Source: [${vendor_name} product page](${url}) — ${confNote}*`,
      },
    ],
  };
}

function makeCrawlReasoningPair(extraction: CrawlExtraction): SftRecord | null {
  const { spec, url, validation, raw_html_snippet } = extraction;
  if (!raw_html_snippet || !spec.part_number) return null;

  const issues = [
    ...validation.spec_errors.map((e) => `ERROR: ${e}`),
    ...validation.spec_warnings.map((w) => `WARN: ${w}`),
  ];

  return {
    id: makeId("crawl-cot", url),
    source: `crawl-reasoning:${extraction.vendor_slug}:${url}`,
    kind: "sft-jsonl",
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      {
        role: "user",
        content: `I have this HTML snippet from ${url}:\n\n---\n${raw_html_snippet.slice(0, 1500)}\n---\n\nExtract the transceiver part number, form factor, speed, fiber type, wavelength, and reach.`,
      },
      {
        role: "assistant",
        content: `From the HTML I can extract:\n\n${specToMarkdown(spec)}\n\n**Validation** (confidence: ${validation.adjusted_confidence.toFixed(2)}, tier: ${validation.tier}):\n${issues.length > 0 ? issues.map((i) => `- ${i}`).join("\n") : "- All checks passed."}\n\n**Source evidence**: The part number "${spec.part_number}" appears in a product header or SKU field. Speed and form factor are confirmed by the specification table.`,
      },
    ],
  };
}

function makeValidationPair(extraction: CrawlExtraction): SftRecord {
  const { spec, validation, url } = extraction;

  const question = `Is the following transceiver specification physically plausible?\n\n${specToMarkdown(spec)}\n\nSource: ${url}`;

  let answer: string;
  if (validation.passed) {
    const issues = validation.spec_warnings.length > 0
      ? `\n\n**Warnings** (non-blocking):\n${validation.spec_warnings.map((w) => `- ${w}`).join("\n")}`
      : "";
    answer = `**Valid** (tier: ${validation.tier}, confidence: ${validation.adjusted_confidence.toFixed(2)})\n\nAll critical physical checks pass — form factor/speed compatibility, fiber type/wavelength consistency, and reach limits are all within bounds.${issues}`;
  } else {
    const errList = validation.spec_errors.map((e) => `- ❌ ${e}`).join("\n");
    answer = `**Invalid** (tier: ${validation.tier})\n\nThe following physical plausibility checks failed:\n\n${errList}\n\nThis data should not be persisted to the transceiver database without manual review.`;
  }

  return {
    id: makeId("spec-valid", JSON.stringify(spec) + url),
    source: `validation:${extraction.vendor_slug}:${url}`,
    kind: "sft-jsonl",
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      { role: "user", content: question },
      { role: "assistant", content: answer },
    ],
  };
}

function makeDiscoveryPair(vendorSlug: string, vendorName: string, catalogUrl: string, productCount: number): SftRecord {
  return {
    id: makeId("discovery", vendorSlug + catalogUrl),
    source: `discovery:${vendorSlug}`,
    kind: "sft-jsonl",
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      {
        role: "user",
        content: `Where can I find ${vendorName}'s transceiver catalog to scrape product specifications?`,
      },
      {
        role: "assistant",
        content: `**${vendorName} Transceiver Catalog**\n\n- Catalog URL: ${catalogUrl}\n- Products discovered: ${productCount}\n- Crawl strategy: Navigate to the catalog URL, identify product listing pages, follow individual product links to extract SKU, form factor, speed, fiber type, wavelength, and reach specifications.\n\nWhen crawling ${vendorName}, look for product table structures, data sheets linked as PDFs, and compatibility matrices in the navigation sidebar.`,
      },
    ],
  };
}

// ─────────────────────────────────────────────────────────────────────────────
// File path routing
// ─────────────────────────────────────────────────────────────────────────────

function getOutputFile(type: "spec_qa" | "crawl_reasoning" | "validation" | "discovery", tier: string): string {
  const dir = join(REPO_DIR, "qa-pairs");
  ensureDir(dir);
  return join(dir, `${type}-${tier}.jsonl`);
}

function getRawExtractionFile(vendorSlug: string): string {
  ensureTrainingRepoLayout();
  const dir = join(REPO_DIR, "crawl-extractions", vendorSlug);
  ensureDir(dir);
  const date = new Date().toISOString().split("T")[0];
  return join(dir, `${date}.jsonl`);
}

function getValidatedSpecFile(tier: string): string {
  ensureTrainingRepoLayout();
  const dir = join(REPO_DIR, "validated-specs");
  ensureDir(dir);
  return join(dir, `${tier}.jsonl`);
}

// ─────────────────────────────────────────────────────────────────────────────
// Git operations
// ─────────────────────────────────────────────────────────────────────────────

let pendingChanges = 0;
const BATCH_COMMIT_THRESHOLD = 50;  // commit every N records

function gitAddAll(): void {
  execSync("git add -A", { cwd: REPO_DIR, stdio: "pipe" });
}

function gitCommit(message: string): void {
  try {
    execSync(
      `git -c user.email="tip-crawler@context-x.org" -c user.name="TIP Crawler" commit -m "${message}"`,
      { cwd: REPO_DIR, stdio: "pipe" }
    );
  } catch {
    // Empty commit (nothing new) — ignore
  }
}

function gitPush(): void {
  if (!GITEA_TOKEN) {
    execSync("git push origin main", { cwd: REPO_DIR, stdio: "pipe" });
  } else {
    const remote = `http://rene:${GITEA_TOKEN}@${GITEA_BASE.replace("http://", "")}/${GITEA_REPO}.git`;
    execSync(`git push "${remote}" main`, { cwd: REPO_DIR, stdio: "pipe" });
  }
}

export function flushToGitea(label = "batch"): void {
  try {
    ensureTrainingRepoLayout();
    gitAddAll();
    gitCommit(`crawl: add ${label} training records [${new Date().toISOString()}]`);
    gitPush();
    pendingChanges = 0;
    console.log(`[training-writer] Pushed to Gitea: ${label}`);
  } catch (err) {
    console.warn(`[training-writer] Git push failed (non-fatal): ${(err as Error).message.slice(0, 120)}`);
  }
}

function makeRobotPair(experience: RobotExperience): SftRecord {
  const prompt = [
    "You are TIPLLM controlling TIP crawler and verification robots.",
    "Given this current observation, choose the safest next robot action.",
    "",
    JSON.stringify({
      event: experience.event,
      actor: experience.actor,
      profile: experience.profile,
      wave: experience.wave,
      vendor: experience.vendor,
      input: experience.input,
      safety_notes: experience.safety_notes ?? [],
    }, null, 2),
  ].join("\n");

  const response = {
    summary: experience.summary,
    decision: experience.decision,
    outcome: experience.outcome ?? null,
    constraints: [
      "Use TIPLLM only for planning/extraction feedback.",
      "Do not start heavy crawler waves on Erik.",
      "Prefer small, observable queue batches.",
      "Write useful outcomes back into the TIPLLM training pool.",
    ],
  };

  return {
    id: makeId("robot-exp", JSON.stringify(experience)),
    source: `robot-experience:${experience.event}:${experience.actor}`,
    kind: "sft-jsonl",
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      { role: "user", content: prompt },
      { role: "assistant", content: JSON.stringify(response, null, 2) },
    ],
  };
}

function getRobotExperienceFile(): string {
  ensureTrainingRepoLayout();
  const date = new Date().toISOString().split("T")[0];
  return join(REPO_DIR, "robot-experiences", `${date}.jsonl`);
}

/**
 * Write one operational crawler/robot experience to the TIPLLM training pool.
 * This produces both a raw audit row and an SFT chat row under qa-pairs.
 */
export function writeRobotExperience(experience: RobotExperience, options: { flush?: boolean } = {}): void {
  ensureTrainingRepoLayout();
  appendFileSync(getRobotExperienceFile(), JSON.stringify(experience) + "\n", "utf8");
  appendRecord(join(REPO_DIR, "qa-pairs", "robot-control-high.jsonl"), makeRobotPair(experience));
  pendingChanges += 2;

  if (options.flush || pendingChanges >= BATCH_COMMIT_THRESHOLD) {
    flushToGitea(`robot-${experience.event}`);
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// Public API
// ─────────────────────────────────────────────────────────────────────────────

/**
 * Write a single crawl extraction to all appropriate training files.
 * Skips if confidence is below the minimum threshold.
 */
export function writeExtractionRecord(extraction: CrawlExtraction): {
  written: boolean;
  pairs: number;
  reason?: string;
} {
  const { validation, spec } = extraction;

  // Reject very low confidence
  if (validation.adjusted_confidence < MIN_CONFIDENCE_LOW) {
    return { written: false, pairs: 0, reason: `confidence ${validation.adjusted_confidence} < threshold` };
  }

  const tier = validation.tier === "rejected" ? "low" : validation.tier;
  let pairsWritten = 0;

  // 1. Raw extraction log (always, for audit)
  appendFileSync(
    getRawExtractionFile(extraction.vendor_slug),
    JSON.stringify({ ...extraction, raw_html_snippet: undefined }) + "\n",
    "utf8"
  );

  // 2. Validated spec archive
  if (validation.passed) {
    appendFileSync(
      getValidatedSpecFile(tier),
      JSON.stringify({ spec, url: extraction.url, vendor: extraction.vendor_slug, confidence: validation.adjusted_confidence }) + "\n",
      "utf8"
    );
  }

  // 3. Spec QA pair
  const qaPair = makeSpecQaPair(extraction);
  if (qaPair) {
    appendRecord(getOutputFile("spec_qa", tier), qaPair);
    pairsWritten++;
  }

  // 4. Crawl reasoning pair (CoT) — high tier only to avoid polluting with noisy traces
  if (tier === "high" || tier === "medium") {
    const cotPair = makeCrawlReasoningPair(extraction);
    if (cotPair) {
      appendRecord(getOutputFile("crawl_reasoning", tier), cotPair);
      pairsWritten++;
    }
  }

  // 5. Validation pair — always valuable (includes both passed and failed examples)
  if (spec.part_number || spec.form_factor) {
    const valPair = makeValidationPair(extraction);
    appendRecord(getOutputFile("validation", tier), valPair);
    pairsWritten++;
  }

  pendingChanges += pairsWritten;

  // Auto-flush when threshold reached
  if (pendingChanges >= BATCH_COMMIT_THRESHOLD) {
    flushToGitea(`auto-${extraction.vendor_slug}`);
  }

  return { written: true, pairs: pairsWritten };
}

/**
 * Write a vendor discovery record when we successfully crawl a new catalog.
 */
export function writeDiscoveryRecord(
  vendorSlug: string,
  vendorName: string,
  catalogUrl: string,
  productCount: number
): void {
  const pair = makeDiscoveryPair(vendorSlug, vendorName, catalogUrl, productCount);
  const file = getOutputFile("discovery", "high");
  appendRecord(file, pair);
  pendingChanges++;
}

/**
 * Force push all pending changes to Gitea (call at end of crawler run).
 */
export function finalFlush(vendorSlug: string): void {
  if (pendingChanges > 0) {
    flushToGitea(`final-${vendorSlug}`);
  }
}
