/**
 * Crawlee instance-level storage isolation.
 *
 * WHY THIS EXISTS:
 * ----------------
 * All pg-boss workers run concurrently inside a single Node.js process.
 * The old approach (setting process.env.CRAWLEE_STORAGE_DIR) is a global
 * env-var mutation — if two workers run simultaneously, one worker's
 * writeToEnv() can overwrite another's before the crawler reads it.
 *
 * The fix: pass a `Configuration` instance directly to each Crawlee
 * constructor. This is instance-level (not global), so concurrent scrapers
 * each get their own isolated storage directory.
 *
 * Usage:
 *   import { makeCrawleeConfig } from "../utils/crawlee-config";
 *
 *   // In PlaywrightCrawler:
 *   const crawler = new PlaywrightCrawler({ ... }, makeCrawleeConfig("optcore"));
 *
 *   // In CheerioCrawler:
 *   const crawler = new CheerioCrawler({ ... }, makeCrawleeConfig("edgecore"));
 *
 *   // With explicit RequestQueue (community-issues pattern):
 *   const cfg = makeCrawleeConfig("community-issues");
 *   const queue = await RequestQueue.open(null, { config: cfg });
 *   const crawler = new CheerioCrawler({ requestQueue: queue, ... }, cfg);
 */

import { Configuration } from "crawlee";
import { join } from "node:path";
import { mkdirSync, existsSync, writeFileSync, rmSync, readdirSync, statSync } from "node:fs";

/**
 * Crawlee scratch space lives in /tmp/tip-crawlers/<name>/ (NOT next to the repo).
 * /tmp is tmpfs → survives process restarts but not server reboots.
 * Keeps build artefacts and persistent storage separate.
 */
const CRAWLEE_TMP_ROOT = process.env.CRAWLEE_TMP_ROOT ?? "/tmp/tip-crawlers";

/** Absolute path to the per-scraper Crawlee storage root on disk. */
export function crawleeStorageDir(scraperName: string): string {
  return join(CRAWLEE_TMP_ROOT, scraperName);
}

/**
 * Remove crawler temp dirs older than `maxAgeMs` (default 48 h).
 * Called automatically on each scraper start — keeps /tmp clean.
 */
export function cleanCrawleeTempDirs(maxAgeMs = 48 * 60 * 60 * 1_000): void {
  if (!existsSync(CRAWLEE_TMP_ROOT)) return;
  const cutoff = Date.now() - maxAgeMs;
  try {
    for (const entry of readdirSync(CRAWLEE_TMP_ROOT)) {
      const full = join(CRAWLEE_TMP_ROOT, entry);
      try {
        const st = statSync(full);
        if (st.isDirectory() && st.mtimeMs < cutoff) {
          rmSync(full, { recursive: true, force: true });
        }
      } catch { /* skip */ }
    }
  } catch { /* skip if /tmp not writable */ }
}

/**
 * Create and return a Crawlee Configuration with an isolated storageDir.
 *
 * Idempotent — safe to call every scraper run:
 *   - Directories are created if they don't exist (recursive: true).
 *   - SDK_SESSION_POOL_STATE.json is seeded once so Crawlee v3.16 doesn't
 *     throw "Could not find file" on first run (this version reads before
 *     writing on session-pool init).
 */
export function makeCrawleeConfig(scraperName: string): Configuration {
  const storageDir = crawleeStorageDir(scraperName);

  // Clean up stale tmp dirs from other scrapers before starting
  cleanCrawleeTempDirs();

  // Wipe the request queue from the previous run so Crawlee doesn't skip URLs
  // that were already marked as HANDLED (state=4, orderNo=null). The queue
  // persists between runs because purgeOnStart is false. Without this clear,
  // crawler.run(startUrls) deduplicates all start URLs → requestsTotal=0 →
  // immediate finish with 0 scraped pages. Session pool state lives in
  // key_value_stores/ (not request_queues/), so wiping the queue is safe.
  rmSync(join(storageDir, "request_queues", "default"), { recursive: true, force: true });

  // Pre-create internal directory tree
  mkdirSync(join(storageDir, "request_queues", "default"), { recursive: true });
  mkdirSync(join(storageDir, "datasets",       "default"), { recursive: true });
  mkdirSync(join(storageDir, "key_value_stores", "default"), { recursive: true });

  // Seed empty session-pool state to avoid "Could not find file" crash in v3.16
  const sessionFile = join(storageDir, "key_value_stores", "default", "SDK_SESSION_POOL_STATE.json");
  if (!existsSync(sessionFile)) {
    writeFileSync(sessionFile, JSON.stringify({
      usableSessionsCount: 0,
      retiredSessionsCount: 0,
      sessions: [],
    }));
  }

  // `localDataDirectory` is the MemoryStorage option for the base storage path.
  // Pass it via `storageClientOptions` so the Configuration uses this path
  // instance-locally (not via global CRAWLEE_STORAGE_DIR env var).
  return new Configuration({
    storageClientOptions: {
      localDataDirectory: storageDir,
      persistStorage: true,
    },
    purgeOnStart: false, // Preserve session pool state between runs
  });
}