115 lines
4.6 KiB
TypeScript
115 lines
4.6 KiB
TypeScript
/**
|
|
* Crawlee instance-level storage isolation.
|
|
*
|
|
* WHY THIS EXISTS:
|
|
* ----------------
|
|
* All pg-boss workers run concurrently inside a single Node.js process.
|
|
* The old approach (setting process.env.CRAWLEE_STORAGE_DIR) is a global
|
|
* env-var mutation — if two workers run simultaneously, one worker's
|
|
* writeToEnv() can overwrite another's before the crawler reads it.
|
|
*
|
|
* The fix: pass a `Configuration` instance directly to each Crawlee
|
|
* constructor. This is instance-level (not global), so concurrent scrapers
|
|
* each get their own isolated storage directory.
|
|
*
|
|
* Usage:
|
|
* import { makeCrawleeConfig } from "../utils/crawlee-config";
|
|
*
|
|
* // In PlaywrightCrawler:
|
|
* const crawler = new PlaywrightCrawler({ ... }, makeCrawleeConfig("optcore"));
|
|
*
|
|
* // In CheerioCrawler:
|
|
* const crawler = new CheerioCrawler({ ... }, makeCrawleeConfig("edgecore"));
|
|
*
|
|
* // With explicit RequestQueue (community-issues pattern):
|
|
* const cfg = makeCrawleeConfig("community-issues");
|
|
* const queue = await RequestQueue.open(null, { config: cfg });
|
|
* const crawler = new CheerioCrawler({ requestQueue: queue, ... }, cfg);
|
|
*/
|
|
|
|
import { Configuration } from "crawlee";
|
|
import { join } from "node:path";
|
|
import { mkdirSync, existsSync, writeFileSync, rmSync, readdirSync, statSync } from "node:fs";
|
|
|
|
/**
|
|
* Crawlee scratch space lives in /tmp/tip-crawlers/<name>/ (NOT next to the repo).
|
|
* /tmp is tmpfs → survives process restarts but not server reboots.
|
|
* Keeps build artefacts and persistent storage separate.
|
|
*/
|
|
const CRAWLEE_TMP_ROOT = process.env.CRAWLEE_TMP_ROOT ?? "/tmp/tip-crawlers";
|
|
|
|
/** Absolute path to the per-scraper Crawlee storage root on disk. */
|
|
export function crawleeStorageDir(scraperName: string): string {
|
|
return join(CRAWLEE_TMP_ROOT, scraperName);
|
|
}
|
|
|
|
/**
|
|
* Remove crawler temp dirs older than `maxAgeMs` (default 48 h).
|
|
* Called automatically on each scraper start — keeps /tmp clean.
|
|
*/
|
|
export function cleanCrawleeTempDirs(maxAgeMs = 48 * 60 * 60 * 1_000): void {
|
|
if (!existsSync(CRAWLEE_TMP_ROOT)) return;
|
|
const cutoff = Date.now() - maxAgeMs;
|
|
try {
|
|
for (const entry of readdirSync(CRAWLEE_TMP_ROOT)) {
|
|
const full = join(CRAWLEE_TMP_ROOT, entry);
|
|
try {
|
|
const st = statSync(full);
|
|
if (st.isDirectory() && st.mtimeMs < cutoff) {
|
|
rmSync(full, { recursive: true, force: true });
|
|
}
|
|
} catch { /* skip */ }
|
|
}
|
|
} catch { /* skip if /tmp not writable */ }
|
|
}
|
|
|
|
/**
|
|
* Create and return a Crawlee Configuration with an isolated storageDir.
|
|
*
|
|
* Idempotent — safe to call every scraper run:
|
|
* - Directories are created if they don't exist (recursive: true).
|
|
* - SDK_SESSION_POOL_STATE.json is seeded once so Crawlee v3.16 doesn't
|
|
* throw "Could not find file" on first run (this version reads before
|
|
* writing on session-pool init).
|
|
*/
|
|
export function makeCrawleeConfig(scraperName: string): Configuration {
|
|
const storageDir = crawleeStorageDir(scraperName);
|
|
|
|
// Clean up stale tmp dirs from other scrapers before starting
|
|
cleanCrawleeTempDirs();
|
|
|
|
// Wipe the request queue from the previous run so Crawlee doesn't skip URLs
|
|
// that were already marked as HANDLED (state=4, orderNo=null). The queue
|
|
// persists between runs because purgeOnStart is false. Without this clear,
|
|
// crawler.run(startUrls) deduplicates all start URLs → requestsTotal=0 →
|
|
// immediate finish with 0 scraped pages. Session pool state lives in
|
|
// key_value_stores/ (not request_queues/), so wiping the queue is safe.
|
|
rmSync(join(storageDir, "request_queues", "default"), { recursive: true, force: true });
|
|
|
|
// Pre-create internal directory tree
|
|
mkdirSync(join(storageDir, "request_queues", "default"), { recursive: true });
|
|
mkdirSync(join(storageDir, "datasets", "default"), { recursive: true });
|
|
mkdirSync(join(storageDir, "key_value_stores", "default"), { recursive: true });
|
|
|
|
// Seed empty session-pool state to avoid "Could not find file" crash in v3.16
|
|
const sessionFile = join(storageDir, "key_value_stores", "default", "SDK_SESSION_POOL_STATE.json");
|
|
if (!existsSync(sessionFile)) {
|
|
writeFileSync(sessionFile, JSON.stringify({
|
|
usableSessionsCount: 0,
|
|
retiredSessionsCount: 0,
|
|
sessions: [],
|
|
}));
|
|
}
|
|
|
|
// `localDataDirectory` is the MemoryStorage option for the base storage path.
|
|
// Pass it via `storageClientOptions` so the Configuration uses this path
|
|
// instance-locally (not via global CRAWLEE_STORAGE_DIR env var).
|
|
return new Configuration({
|
|
storageClientOptions: {
|
|
localDataDirectory: storageDir,
|
|
persistStorage: true,
|
|
},
|
|
purgeOnStart: false, // Preserve session pool state between runs
|
|
});
|
|
}
|