c7d7456de9
- Add utils/crawlee-config.ts: makeCrawleeConfig(name) returns a Crawlee Configuration with isolated localDataDirectory per scraper. Uses storageClientOptions (not global CRAWLEE_STORAGE_DIR) so concurrent pg-boss workers in the same process don't race on the shared env var. - Apply makeCrawleeConfig to all 6 Crawlee-based scrapers: optcore (PlaywrightCrawler), atgbics (PlaywrightCrawler), community-issues (CheerioCrawler + RequestQueue), edgecore (CheerioCrawler), ufispace (CheerioCrawler), market-intelligence (CheerioCrawler). - scheduler.ts: add withIsolatedStorage for optcore and market-intel workers (was missing, caused storage-fs path bleed from fs scraper). - ebay-enricher.ts: fix vendor type 'marketplace' -> 'reseller' to satisfy vendors_type_check constraint ['manufacturer','distributor','oem','reseller','compatible'].
79 lines
3.2 KiB
TypeScript
79 lines
3.2 KiB
TypeScript
/**
|
|
* Crawlee instance-level storage isolation.
|
|
*
|
|
* WHY THIS EXISTS:
|
|
* ----------------
|
|
* All pg-boss workers run concurrently inside a single Node.js process.
|
|
* The old approach (setting process.env.CRAWLEE_STORAGE_DIR) is a global
|
|
* env-var mutation — if two workers run simultaneously, one worker's
|
|
* writeToEnv() can overwrite another's before the crawler reads it.
|
|
*
|
|
* The fix: pass a `Configuration` instance directly to each Crawlee
|
|
* constructor. This is instance-level (not global), so concurrent scrapers
|
|
* each get their own isolated storage directory.
|
|
*
|
|
* Usage:
|
|
* import { makeCrawleeConfig } from "../utils/crawlee-config";
|
|
*
|
|
* // In PlaywrightCrawler:
|
|
* const crawler = new PlaywrightCrawler({ ... }, makeCrawleeConfig("optcore"));
|
|
*
|
|
* // In CheerioCrawler:
|
|
* const crawler = new CheerioCrawler({ ... }, makeCrawleeConfig("edgecore"));
|
|
*
|
|
* // With explicit RequestQueue (community-issues pattern):
|
|
* const cfg = makeCrawleeConfig("community-issues");
|
|
* const queue = await RequestQueue.open(null, { config: cfg });
|
|
* const crawler = new CheerioCrawler({ requestQueue: queue, ... }, cfg);
|
|
*/
|
|
|
|
import { Configuration } from "crawlee";
|
|
import { join } from "node:path";
|
|
import { mkdirSync, existsSync, writeFileSync } from "node:fs";
|
|
|
|
/** Absolute path to the per-scraper Crawlee storage root on disk. */
|
|
export function crawleeStorageDir(scraperName: string): string {
|
|
// dist layout: packages/scraper/dist/utils/ → go 4 levels up → repo root
|
|
// Then store beside packages/ as storage-<name>/
|
|
return join(__dirname, "..", "..", "..", "..", `storage-${scraperName}`);
|
|
}
|
|
|
|
/**
|
|
* Create and return a Crawlee Configuration with an isolated storageDir.
|
|
*
|
|
* Idempotent — safe to call every scraper run:
|
|
* - Directories are created if they don't exist (recursive: true).
|
|
* - SDK_SESSION_POOL_STATE.json is seeded once so Crawlee v3.16 doesn't
|
|
* throw "Could not find file" on first run (this version reads before
|
|
* writing on session-pool init).
|
|
*/
|
|
export function makeCrawleeConfig(scraperName: string): Configuration {
|
|
const storageDir = crawleeStorageDir(scraperName);
|
|
|
|
// Pre-create internal directory tree
|
|
mkdirSync(join(storageDir, "request_queues", "default"), { recursive: true });
|
|
mkdirSync(join(storageDir, "datasets", "default"), { recursive: true });
|
|
mkdirSync(join(storageDir, "key_value_stores", "default"), { recursive: true });
|
|
|
|
// Seed empty session-pool state to avoid "Could not find file" crash in v3.16
|
|
const sessionFile = join(storageDir, "key_value_stores", "default", "SDK_SESSION_POOL_STATE.json");
|
|
if (!existsSync(sessionFile)) {
|
|
writeFileSync(sessionFile, JSON.stringify({
|
|
usableSessionsCount: 0,
|
|
retiredSessionsCount: 0,
|
|
sessions: [],
|
|
}));
|
|
}
|
|
|
|
// `localDataDirectory` is the MemoryStorage option for the base storage path.
|
|
// Pass it via `storageClientOptions` so the Configuration uses this path
|
|
// instance-locally (not via global CRAWLEE_STORAGE_DIR env var).
|
|
return new Configuration({
|
|
storageClientOptions: {
|
|
localDataDirectory: storageDir,
|
|
persistStorage: true,
|
|
},
|
|
purgeOnStart: false, // Preserve session pool state between runs
|
|
});
|
|
}
|