diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index ef2afcc..0fdafe5 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -31,11 +31,16 @@ async function withIsolatedStorage(name: string, fn: () => Promise): Promi mkdirSync(join(dir, "datasets", "default"), { recursive: true }); mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true }); const prev = process.env.CRAWLEE_STORAGE_DIR; + const prevPurge = process.env.CRAWLEE_PURGE_ON_START; process.env.CRAWLEE_STORAGE_DIR = dir; + // Force Crawlee to initialize fresh — prevents "Could not find SDK_SESSION_POOL_STATE.json" + // when the isolated storage dir was just created and has no pre-existing state files. + process.env.CRAWLEE_PURGE_ON_START = "1"; try { await fn(); } finally { process.env.CRAWLEE_STORAGE_DIR = prev ?? ""; + process.env.CRAWLEE_PURGE_ON_START = prevPurge ?? ""; // Clean up after successful run try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }