diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index b0252cb..7e88060 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -42,7 +42,10 @@ import { rmSync, mkdirSync } from "fs"; /** Run a scraper with an isolated Crawlee storage directory to prevent queue collisions */ async function withIsolatedStorage(name: string, fn: () => Promise): Promise { const dir = join(__dirname, "..", "..", "..", `storage-${name}`); - mkdirSync(dir, { recursive: true }); + // Pre-create Crawlee's internal subdirectory tree to avoid ENOENT races + mkdirSync(join(dir, "request_queues", "default"), { recursive: true }); + mkdirSync(join(dir, "datasets", "default"), { recursive: true }); + mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true }); const prev = process.env.CRAWLEE_STORAGE_DIR; process.env.CRAWLEE_STORAGE_DIR = dir; try {