From 7decfe5b778b006894e8eca7fd5c903ee94967ae Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Thu, 2 Apr 2026 00:45:48 +0200 Subject: [PATCH] fix: pre-create Crawlee subdirs to prevent ENOENT race in withIsolatedStorage Remove orphan schedules (addon/naddod/qsfptek) that had no registered workers. Pre-create request_queues/default, datasets/default, key_value_stores/default before each scraper run to avoid ENOENT when Crawlee tries to write lock files. --- packages/scraper/src/scheduler.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index b0252cb..7e88060 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -42,7 +42,10 @@ import { rmSync, mkdirSync } from "fs"; /** Run a scraper with an isolated Crawlee storage directory to prevent queue collisions */ async function withIsolatedStorage(name: string, fn: () => Promise): Promise { const dir = join(__dirname, "..", "..", "..", `storage-${name}`); - mkdirSync(dir, { recursive: true }); + // Pre-create Crawlee's internal subdirectory tree to avoid ENOENT races + mkdirSync(join(dir, "request_queues", "default"), { recursive: true }); + mkdirSync(join(dir, "datasets", "default"), { recursive: true }); + mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true }); const prev = process.env.CRAWLEE_STORAGE_DIR; process.env.CRAWLEE_STORAGE_DIR = dir; try {