rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: pre-create Crawlee subdirs to prevent ENOENT race in withIsolatedStorage

Browse Source 
Remove orphan schedules (addon/naddod/qsfptek) that had no registered workers.
Pre-create request_queues/default, datasets/default, key_value_stores/default
before each scraper run to avoid ENOENT when Crawlee tries to write lock files.
This commit is contained in:
Rene Fichtmueller 2026-04-02 00:45:48 +02:00
parent bb80e8bbe6
commit 1c1fb28189
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/scraper/src/scheduler.ts
View File

@ -42,7 +42,10 @@ import { rmSync, mkdirSync } from "fs";
/** Run a scraper with an isolated Crawlee storage directory to prevent queue collisions */ /** Run a scraper with an isolated Crawlee storage directory to prevent queue collisions */
async function withIsolatedStorage(name: string, fn: () => Promise<void>): Promise<void> { async function withIsolatedStorage(name: string, fn: () => Promise<void>): Promise<void> {
const dir = join(__dirname, "..", "..", "..", `storage-${name}`); const dir = join(__dirname, "..", "..", "..", `storage-${name}`);
mkdirSync(dir, { recursive: true }); // Pre-create Crawlee's internal subdirectory tree to avoid ENOENT races
mkdirSync(join(dir, "request_queues", "default"), { recursive: true });
mkdirSync(join(dir, "datasets", "default"), { recursive: true });
mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true });
const prev = process.env.CRAWLEE_STORAGE_DIR; const prev = process.env.CRAWLEE_STORAGE_DIR;
process.env.CRAWLEE_STORAGE_DIR = dir; process.env.CRAWLEE_STORAGE_DIR = dir;
try { try {