1026787318
- Add TIP Proxy Network (packages/proxy-agent): SOCKS5 proxy agent for residential IP bypass of CloudFront WAF blocks - Add /api/proxy/* routes: node registration, heartbeat, load balancing - Add image extraction to Flexoptix catalog scraper (GraphQL small_image) - Add image extraction to Optcore scraper (Playwright gallery img) - Fix Fluxlight price scraping (BigCommerce HTML structure: data-product-price-without-tax) - Add SmartOptics scraper (8 DWDM/coherent products, og:image extraction) - Fix findOrCreateScrapedTransceiver to update image_url for existing records - Add image backfill script (backfill-images.ts): 178 Flexoptix images added - Fix DB connection pool: max 5, idleTimeoutMillis 10s (was unlimited, caused >100 connections) - Add proxy.ts utility for scraper proxy rotation
83 lines
2.2 KiB
TypeScript
83 lines
2.2 KiB
TypeScript
/**
|
|
* Heartbeat — sends periodic status updates to TIP routing server.
|
|
* POST /api/proxy/heartbeat every 30s.
|
|
*/
|
|
import * as https from "https";
|
|
import * as http from "http";
|
|
|
|
export interface HeartbeatPayload {
|
|
token: string;
|
|
ip?: string;
|
|
port: number;
|
|
bytesProxied: number;
|
|
requestsProxied: number;
|
|
latencyMs?: number;
|
|
version: string;
|
|
}
|
|
|
|
export interface HeartbeatResult {
|
|
ok: boolean;
|
|
latencyMs: number;
|
|
error?: string;
|
|
}
|
|
|
|
function postJson(url: string, body: unknown): Promise<{ status: number; latencyMs: number }> {
|
|
return new Promise((resolve, reject) => {
|
|
const start = Date.now();
|
|
const json = JSON.stringify(body);
|
|
const parsed = new URL(url);
|
|
const mod = parsed.protocol === "https:" ? https : http;
|
|
|
|
const req = mod.request(
|
|
{
|
|
hostname: parsed.hostname,
|
|
port: parsed.port || (parsed.protocol === "https:" ? 443 : 80),
|
|
path: parsed.pathname + parsed.search,
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
"Content-Length": Buffer.byteLength(json),
|
|
"User-Agent": "TIP-Agent/1.0.0",
|
|
},
|
|
timeout: 10_000,
|
|
},
|
|
(res) => {
|
|
res.resume();
|
|
resolve({ status: res.statusCode ?? 0, latencyMs: Date.now() - start });
|
|
}
|
|
);
|
|
|
|
req.on("timeout", () => { req.destroy(); reject(new Error("timeout")); });
|
|
req.on("error", reject);
|
|
req.write(json);
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
export async function sendHeartbeat(
|
|
serverUrl: string,
|
|
payload: HeartbeatPayload
|
|
): Promise<HeartbeatResult> {
|
|
const url = `${serverUrl}/api/proxy/heartbeat`;
|
|
try {
|
|
const { status, latencyMs } = await postJson(url, payload);
|
|
return { ok: status >= 200 && status < 300, latencyMs };
|
|
} catch (err) {
|
|
return { ok: false, latencyMs: 0, error: String(err) };
|
|
}
|
|
}
|
|
|
|
export function startHeartbeatLoop(
|
|
serverUrl: string,
|
|
getPayload: () => HeartbeatPayload,
|
|
onResult?: (r: HeartbeatResult) => void,
|
|
intervalMs = 30_000
|
|
): NodeJS.Timeout {
|
|
const tick = async () => {
|
|
const result = await sendHeartbeat(serverUrl, getPayload());
|
|
onResult?.(result);
|
|
};
|
|
tick();
|
|
return setInterval(tick, intervalMs);
|
|
}
|