Rene Fichtmueller 5b35b2b8be feat(scraper+api): warehouse stock data pipeline — FS.com v2, SmartOptics v2, Stock API
Scraper changes:
- fs-com.ts v2: Playwright stealth patches + www.fs.com/de/ URL fix (de.fs.com DNS NXDOMAIN).
  Extracts DE-Lager, Global-Lager, Nachlieferung, units_sold, compatible_brands, price_net.
  Mac-side runner (run-fs-scraper-mac.sh) via SSH tunnel for residential IP access.
  Fast-fail connectivity check on datacenter IPs that are blocked by Cloudflare.
- smartoptics.ts v2: WooCommerce REST API fallback + 8 catalog categories + relative URL fix.
  Was finding only 8 products, now discovers 18+ with multi-category crawl.

DB layer:
- db.ts: add upsertStockObservation() — writes 10 new stock_observations columns
  (warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, compatible_brands,
  price_net, product_url, delivery dates) with dedup check.

API:
- routes/stock.ts: GET /api/stock, /api/stock/summary, /api/stock/:id
  Warehouse breakdowns per transceiver/vendor with top-sellers and vendor summary.
- routes/review.ts: equivalence review queue (approve/reject/bulk-approve).
- index.ts: register /api/stock and /api/review routes.

Dashboard:
- index.html: 🏭 Stock tab with stat cards (DE-Lager, Global-Lager, Nachlieferung totals),
  top-sellers table, vendor breakdown, recently-restocked events, part-number lookup.

SQL migrations:
- 034: blog-review-tag, 035: price-observations is_anomalous, 036: transceiver-equivalences.
2026-04-17 10:45:59 +02:00

138 lines
5.0 KiB
TypeScript

import express from "express";
import cors from "cors";
import helmet from "helmet";
import rateLimit from "express-rate-limit";
import { join } from "path";
import { cfg } from "./config";
import { authRouter } from "./routes/auth";
import { requireAuth } from "./middleware/require-auth";
import { transceiverRouter } from "./routes/transceivers";
import { switchRouter } from "./routes/switches";
import { vendorRouter } from "./routes/vendors";
import { standardRouter } from "./routes/standards";
import { healthRouter } from "./routes/health";
import { hypeCycleRouter } from "./routes/hype-cycle";
import { searchRouter } from "./routes/search";
import { documentRouter } from "./routes/documents";
import { blogRouter } from "./routes/blog";
import { blogSllRouter } from "./routes/blog-sll";
import { scraperRouter } from "./routes/scrapers";
import { finderRouter } from "./routes/finder";
import { competitorRouter } from "./routes/competitor-alerts";
import { forecastRouter } from "./routes/forecast";
import { transportRouter } from "./routes/transport";
import { datasheetRouter } from "./routes/datasheets";
import { hotTopicsRouter } from "./routes/hot-topics";
import { adoptionRouter } from "./routes/adoption";
import { procurementRouter } from "./routes/procurement";
import { changelogRouter } from "./routes/changelog";
import { newsRouter } from "./routes/news";
import { proxyRouter } from "./routes/proxy";
import { reviewRouter } from "./routes/review";
import { stockRouter } from "./routes/stock";
const app = express();
// Trust Cloudflare proxy (required for express-rate-limit with X-Forwarded-For)
app.set("trust proxy", 1);
// Middleware
app.use(helmet({ contentSecurityPolicy: false }));
app.use(cors());
app.use(express.json());
app.use(
rateLimit({
windowMs: 60 * 1000,
max: 200,
standardHeaders: true,
legacyHeaders: false,
})
);
// Auth (public — no requireAuth here)
app.use("/api/auth", authRouter);
// Proxy public endpoints (register + heartbeat + stats + next — no auth)
app.use("/api/proxy", proxyRouter);
// All other API routes require a valid token
app.use("/api", (req, res, next) => {
// Always allow: health check, auth endpoints, proxy public routes, hot-topics (public market data)
if (req.path.startsWith("/health") || req.path.startsWith("/auth")) return next();
if (req.path.startsWith("/proxy")) return next();
if (req.path.startsWith("/hot-topics")) return next();
requireAuth(req, res, next);
});
// Routes
app.use("/api/transceivers", transceiverRouter);
app.use("/api/switches", switchRouter);
app.use("/api/vendors", vendorRouter);
app.use("/api/standards", standardRouter);
app.use("/api/health", healthRouter);
app.use("/api/hype-cycle", hypeCycleRouter);
app.use("/api/search", searchRouter);
app.use("/api/documents", documentRouter);
app.use("/api/blog", blogSllRouter);
app.use("/api/blog", blogRouter);
app.use("/api/scrapers", scraperRouter);
app.use("/api/finder", finderRouter);
app.use("/api/competitor-alerts", competitorRouter);
app.use("/api/forecast", forecastRouter);
app.use("/api/transport", transportRouter);
app.use("/api/datasheets", datasheetRouter);
app.use("/api/adoption", adoptionRouter);
app.use("/api/hot-topics", hotTopicsRouter);
app.use("/api/procurement", procurementRouter);
app.use("/api/changelog", changelogRouter);
app.use("/api/news", newsRouter);
app.use("/api/review", reviewRouter);
app.use("/api/stock", stockRouter);
// Dashboard (static HTML)
app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard")));
// Root — redirect to dashboard
app.get("/", (_req, res) => {
res.redirect("/dashboard/");
});
// API info
app.get("/api", (_req, res) => {
res.json({
name: "Transceiver Intelligence Platform",
version: "0.3.0",
endpoints: [
"GET /api/transceivers?q=&form_factor=&speed=&category=&fiber_type=&wdm_type=&coherent=",
"GET /api/transceivers/:id",
"GET /api/switches?q=&category=",
"GET /api/switches/:id",
"GET /api/switches/:id/compatibility",
"GET /api/vendors?type=",
"GET /api/standards?speed=",
"GET /api/health",
"GET /api/hype-cycle",
"GET /api/hype-cycle/:tech",
"GET /api/search?q=&collection=&limit=",
"GET /api/search/products?q=&form_factor=&speed_gbps=&fiber_type=",
"GET /api/search/documents?q=&doc_type=&vendor=&collection=",
"GET /api/search/news?q=&source=",
"GET /api/search/stats",
"POST /api/documents/process {url, title?, doc_type?, vendor?, collection?}",
"GET /api/documents",
"GET /api/documents/:id",
"POST /api/blog/generate {topic, speed?, form_factor?, use_case?}",
"GET /api/blog",
"GET /api/blog/:id",
"PUT /api/blog/:id/status {status: draft|review|approved|published}",
],
});
});
// Start
app.listen(cfg.port, () => {
console.log(`\n TIP API running on http://localhost:${cfg.port}`);
console.log(` Environment: ${cfg.nodeEnv}`);
console.log(` Database: ${cfg.db.host}:${cfg.db.port}/${cfg.db.database}\n`);
});