Scraper changes:
- fs-com.ts v2: Playwright stealth patches + www.fs.com/de/ URL fix (de.fs.com DNS NXDOMAIN).
Extracts DE-Lager, Global-Lager, Nachlieferung, units_sold, compatible_brands, price_net.
Mac-side runner (run-fs-scraper-mac.sh) via SSH tunnel for residential IP access.
Fast-fail connectivity check on datacenter IPs that are blocked by Cloudflare.
- smartoptics.ts v2: WooCommerce REST API fallback + 8 catalog categories + relative URL fix.
Was finding only 8 products, now discovers 18+ with multi-category crawl.
DB layer:
- db.ts: add upsertStockObservation() — writes 10 new stock_observations columns
(warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, compatible_brands,
price_net, product_url, delivery dates) with dedup check.
API:
- routes/stock.ts: GET /api/stock, /api/stock/summary, /api/stock/:id
Warehouse breakdowns per transceiver/vendor with top-sellers and vendor summary.
- routes/review.ts: equivalence review queue (approve/reject/bulk-approve).
- index.ts: register /api/stock and /api/review routes.
Dashboard:
- index.html: 🏭 Stock tab with stat cards (DE-Lager, Global-Lager, Nachlieferung totals),
top-sellers table, vendor breakdown, recently-restocked events, part-number lookup.
SQL migrations:
- 034: blog-review-tag, 035: price-observations is_anomalous, 036: transceiver-equivalences.
138 lines
5.0 KiB
TypeScript
138 lines
5.0 KiB
TypeScript
import express from "express";
|
|
import cors from "cors";
|
|
import helmet from "helmet";
|
|
import rateLimit from "express-rate-limit";
|
|
import { join } from "path";
|
|
import { cfg } from "./config";
|
|
import { authRouter } from "./routes/auth";
|
|
import { requireAuth } from "./middleware/require-auth";
|
|
import { transceiverRouter } from "./routes/transceivers";
|
|
import { switchRouter } from "./routes/switches";
|
|
import { vendorRouter } from "./routes/vendors";
|
|
import { standardRouter } from "./routes/standards";
|
|
import { healthRouter } from "./routes/health";
|
|
import { hypeCycleRouter } from "./routes/hype-cycle";
|
|
import { searchRouter } from "./routes/search";
|
|
import { documentRouter } from "./routes/documents";
|
|
import { blogRouter } from "./routes/blog";
|
|
import { blogSllRouter } from "./routes/blog-sll";
|
|
import { scraperRouter } from "./routes/scrapers";
|
|
import { finderRouter } from "./routes/finder";
|
|
import { competitorRouter } from "./routes/competitor-alerts";
|
|
import { forecastRouter } from "./routes/forecast";
|
|
import { transportRouter } from "./routes/transport";
|
|
import { datasheetRouter } from "./routes/datasheets";
|
|
import { hotTopicsRouter } from "./routes/hot-topics";
|
|
import { adoptionRouter } from "./routes/adoption";
|
|
import { procurementRouter } from "./routes/procurement";
|
|
import { changelogRouter } from "./routes/changelog";
|
|
import { newsRouter } from "./routes/news";
|
|
import { proxyRouter } from "./routes/proxy";
|
|
import { reviewRouter } from "./routes/review";
|
|
import { stockRouter } from "./routes/stock";
|
|
|
|
const app = express();
|
|
|
|
// Trust Cloudflare proxy (required for express-rate-limit with X-Forwarded-For)
|
|
app.set("trust proxy", 1);
|
|
|
|
// Middleware
|
|
app.use(helmet({ contentSecurityPolicy: false }));
|
|
app.use(cors());
|
|
app.use(express.json());
|
|
app.use(
|
|
rateLimit({
|
|
windowMs: 60 * 1000,
|
|
max: 200,
|
|
standardHeaders: true,
|
|
legacyHeaders: false,
|
|
})
|
|
);
|
|
|
|
// Auth (public — no requireAuth here)
|
|
app.use("/api/auth", authRouter);
|
|
|
|
// Proxy public endpoints (register + heartbeat + stats + next — no auth)
|
|
app.use("/api/proxy", proxyRouter);
|
|
|
|
// All other API routes require a valid token
|
|
app.use("/api", (req, res, next) => {
|
|
// Always allow: health check, auth endpoints, proxy public routes, hot-topics (public market data)
|
|
if (req.path.startsWith("/health") || req.path.startsWith("/auth")) return next();
|
|
if (req.path.startsWith("/proxy")) return next();
|
|
if (req.path.startsWith("/hot-topics")) return next();
|
|
requireAuth(req, res, next);
|
|
});
|
|
|
|
// Routes
|
|
app.use("/api/transceivers", transceiverRouter);
|
|
app.use("/api/switches", switchRouter);
|
|
app.use("/api/vendors", vendorRouter);
|
|
app.use("/api/standards", standardRouter);
|
|
app.use("/api/health", healthRouter);
|
|
app.use("/api/hype-cycle", hypeCycleRouter);
|
|
app.use("/api/search", searchRouter);
|
|
app.use("/api/documents", documentRouter);
|
|
app.use("/api/blog", blogSllRouter);
|
|
app.use("/api/blog", blogRouter);
|
|
app.use("/api/scrapers", scraperRouter);
|
|
app.use("/api/finder", finderRouter);
|
|
app.use("/api/competitor-alerts", competitorRouter);
|
|
app.use("/api/forecast", forecastRouter);
|
|
app.use("/api/transport", transportRouter);
|
|
app.use("/api/datasheets", datasheetRouter);
|
|
app.use("/api/adoption", adoptionRouter);
|
|
app.use("/api/hot-topics", hotTopicsRouter);
|
|
app.use("/api/procurement", procurementRouter);
|
|
app.use("/api/changelog", changelogRouter);
|
|
app.use("/api/news", newsRouter);
|
|
app.use("/api/review", reviewRouter);
|
|
app.use("/api/stock", stockRouter);
|
|
|
|
// Dashboard (static HTML)
|
|
app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard")));
|
|
|
|
// Root — redirect to dashboard
|
|
app.get("/", (_req, res) => {
|
|
res.redirect("/dashboard/");
|
|
});
|
|
|
|
// API info
|
|
app.get("/api", (_req, res) => {
|
|
res.json({
|
|
name: "Transceiver Intelligence Platform",
|
|
version: "0.3.0",
|
|
endpoints: [
|
|
"GET /api/transceivers?q=&form_factor=&speed=&category=&fiber_type=&wdm_type=&coherent=",
|
|
"GET /api/transceivers/:id",
|
|
"GET /api/switches?q=&category=",
|
|
"GET /api/switches/:id",
|
|
"GET /api/switches/:id/compatibility",
|
|
"GET /api/vendors?type=",
|
|
"GET /api/standards?speed=",
|
|
"GET /api/health",
|
|
"GET /api/hype-cycle",
|
|
"GET /api/hype-cycle/:tech",
|
|
"GET /api/search?q=&collection=&limit=",
|
|
"GET /api/search/products?q=&form_factor=&speed_gbps=&fiber_type=",
|
|
"GET /api/search/documents?q=&doc_type=&vendor=&collection=",
|
|
"GET /api/search/news?q=&source=",
|
|
"GET /api/search/stats",
|
|
"POST /api/documents/process {url, title?, doc_type?, vendor?, collection?}",
|
|
"GET /api/documents",
|
|
"GET /api/documents/:id",
|
|
"POST /api/blog/generate {topic, speed?, form_factor?, use_case?}",
|
|
"GET /api/blog",
|
|
"GET /api/blog/:id",
|
|
"PUT /api/blog/:id/status {status: draft|review|approved|published}",
|
|
],
|
|
});
|
|
});
|
|
|
|
// Start
|
|
app.listen(cfg.port, () => {
|
|
console.log(`\n TIP API running on http://localhost:${cfg.port}`);
|
|
console.log(` Environment: ${cfg.nodeEnv}`);
|
|
console.log(` Database: ${cfg.db.host}:${cfg.db.port}/${cfg.db.database}\n`);
|
|
});
|