Rene Fichtmueller d7144731e0 feat(scraper): add 100+ OEM seed scrapers + tip-llm-guided inference layer
New OEM transceiver seed scrapers (94 cron-scheduled, 24/7):
- Media/Broadcast: Evertz, Grass Valley, Haivision, Viasat
- Asian Optical: FiberHome, Oplink, Accelink, Hisense Broadband
- Optical Mfrs: Lumentum, II-VI/Coherent, Source Photonics, O-Net,
  InnoLight, AOI, Sumitomo Electric, NeoPhotonics
- Industrial: GE Grid, Schweitzer, Moxa Industrial, Cisco IE,
  Phoenix Contact, Beckhoff, Omron, ABB, Siemens, Schneider, Rockwell, Belden
- Enterprise/DC: Arista, Pica8, Pluribus, DriveNets, Cisco (Meraki/Catalyst/Nexus/ASR)
- Cloud: AWS, Azure, Google Cloud, Meta
- Storage: NetApp, Pure Storage, HPE Storage, IBM Storage, Dell Storage, Hitachi Vantara
- 5G/RAN: Samsung Networks, Nokia AirScale, Ericsson RAN, Mavenir
- Security: Check Point, Barracuda, Fortinet, Palo Alto
- Telecom Optical: ADVA, PacketLight, FiberHome, Accelink, Hisense

API: tip-llm-guided inference layer (strict schema + repair-retry + safe fallback)
- POST /api/tip-llm/infer|research-plan|extract|finding|health
- Hard JSON schema enforcement, create_finding=false on empty evidence
- Confidence gate (>= 0.4), validation with consistency check

Build: added incremental=true to scraper tsconfig (OOM prevention)
Scheduler: 87 → 94 registered workers
2026-04-27 00:00:14 +02:00

156 lines
5.9 KiB
TypeScript

import express from "express";
import cors from "cors";
import helmet from "helmet";
import rateLimit from "express-rate-limit";
import { join } from "path";
import { cfg } from "./config";
import { authRouter } from "./routes/auth";
import { requireAuth } from "./middleware/require-auth";
import { transceiverRouter } from "./routes/transceivers";
import { switchRouter } from "./routes/switches";
import { vendorRouter } from "./routes/vendors";
import { standardRouter } from "./routes/standards";
import { healthRouter } from "./routes/health";
import { hypeCycleRouter } from "./routes/hype-cycle";
import { searchRouter } from "./routes/search";
import { documentRouter } from "./routes/documents";
import { blogRouter } from "./routes/blog";
import { blogSllRouter } from "./routes/blog-sll";
import { scraperRouter } from "./routes/scrapers";
import { finderRouter } from "./routes/finder";
import { competitorRouter } from "./routes/competitor-alerts";
import { forecastRouter } from "./routes/forecast";
import { transportRouter } from "./routes/transport";
import { datasheetRouter } from "./routes/datasheets";
import { hotTopicsRouter } from "./routes/hot-topics";
import { adoptionRouter } from "./routes/adoption";
import { procurementRouter } from "./routes/procurement";
import { changelogRouter } from "./routes/changelog";
import { newsRouter } from "./routes/news";
import { proxyRouter } from "./routes/proxy";
import { reviewRouter } from "./routes/review";
import { stockRouter } from "./routes/stock";
import { priceComparisonRouter } from "./routes/price-comparison";
import { selflearningRouter } from "./routes/selflearning";
import { internalDemandRouter } from "./routes/internal-demand";
import { formFactorsRouter } from "./routes/form-factors";
import { tipLlmRouter } from "./routes/tip-llm";
const app = express();
// Trust Cloudflare proxy (required for express-rate-limit with X-Forwarded-For)
app.set("trust proxy", 1);
// Middleware
app.use(helmet({ contentSecurityPolicy: false }));
app.use(cors());
app.use(express.json());
app.use(
rateLimit({
windowMs: 60 * 1000,
max: 200,
standardHeaders: true,
legacyHeaders: false,
})
);
// Auth (public — no requireAuth here)
app.use("/api/auth", authRouter);
// Proxy public endpoints (register + heartbeat + stats + next — no auth)
app.use("/api/proxy", proxyRouter);
// All other API routes require a valid token
app.use("/api", (req, res, next) => {
// Always allow: health check, auth endpoints, proxy public routes, hot-topics (public market data)
if (req.path.startsWith("/health") || req.path.startsWith("/auth")) return next();
if (req.path.startsWith("/proxy")) return next();
if (req.path.startsWith("/hot-topics")) return next();
if (req.path.startsWith("/price-comparison")) return next();
requireAuth(req, res, next);
});
// Routes
app.use("/api/transceivers", transceiverRouter);
app.use("/api/switches", switchRouter);
app.use("/api/vendors", vendorRouter);
app.use("/api/standards", standardRouter);
app.use("/api/health", healthRouter);
app.use("/api/hype-cycle", hypeCycleRouter);
app.use("/api/search", searchRouter);
app.use("/api/documents", documentRouter);
app.use("/api/blog", blogSllRouter);
app.use("/api/blog", blogRouter);
app.use("/api/scrapers", scraperRouter);
app.use("/api/finder", finderRouter);
app.use("/api/competitor-alerts", competitorRouter);
app.use("/api/forecast", forecastRouter);
app.use("/api/transport", transportRouter);
app.use("/api/datasheets", datasheetRouter);
app.use("/api/adoption", adoptionRouter);
app.use("/api/hot-topics", hotTopicsRouter);
app.use("/api/procurement", procurementRouter);
app.use("/api/changelog", changelogRouter);
app.use("/api/news", newsRouter);
app.use("/api/review", reviewRouter);
app.use("/api/stock", stockRouter);
app.use("/api/price-comparison", priceComparisonRouter);
app.use("/api/selflearning", selflearningRouter);
// Form Factors reference
app.use("/api/form-factors", formFactorsRouter);
// Internal-only — restricted to localhost / LAN by the router itself
app.use("/api/internal/demand", internalDemandRouter);
// tip-llm-v1 guided inference
app.use("/api/tip-llm", tipLlmRouter);
// Dashboard (static HTML)
app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard")));
// Root — redirect to dashboard
app.get("/", (_req, res) => {
res.redirect("/dashboard/");
});
// API info
app.get("/api", (_req, res) => {
res.json({
name: "Transceiver Intelligence Platform",
version: "0.3.0",
endpoints: [
"GET /api/transceivers?q=&form_factor=&speed=&category=&fiber_type=&wdm_type=&coherent=",
"GET /api/transceivers/:id",
"GET /api/switches?q=&category=",
"GET /api/switches/:id",
"GET /api/switches/:id/compatibility",
"GET /api/vendors?type=",
"GET /api/standards?speed=",
"GET /api/health",
"GET /api/hype-cycle",
"GET /api/hype-cycle/:tech",
"GET /api/search?q=&collection=&limit=",
"GET /api/search/products?q=&form_factor=&speed_gbps=&fiber_type=",
"GET /api/search/documents?q=&doc_type=&vendor=&collection=",
"GET /api/search/news?q=&source=",
"GET /api/search/stats",
"POST /api/documents/process {url, title?, doc_type?, vendor?, collection?}",
"GET /api/documents",
"GET /api/documents/:id",
"POST /api/blog/generate {topic, speed?, form_factor?, use_case?}",
"GET /api/blog",
"GET /api/blog/:id",
"PUT /api/blog/:id/status {status: draft|review|approved|published}",
"GET /api/selflearning/status",
"POST /api/selflearning/build",
"POST /api/selflearning/publish-hf",
"POST /api/selflearning/train {lane, provider, seed_only?, max_steps?}",
],
});
});
// Start
app.listen(cfg.port, () => {
console.log(`\n TIP API running on http://localhost:${cfg.port}`);
console.log(` Environment: ${cfg.nodeEnv}`);
console.log(` Database: ${cfg.db.host}:${cfg.db.port}/${cfg.db.database}\n`);
});