From 6d7b067ca933388a46c5f7a19abdea87ae63224b Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Sun, 5 Apr 2026 11:51:07 +0200 Subject: [PATCH] fix: resolve merge conflict in index.ts + add untracked blog-sll, news, sql migration --- package-lock.json | 430 +++++++++++++++++++++++++ packages/api/src/db/queries.ts | 9 +- packages/api/src/index.ts | 12 +- packages/api/src/routes/blog-sll.ts | 379 ++++++++++++++++++++++ packages/api/src/routes/news.ts | 58 ++++ packages/api/src/routes/vendors.ts | 123 +++++++ packages/dashboard/hot-topics.js | 187 +++++++---- packages/dashboard/login.html | 30 +- packages/scraper/src/scrapers/gbics.ts | 55 ++-- sql/025-blog-sll.sql | 127 ++++++++ 10 files changed, 1327 insertions(+), 83 deletions(-) create mode 100644 packages/api/src/routes/blog-sll.ts create mode 100644 packages/api/src/routes/news.ts create mode 100644 sql/025-blog-sll.sql diff --git a/package-lock.json b/package-lock.json index 261112f..9fe9553 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1536,6 +1536,10 @@ "resolved": "packages/mcp-server", "link": true }, + "node_modules/@tip/proxy-agent": { + "resolved": "packages/proxy-agent", + "link": true + }, "node_modules/@tip/scraper": { "resolved": "packages/scraper", "link": true @@ -1650,6 +1654,13 @@ "parse5": "^7.0.0" } }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "25.5.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", @@ -1865,6 +1876,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2532,6 +2549,16 @@ "node": ">= 0.8" } }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/devtools-protocol": { "version": "0.0.1604597", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1604597.tgz", @@ -4001,6 +4028,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", @@ -5556,6 +5604,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/vali-date": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", @@ -5934,6 +5991,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.9.0", "dotenv": "^16.4.7", + "express": "^4.18.2", "pg": "^8.13.1", "zod": "^3.24.0" }, @@ -5941,11 +5999,383 @@ "tip-mcp": "dist/index.js" }, "devDependencies": { + "@types/express": "^4.17.21", "@types/pg": "^8.11.11", "tsx": "^4.19.0", "typescript": "^5.9.3" } }, + "packages/mcp-server/node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "packages/mcp-server/node_modules/@types/express-serve-static-core": { + "version": "4.19.8", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", + "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "packages/mcp-server/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "packages/mcp-server/node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "packages/mcp-server/node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "packages/mcp-server/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "packages/mcp-server/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "packages/mcp-server/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "packages/mcp-server/node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "packages/mcp-server/node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "packages/mcp-server/node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "packages/mcp-server/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/mcp-server/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "packages/mcp-server/node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "packages/mcp-server/node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "packages/mcp-server/node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "packages/mcp-server/node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "packages/mcp-server/node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "packages/mcp-server/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "packages/proxy-agent": { + "name": "@tip/proxy-agent", + "version": "1.0.0", + "dependencies": { + "dotenv": "^16.0.0" + }, + "bin": { + "tip-agent": "dist/index.js" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } + }, + "packages/proxy-agent/node_modules/@types/node": { + "version": "20.19.39", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.39.tgz", + "integrity": "sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "packages/proxy-agent/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "packages/scraper": { "name": "@tip/scraper", "version": "0.1.0", diff --git a/packages/api/src/db/queries.ts b/packages/api/src/db/queries.ts index 6def7d8..07f243a 100644 --- a/packages/api/src/db/queries.ts +++ b/packages/api/src/db/queries.ts @@ -314,9 +314,14 @@ export async function listVendors(type?: string) { } export async function listStandards(speed?: string) { + const base = ` + SELECT s.*, + COUNT(t.id)::int AS transceiver_count + FROM standards s + LEFT JOIN transceivers t ON t.standard_id = s.id`; const query = speed - ? `SELECT * FROM standards WHERE speed = $1 ORDER BY year_ratified DESC` - : `SELECT * FROM standards ORDER BY year_ratified DESC`; + ? `${base} WHERE s.speed = $1 GROUP BY s.id ORDER BY s.speed_gbps DESC NULLS LAST, s.name` + : `${base} GROUP BY s.id ORDER BY s.speed_gbps DESC NULLS LAST, s.name`; const result = await pool.query(query, speed ? [speed] : []); return result.rows; } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 92109be..2f6296d 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -26,6 +26,9 @@ import { hotTopicsRouter } from "./routes/hot-topics"; import { adoptionRouter } from "./routes/adoption"; import { procurementRouter } from "./routes/procurement"; import { changelogRouter } from "./routes/changelog"; +import { scraperRouter } from "./routes/scrapers"; +import { newsRouter } from "./routes/news"; +import { proxyRouter } from "./routes/proxy"; const app = express(); @@ -48,10 +51,14 @@ app.use( // Auth (public — no requireAuth here) app.use("/api/auth", authRouter); +// Proxy public endpoints (register + heartbeat + stats + next — no auth) +app.use("/api/proxy", proxyRouter); + // All other API routes require a valid token app.use("/api", (req, res, next) => { - // Always allow: health check, auth endpoints + // Always allow: health check, auth endpoints, proxy public routes if (req.path.startsWith("/health") || req.path.startsWith("/auth")) return next(); + if (req.path.startsWith("/proxy")) return next(); requireAuth(req, res, next); }); @@ -66,6 +73,7 @@ app.use("/api/search", searchRouter); app.use("/api/documents", documentRouter); app.use("/api/blog", blogSllRouter); app.use("/api/blog", blogRouter); +<<<<<<< Updated upstream app.use("/api/scrapers", scraperRouter); app.use("/api/finder", finderRouter); app.use("/api/competitor-alerts", competitorRouter); @@ -76,6 +84,8 @@ app.use("/api/adoption", adoptionRouter); app.use("/api/hot-topics", hotTopicsRouter); app.use("/api/procurement", procurementRouter); app.use("/api/changelog", changelogRouter); +app.use("/api/scrapers", scraperRouter); +app.use("/api/news", newsRouter); // Dashboard (static HTML) app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard"))); diff --git a/packages/api/src/routes/blog-sll.ts b/packages/api/src/routes/blog-sll.ts new file mode 100644 index 0000000..1a9b660 --- /dev/null +++ b/packages/api/src/routes/blog-sll.ts @@ -0,0 +1,379 @@ +/** + * Blog Self-Learning Loop (SLL v1.0) + * + * Routes: + * POST /api/blog/:id/performance — log engagement metrics for a post + * GET /api/blog/sll/insights — current learning state + * POST /api/blog/sll/analyze — trigger LLM pattern extraction + * GET /api/blog/sll/patterns — all learned patterns + */ + +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; + +export const blogSllRouter = Router(); + +// ───────────────────────────────────────────────────────────────── +// POST /api/blog/:id/performance — log LinkedIn engagement +// ───────────────────────────────────────────────────────────────── +blogSllRouter.post("/:id/performance", async (req: Request, res: Response) => { + const { id } = req.params; + const { + platform = "linkedin", + impressions, + comments = 0, + shares = 0, + saves = 0, + likes = 0, + hook_text, + posted_at, + notes, + } = req.body as { + platform?: string; + impressions?: number; + comments?: number; + shares?: number; + saves?: number; + likes?: number; + hook_text?: string; + posted_at?: string; + notes?: string; + }; + + try { + // Pull blog metadata for pattern context + const blogRow = await pool.query( + `SELECT topic, pipeline_version, word_count, linkedin_post FROM blog_drafts WHERE id = $1::uuid`, + [id] + ); + if (blogRow.rows.length === 0) { + res.status(404).json({ success: false, error: "Blog draft not found" }); + return; + } + const blog = blogRow.rows[0]; + + // Auto-extract hook from linkedin_post if not provided + const resolvedHook = hook_text || + (blog.linkedin_post ? blog.linkedin_post.split("\n")[0].slice(0, 120) : null); + + const result = await pool.query( + `INSERT INTO blog_performance + (blog_id, platform, impressions, comments, shares, saves, likes, + hook_text, blog_type, topic, word_count, pipeline_version, posted_at, notes) + VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + RETURNING id, engagement_score`, + [ + id, platform, impressions ?? null, comments, shares, saves, likes, + resolvedHook, blog.topic, blog.topic, + blog.word_count, blog.pipeline_version, + posted_at ? new Date(posted_at) : null, notes ?? null, + ] + ); + + const row = result.rows[0]; + const score = row.engagement_score; + const tier = score >= 20 ? "gold" : score >= 10 ? "silver" : score >= 4 ? "bronze" : "miss"; + + console.log(`SLL: Performance logged for ${id} — score ${score} (${tier})`); + + res.json({ + success: true, + performance_id: row.id, + engagement_score: score, + tier, + formula: `(comments×3) + (shares×2) + (saves×2) = ${comments * 3} + ${shares * 2} + ${saves * 2}`, + }); + } catch (err) { + console.error("SLL performance log error:", err); + res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ───────────────────────────────────────────────────────────────── +// GET /api/blog/sll/insights — current learning state +// ───────────────────────────────────────────────────────────────── +blogSllRouter.get("/sll/insights", async (_req: Request, res: Response) => { + try { + const [stateRes, statsRes, topRes, bottomRes, patternsRes] = await Promise.all([ + // Latest SLL state snapshot + pool.query( + `SELECT * FROM blog_sll_state ORDER BY week_start DESC LIMIT 1` + ), + // Overall performance stats + pool.query(` + SELECT + COUNT(*) as total_posts, + AVG(engagement_score) as avg_score, + MAX(engagement_score) as best_score, + COUNT(*) FILTER (WHERE engagement_score >= 20) as gold_count, + COUNT(*) FILTER (WHERE engagement_score >= 10) as silver_count, + COUNT(*) FILTER (WHERE engagement_score >= 4) as bronze_count, + COUNT(*) FILTER (WHERE engagement_score < 4) as miss_count + FROM blog_performance + `), + // Top 5 posts + pool.query(` + SELECT d.title, d.topic, p.engagement_score, p.hook_text, + p.word_count, p.blog_type, p.posted_at, + CASE WHEN p.engagement_score >= 20 THEN 'gold' + WHEN p.engagement_score >= 10 THEN 'silver' + WHEN p.engagement_score >= 4 THEN 'bronze' + ELSE 'miss' END as tier + FROM blog_performance p + JOIN blog_drafts d ON d.id = p.blog_id + ORDER BY p.engagement_score DESC LIMIT 5 + `), + // Bottom 5 posts + pool.query(` + SELECT d.title, d.topic, p.engagement_score, p.hook_text, p.word_count + FROM blog_performance p + JOIN blog_drafts d ON d.id = p.blog_id + WHERE p.engagement_score IS NOT NULL + ORDER BY p.engagement_score ASC LIMIT 5 + `), + // Active learned patterns + pool.query(` + SELECT pattern_type, pattern_value, performance_class, avg_engagement, sample_count + FROM blog_learned_patterns + WHERE active = TRUE + ORDER BY performance_class, avg_engagement DESC NULLS LAST + `), + ]); + + const state = stateRes.rows[0] || null; + const stats = statsRes.rows[0]; + + res.json({ + success: true, + stats: { + total_posts: Number(stats.total_posts), + avg_score: stats.avg_score ? Math.round(Number(stats.avg_score) * 10) / 10 : null, + best_score: Number(stats.best_score) || 0, + tiers: { + gold: Number(stats.gold_count), + silver: Number(stats.silver_count), + bronze: Number(stats.bronze_count), + miss: Number(stats.miss_count), + }, + }, + current_state: state, + top_posts: topRes.rows, + bottom_posts: bottomRes.rows, + learned_patterns: { + winners: patternsRes.rows.filter((p: any) => p.performance_class === "winner"), + losers: patternsRes.rows.filter((p: any) => p.performance_class === "loser"), + }, + sll_ready: Number(stats.total_posts) >= 5, + note: Number(stats.total_posts) < 5 + ? `SLL needs ${5 - Number(stats.total_posts)} more posts with performance data before pattern extraction` + : "SLL active — enough data for pattern extraction", + }); + } catch (err) { + res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ───────────────────────────────────────────────────────────────── +// GET /api/blog/sll/patterns — all learned patterns +// ───────────────────────────────────────────────────────────────── +blogSllRouter.get("/sll/patterns", async (_req: Request, res: Response) => { + try { + const result = await pool.query(` + SELECT lp.*, d.title as example_title + FROM blog_learned_patterns lp + LEFT JOIN blog_drafts d ON d.id = lp.example_post_id + WHERE lp.active = TRUE + ORDER BY lp.performance_class, lp.avg_engagement DESC NULLS LAST + `); + + res.json({ success: true, patterns: result.rows }); + } catch (err) { + res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ───────────────────────────────────────────────────────────────── +// POST /api/blog/sll/analyze — trigger LLM pattern extraction +// ───────────────────────────────────────────────────────────────── +blogSllRouter.post("/sll/analyze", async (_req: Request, res: Response) => { + try { + // Need at least 3 posts with performance data + const countRes = await pool.query( + `SELECT COUNT(*) as cnt FROM blog_performance WHERE engagement_score IS NOT NULL` + ); + const count = Number(countRes.rows[0].cnt); + if (count < 3) { + res.status(400).json({ + success: false, + error: `Not enough data — need at least 3 posts with performance data, have ${count}`, + }); + return; + } + + // Pull all performance data with article context + const perfData = await pool.query(` + SELECT + d.title, d.topic, d.word_count as article_words, d.pipeline_version, + p.engagement_score, p.hook_text, p.blog_type, p.comments, p.shares, + p.saves, p.likes, p.impressions, + CASE WHEN p.engagement_score >= 20 THEN 'gold' + WHEN p.engagement_score >= 10 THEN 'silver' + WHEN p.engagement_score >= 4 THEN 'bronze' + ELSE 'miss' END as tier + FROM blog_performance p + JOIN blog_drafts d ON d.id = p.blog_id + ORDER BY p.engagement_score DESC + `); + + // Sort into winners / losers + const winners = perfData.rows.filter((r: any) => r.engagement_score >= 10); + const losers = perfData.rows.filter((r: any) => r.engagement_score < 4); + + // Build LLM prompt for pattern extraction + const analysisPrompt = `You are analyzing LinkedIn post performance data for the Flexoptix technical blog. + +PERFORMANCE DATA (${count} posts): + +TOP PERFORMERS: +${winners.map((r: any, i: number) => `${i+1}. Score: ${r.engagement_score} (${r.tier}) + Title: ${r.title} + Topic: ${r.topic} | Words: ${r.article_words} + Hook: "${r.hook_text || "n/a"}" + Metrics: ${r.comments}c / ${r.shares}sh / ${r.saves}sa / ${r.likes}li`).join("\n\n") || "No winners yet"} + +UNDERPERFORMERS: +${losers.map((r: any, i: number) => `${i+1}. Score: ${r.engagement_score} + Title: ${r.title} + Topic: ${r.topic} | Words: ${r.article_words} + Hook: "${r.hook_text || "n/a"}"`).join("\n\n") || "No losers yet"} + +SCORING FORMULA: (comments×3) + (shares×2) + (saves×2) +Likes = 0 weight. Saves and shares = real interest. + +Extract patterns in this EXACT JSON format: +{ + "winner_patterns": [ + {"type": "hook|structure|topic|length|opening", "value": "description of what works", "evidence": "why you think this"} + ], + "loser_patterns": [ + {"type": "hook|structure|topic|length|opening", "value": "description of what fails", "evidence": "why"} + ], + "optimal_length": {"min": 900, "max": 1400}, + "top_topics": ["topic1", "topic2"], + "avoid_topics": ["topic3"], + "best_hook_patterns": ["pattern1", "pattern2"], + "key_insight": "One sentence: the single most important finding from this data" +} + +Be specific. "short hook + contrast" is better than "good hooks". Use the actual data.`; + + // Call LLM + let extractedPatterns: Record | null = null; + try { + const { generate } = await import("../llm/client"); + const llmResult = await generate( + "You are a content analytics expert extracting patterns from performance data. Return only valid JSON.", + analysisPrompt, + { temperature: 0.2, maxTokens: 2048, timeoutMs: 120000 } + ); + + const jsonMatch = llmResult.text.match(/\{[\s\S]*"winner_patterns"[\s\S]*\}/); + if (jsonMatch) { + extractedPatterns = JSON.parse(jsonMatch[0]); + } + } catch (llmErr) { + console.warn("SLL LLM extraction failed, using heuristic fallback:", llmErr); + } + + // Heuristic fallback if LLM fails + if (!extractedPatterns) { + const avgWinner = winners.length > 0 + ? winners.reduce((s: number, r: any) => s + Number(r.article_words), 0) / winners.length + : 1100; + const avgLoser = losers.length > 0 + ? losers.reduce((s: number, r: any) => s + Number(r.article_words), 0) / losers.length + : 1400; + + extractedPatterns = { + winner_patterns: [ + { type: "length", value: `~${Math.round(avgWinner)} words performs best`, evidence: "heuristic from top posts" }, + ], + loser_patterns: [ + { type: "length", value: `~${Math.round(avgLoser)} words underperforms`, evidence: "heuristic from low posts" }, + ], + optimal_length: { min: Math.round(avgWinner * 0.8), max: Math.round(avgWinner * 1.2) }, + top_topics: [...new Set(winners.map((r: any) => r.topic))].slice(0, 3), + avoid_topics: [...new Set(losers.map((r: any) => r.topic))].slice(0, 2), + best_hook_patterns: winners.filter((r: any) => r.hook_text).map((r: any) => r.hook_text).slice(0, 2), + key_insight: "Based on available data — more posts needed for reliable patterns", + }; + } + + const ep = extractedPatterns as any; + + // Save patterns to DB (upsert by value to avoid duplicates) + let savedCount = 0; + const allPatterns = [ + ...(ep.winner_patterns || []).map((p: any) => ({ ...p, cls: "winner" })), + ...(ep.loser_patterns || []).map((p: any) => ({ ...p, cls: "loser" })), + ]; + + for (const p of allPatterns) { + await pool.query( + `INSERT INTO blog_learned_patterns (pattern_type, pattern_value, performance_class, sample_count) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO NOTHING`, + [p.type, p.value, p.cls, count] + ); + savedCount++; + } + + // Save weekly SLL state + const weekStart = new Date(); + weekStart.setDate(weekStart.getDate() - weekStart.getDay()); // Monday + await pool.query( + `INSERT INTO blog_sll_state + (week_start, winner_patterns, loser_patterns, top_topics, avoid_topics, + optimal_length_min, optimal_length_max, best_hook_patterns, posts_analyzed, generated_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'sll-analyze') + ON CONFLICT (week_start) DO UPDATE SET + winner_patterns = EXCLUDED.winner_patterns, + loser_patterns = EXCLUDED.loser_patterns, + top_topics = EXCLUDED.top_topics, + avoid_topics = EXCLUDED.avoid_topics, + optimal_length_min = EXCLUDED.optimal_length_min, + optimal_length_max = EXCLUDED.optimal_length_max, + best_hook_patterns = EXCLUDED.best_hook_patterns, + posts_analyzed = EXCLUDED.posts_analyzed, + generated_at = NOW()`, + [ + weekStart.toISOString().split("T")[0], + JSON.stringify(ep.winner_patterns || []), + JSON.stringify(ep.loser_patterns || []), + JSON.stringify(ep.top_topics || []), + JSON.stringify(ep.avoid_topics || []), + ep.optimal_length?.min ?? null, + ep.optimal_length?.max ?? null, + JSON.stringify(ep.best_hook_patterns || []), + count, + ] + ); + + console.log(`SLL: Pattern analysis complete — ${savedCount} patterns saved, ${count} posts analyzed`); + + res.json({ + success: true, + posts_analyzed: count, + patterns_saved: savedCount, + key_insight: ep.key_insight, + winner_patterns: ep.winner_patterns, + loser_patterns: ep.loser_patterns, + optimal_length: ep.optimal_length, + top_topics: ep.top_topics, + best_hook_patterns: ep.best_hook_patterns, + }); + } catch (err) { + console.error("SLL analyze error:", err); + res.status(500).json({ success: false, error: String(err) }); + } +}); diff --git a/packages/api/src/routes/news.ts b/packages/api/src/routes/news.ts new file mode 100644 index 0000000..d96c853 --- /dev/null +++ b/packages/api/src/routes/news.ts @@ -0,0 +1,58 @@ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; + +export const newsRouter = Router(); + +// GET /api/news?page=1&limit=10&category=&source= +newsRouter.get("/", async (req: Request, res: Response) => { + try { + const page = Math.max(1, parseInt(String(req.query.page || "1"), 10)); + const limit = Math.min(50, Math.max(1, parseInt(String(req.query.limit || "10"), 10))); + const offset = (page - 1) * limit; + const category = req.query.category ? String(req.query.category) : null; + const source = req.query.source ? String(req.query.source) : null; + + const conditions: string[] = []; + const values: unknown[] = []; + let idx = 1; + + if (category) { conditions.push(`category = $${idx++}`); values.push(category); } + if (source) { conditions.push(`source ILIKE $${idx++}`); values.push(`%${source}%`); } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + + const countRes = await pool.query( + `SELECT COUNT(*) AS total FROM news_articles ${where}`, + values + ); + const total = parseInt(countRes.rows[0].total, 10); + + const rows = await pool.query( + `SELECT id, title, summary, source, source_url, published_at, + category, relevance_score, tags, content_hash + FROM news_articles + ${where} + ORDER BY published_at DESC NULLS LAST + LIMIT $${idx} OFFSET $${idx + 1}`, + [...values, limit, offset] + ); + + // distinct categories for filter UI + const catRes = await pool.query( + "SELECT DISTINCT category FROM news_articles WHERE category IS NOT NULL ORDER BY category" + ) as { rows: { category: string }[] }; + + res.json({ + success: true, + articles: rows.rows, + total, + page, + limit, + pages: Math.ceil(total / limit), + categories: catRes.rows.map((r) => r.category), + }); + } catch (err) { + console.error("News route error:", err); + res.status(500).json({ success: false, error: "Internal server error" }); + } +}); diff --git a/packages/api/src/routes/vendors.ts b/packages/api/src/routes/vendors.ts index 825dfdd..bd15814 100644 --- a/packages/api/src/routes/vendors.ts +++ b/packages/api/src/routes/vendors.ts @@ -1,4 +1,5 @@ import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; import { listVendors } from "../db/queries"; export const vendorRouter = Router(); @@ -13,3 +14,125 @@ vendorRouter.get("/", async (req: Request, res: Response) => { res.status(500).json({ success: false, error: "Internal server error" }); } }); + +// POST /api/vendors — Create a new vendor + queue auto-crawl +vendorRouter.post("/", async (req: Request, res: Response) => { + try { + const { + name, + type, + website, + shop_url, + headquarters, + country, + founded_year, + revenue_usd, + employee_count, + market_position, + specialties, + is_competitor, + } = req.body as Record; + + if (!name || typeof name !== "string" || !name.trim()) { + return res.status(400).json({ success: false, error: "name is required" }); + } + + const validTypes = ["manufacturer", "distributor", "oem", "reseller", "compatible"]; + const resolvedType = validTypes.includes(type) ? type : "compatible"; + + // Generate slug from name + const slug = name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, ""); + + // Build specialties array + const specialtiesArr: string[] = Array.isArray(specialties) + ? specialties + : typeof specialties === "string" && specialties.trim() + ? specialties.split(",").map((s: string) => s.trim()).filter(Boolean) + : []; + + // Insert vendor + const insertResult = await pool.query( + `INSERT INTO vendors + (name, slug, type, website, shop_url, headquarters, country, + founded_year, revenue_usd, employee_count, market_position, specialties, + is_competitor, scrape_config) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + RETURNING *`, + [ + name.trim(), + slug, + resolvedType, + website || null, + shop_url || website || null, + headquarters || null, + country || null, + founded_year ? Number(founded_year) : null, + revenue_usd ? Number(revenue_usd) : null, + employee_count ? Number(employee_count) : null, + market_position || null, + specialtiesArr, + is_competitor === true || is_competitor === "true", + website ? JSON.stringify({ url: website, enabled: true, auto_queued: true }) : "{}", + ] + ); + + const vendor = insertResult.rows[0]; + + // Queue crawl job: insert into crawler_llm_log as a pending task signal + // The scraper fleet polls for new vendors with scrape_config.enabled=true + if (website) { + await pool + .query( + `INSERT INTO crawler_llm_log + (vendor_id, url, action, model, tokens_used, created_at) + VALUES ($1, $2, 'vendor_created_auto_crawl', 'system', 0, NOW())`, + [vendor.id, website] + ) + .catch(() => null); // Non-fatal — vendor is created regardless + } + + return res.status(201).json({ + success: true, + vendor, + crawl_queued: !!website, + message: website + ? `Vendor "${name}" created. Auto-crawl queued for ${website}.` + : `Vendor "${name}" created. Add a website URL to enable auto-crawl.`, + }); + } catch (err: any) { + if (err.code === "23505") { + // Unique constraint violation (name or slug) + return res.status(409).json({ success: false, error: "A vendor with this name already exists." }); + } + console.error("Create vendor error:", err); + return res.status(500).json({ success: false, error: "Internal server error" }); + } +}); + +// GET /api/vendors/:id — Get single vendor with full stats +vendorRouter.get("/:id", async (req: Request, res: Response) => { + try { + const { id } = req.params; + const result = await pool.query( + `SELECT v.*, + (SELECT COUNT(*) FROM transceivers t WHERE t.vendor_id = v.id)::int AS transceiver_count, + (SELECT COUNT(*) FROM switches s WHERE s.vendor_id = v.id)::int AS switch_count, + (SELECT COUNT(*) FROM price_observations po + JOIN transceivers t ON po.transceiver_id = t.id + WHERE t.vendor_id = v.id)::int AS price_obs_count + FROM vendors v + WHERE v.id::text = $1 OR v.slug = $1`, + [id] + ); + const vendor = result.rows[0]; + if (!vendor) return res.status(404).json({ success: false, error: "Vendor not found" }); + return res.json({ success: true, vendor }); + } catch (err) { + console.error("Get vendor error:", err); + return res.status(500).json({ success: false, error: "Internal server error" }); + } +}); diff --git a/packages/dashboard/hot-topics.js b/packages/dashboard/hot-topics.js index 06afd5d..22a23f9 100644 --- a/packages/dashboard/hot-topics.js +++ b/packages/dashboard/hot-topics.js @@ -1,12 +1,22 @@ /** - * Hot Topics + Blog Pipeline UX Enhancement (v0.2.5) + * Hot Topics + Blog Pipeline UX Enhancement (v0.3.0) * Loaded after main dashboard script. * Overrides generateBlog + pollBlogLlm with improved versions. + * + * v0.3.0: All fetch() calls now include Authorization Bearer token. */ (function() { var API = window.API || ''; var blogPipelineRunning = false; + /** Get auth headers — uses obfuscated token helper from index.html (loadToken) */ + function authHeaders(extra) { + var token = (window.loadToken ? window.loadToken() : localStorage.getItem('tip_token')) || ''; + var h = { 'Authorization': 'Bearer ' + token }; + if (extra) Object.assign(h, extra); + return h; + } + var STEP_NAMES = [ 'Topic Expansion', 'Angle Selection', 'Outline Generation', 'Master Draft (writing...)', 'Reality Injection', 'Technical Deepening', @@ -36,10 +46,12 @@ var body = { topic: topic }; if (speed) body.speed = speed; + if (customTitle) body.customTitle = customTitle; + if (customAngle) body.customAngle = customAngle; - fetch((API || '') + '/api/blog/generate', { + fetch(API + '/api/blog/generate', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: authHeaders({ 'Content-Type': 'application/json' }), body: JSON.stringify(body) }).then(function(r) { return r.json(); }).then(function(data) { if (data.success && data.draft) { @@ -60,8 +72,50 @@ }); }; + // Track how many consecutive polls had running=false (stall detection) + var _stallCount = 0; + + function showStallWarning(id) { + var status = document.getElementById('bp-status'); + var step = document.getElementById('bp-step'); + if (status) { + status.style.color = '#e6a800'; + status.textContent = '⚠ Pipeline nicht aktiv (API-Neustart?) — LLM läuft evtl. noch'; + } + if (step) { + step.innerHTML = 'Status unklar  ·  ' + + ''; + } + } + + window._resetAndRetry = function(id) { + _stallCount = 0; + // Reset Ollama queue then regenerate + fetch(API + '/api/blog/llm/reset-queue', { method: 'POST', headers: authHeaders() }).catch(function() {}); + var step = document.getElementById('bp-step'); + var status = document.getElementById('bp-status'); + if (status) { status.style.color = '#FF8100'; status.textContent = 'Restarting pipeline…'; } + if (step) step.textContent = 'Sending to LLM…'; + fetch(API + '/api/blog/' + id + '/regenerate', { + method: 'POST', + headers: authHeaders({ 'Content-Type': 'application/json' }) + }).then(function(r) { return r.json(); }) + .then(function(data) { + if (data.success) { + if (typeof showToast === 'function') showToast('Pipeline gestartet', 'LLM läuft neu — warte auf Ergebnis'); + pollPipeline(id, 0); + } else { + if (typeof showToast === 'function') showToast('Fehler', data.error || 'Regenerierung fehlgeschlagen', true); + } + }).catch(function(err) { + if (typeof showToast === 'function') showToast('Network Error', err.message, true); + }); + }; + function pollPipeline(id, attempt) { - if (attempt > 80) { + if (attempt > 90) { blogPipelineRunning = false; var pipelineEl = document.getElementById('blog-pipeline-status'); if (pipelineEl) pipelineEl.innerHTML = ''; @@ -69,26 +123,30 @@ if (typeof loadBlogDrafts === 'function') loadBlogDrafts(); return; } - // Poll progress endpoint (fast) + blog status endpoint (to detect completion) setTimeout(function() { - // 1) Fetch real-time progress from server - fetch((API || '') + '/api/blog/' + id + '/progress') + // 1) Fetch real-time progress + fetch(API + '/api/blog/' + id + '/progress', { headers: authHeaders() }) .then(function(r) { return r.json(); }) .then(function(prog) { if (prog.running) { + _stallCount = 0; var bar = document.getElementById('bp-bar'); var pct = document.getElementById('bp-pct'); var status = document.getElementById('bp-status'); var step = document.getElementById('bp-step'); if (bar) bar.style.width = prog.pct + '%'; if (pct) pct.textContent = prog.pct + '%'; - if (status) status.textContent = prog.label || ('Step ' + prog.step + '/10'); - if (step) step.textContent = 'Running on qwen2.5:14b via Ollama...'; + if (status) { status.style.color = '#FF8100'; status.textContent = prog.label || ('Step ' + prog.step + '/10'); } + if (step) step.textContent = 'Step ' + prog.step + '/10 · qwen2.5:14b via Ollama'; + } else { + _stallCount++; + // After 5 consecutive non-running polls (~40s), show stall warning + if (_stallCount >= 5) showStallWarning(id); } - }).catch(function() {}); + }).catch(function() { _stallCount++; }); - // 2) Fetch blog draft to detect pipeline completion - fetch((API || '') + '/api/blog/' + id) + // 2) Fetch blog draft to detect completion + fetch(API + '/api/blog/' + id, { headers: authHeaders() }) .then(function(r) { return r.json(); }) .then(function(data) { var d = data.draft || data; @@ -96,7 +154,6 @@ var done = gen && gen !== 'tip-blog-engine-template' && gen.length > 0; if (done) { - // Pipeline complete! var bar = document.getElementById('bp-bar'); var pct = document.getElementById('bp-pct'); var status = document.getElementById('bp-status'); @@ -106,6 +163,7 @@ if (status) { status.textContent = '✓ Blog fertig! ' + (d.word_count || '?') + ' Wörter'; status.style.color = '#2d6a4f'; } if (step) step.textContent = 'Engine: ' + gen; blogPipelineRunning = false; + _stallCount = 0; if (typeof showToast === 'function') showToast('Blog Ready!', (d.title || 'Article') + ' — ' + (d.word_count || '?') + ' words'); setTimeout(function() { var pipelineEl = document.getElementById('blog-pipeline-status'); @@ -117,7 +175,7 @@ pollPipeline(id, attempt + 1); } }).catch(function() { pollPipeline(id, attempt + 1); }); - }, 8000); // Poll every 8s instead of 15s for snappier UI + }, 8000); } // Hot topics loader @@ -127,44 +185,44 @@ if (!grid) return; grid.innerHTML = '
Discovering hot topics...
'; - fetch((API || '') + '/api/hot-topics').then(function(r) { return r.json(); }).then(function(data) { - if (!data.topics || data.topics.length === 0) { - grid.innerHTML = '
Hype Cycle Analysis
'; - return; - } + fetch(API + '/api/hot-topics', { headers: authHeaders() }) + .then(function(r) { return r.json(); }) + .then(function(data) { + if (!data.topics || data.topics.length === 0) { + grid.innerHTML = '
Hype Cycle Analysis
'; + return; + } - // Update subtitle with next-refresh countdown - if (subtitle && data.refreshes_at) { - var nextRefresh = new Date(data.refreshes_at); - var hoursLeft = Math.round((nextRefresh - new Date()) / 3600000); - subtitle.textContent = 'auto-discovered · rotates daily · next refresh in ' + hoursLeft + 'h'; - } + if (subtitle && data.refreshes_at) { + var nextRefresh = new Date(data.refreshes_at); + var hoursLeft = Math.round((nextRefresh - new Date()) / 3600000); + subtitle.textContent = data.total + ' topics · rotates daily · next refresh in ' + hoursLeft + 'h · sources: ' + (data.sources || []).join(', '); + } - var colors = { breaking: '#c1121f', hot: '#FF8100', trending: '#e6a800', emerging: '#2d6a4f' }; - grid.innerHTML = data.topics.map(function(t) { - var c = colors[t.urgency] || '#888'; - // Pass full topic title and angle as data attributes to avoid quote-escaping hell - var cardId = 'ht-' + Math.random().toString(36).slice(2, 8); - // Store topic data for onclick - window['_ht_' + cardId] = t; - return '
' + - '
' + - '' + (t.urgency || '') + '' + - '' + (t.source_type || '') + '
' + - '
' + (t.title || '') + '
' + - '
' + (t.suggested_angle || t.description || '').slice(0, 90) + '
' + - '
'; - }).join(''); - }).catch(function() { - grid.innerHTML = - '
Hype Cycle
' + - '
Comparison
' + - '
Tutorial
'; - }); + var colors = { breaking: '#c1121f', hot: '#FF8100', trending: '#e6a800', emerging: '#2d6a4f' }; + grid.innerHTML = data.topics.map(function(t) { + var c = colors[t.urgency] || '#888'; + var cardId = 'ht-' + Math.random().toString(36).slice(2, 8); + window['_ht_' + cardId] = t; + return '
' + + '
' + + '' + (t.urgency || '') + '' + + '' + (t.source_type || '') + ' · ' + (t.source || '') + '
' + + '
' + (t.title || '') + '
' + + '
' + (t.suggested_angle || t.description || '').slice(0, 100) + '
' + + '
'; + }).join(''); + }).catch(function(err) { + console.error('[HotTopics] fetch error:', err); + grid.innerHTML = + '
Hype Cycle
' + + '
Comparison
' + + '
Tutorial
'; + }); }; - // Generate blog from hot topic card — uses title + angle from stored topic object + // Generate blog from hot topic card window._generateFromHotTopic = function(cardId) { var t = window['_ht_' + cardId]; if (!t) return; @@ -191,8 +249,11 @@ window.blogDeleteClick = function(el) { // Delete a single blog draft window.deleteBlogDraft = function(id, title) { if (!confirm('Delete "' + title + '"?')) return; - fetch((window.API || '') + '/api/blog/' + id, { method: 'DELETE' }) - .then(function(r) { return r.json(); }) + var token = (window.loadToken ? window.loadToken() : localStorage.getItem('tip_token')) || ''; + fetch((window.API || '') + '/api/blog/' + id, { + method: 'DELETE', + headers: { 'Authorization': 'Bearer ' + token } + }).then(function(r) { return r.json(); }) .then(function(data) { if (data.success) { if (typeof showToast === 'function') showToast('Deleted', title); @@ -201,15 +262,25 @@ window.deleteBlogDraft = function(id, title) { }); }; -// Delete all template drafts (keep LLM-generated ones) +// Delete all template drafts window.deleteAllTemplateDrafts = function() { if (!confirm('Delete ALL template drafts? LLM-generated articles will be kept.')) return; - fetch((window.API || '') + '/api/blog').then(function(r) { return r.json(); }).then(function(data) { - var templates = (data.drafts || []).filter(function(d) { return d.generated_by === 'tip-blog-engine-template' || !d.generated_by; }); - var count = 0; - templates.forEach(function(d) { - fetch((window.API || '') + '/api/blog/' + d.id, { method: 'DELETE' }).then(function() { count++; if (count === templates.length && typeof loadBlogDrafts === 'function') loadBlogDrafts(); }); + var token = (window.loadToken ? window.loadToken() : localStorage.getItem('tip_token')) || ''; + var authH = { 'Authorization': 'Bearer ' + token }; + fetch((window.API || '') + '/api/blog', { headers: authH }) + .then(function(r) { return r.json(); }) + .then(function(data) { + var templates = (data.drafts || []).filter(function(d) { + return d.generated_by === 'tip-blog-engine-template' || !d.generated_by; + }); + var count = 0; + templates.forEach(function(d) { + fetch((window.API || '') + '/api/blog/' + d.id, { method: 'DELETE', headers: authH }) + .then(function() { + count++; + if (count === templates.length && typeof loadBlogDrafts === 'function') loadBlogDrafts(); + }); + }); + if (typeof showToast === 'function') showToast('Cleaning', 'Deleting ' + templates.length + ' template drafts...'); }); - if (typeof showToast === 'function') showToast('Cleaning', 'Deleting ' + templates.length + ' template drafts...'); - }); }; diff --git a/packages/dashboard/login.html b/packages/dashboard/login.html index 6605d4a..9630f92 100644 --- a/packages/dashboard/login.html +++ b/packages/dashboard/login.html @@ -286,9 +286,35 @@ var API = '/api/auth/login'; var REDIRECT = '/dashboard/'; + // ── Token storage helpers — never store plaintext ─────────────────── + (function() { + var _K = 'tip_v3_tk'; + var _x = 'fx9z2mq8'; + function _enc(s) { + var r = ''; + for (var i = 0; i < s.length; i++) r += String.fromCharCode(s.charCodeAt(i) ^ _x.charCodeAt(i % _x.length)); + return btoa(r); + } + function _dec(s) { + try { + var b = atob(s); var r = ''; + for (var i = 0; i < b.length; i++) r += String.fromCharCode(b.charCodeAt(i) ^ _x.charCodeAt(i % _x.length)); + return r; + } catch(e) { return ''; } + } + window.saveToken = function(t) { localStorage.setItem(_K, _enc(t)); localStorage.removeItem('tip_token'); }; + window.loadToken = function() { + var v = localStorage.getItem(_K); + if (v) return _dec(v) || ''; + var old = localStorage.getItem('tip_token'); + if (old) { window.saveToken(old); return old; } + return ''; + }; + })(); + // If already logged in → skip straight to dashboard (function() { - var t = localStorage.getItem('tip_token'); + var t = window.loadToken(); if (!t) return; fetch('/api/auth/verify', { headers: { Authorization: 'Bearer ' + t } }) .then(function(r) { if (r.ok) window.location.replace(REDIRECT); }) @@ -337,7 +363,7 @@ }); var data = await res.json(); if (res.ok && data.token) { - localStorage.setItem('tip_token', data.token); + window.saveToken(data.token); window.location.replace(REDIRECT); } else { showError(data.error || 'Invalid password. Please try again.'); diff --git a/packages/scraper/src/scrapers/gbics.ts b/packages/scraper/src/scrapers/gbics.ts index 238e1db..92aafe0 100644 --- a/packages/scraper/src/scrapers/gbics.ts +++ b/packages/scraper/src/scrapers/gbics.ts @@ -98,31 +98,23 @@ function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product // Collapse whitespace for easier regex matching const collapsed = html.replace(/\s+/g, " "); - // BigCommerce card-title pattern: - // - const productRegex = /aria-label="([^"]+)"\s+href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"[^>]*data-event-type="product-click"/gi; + // BigCommerce article card pattern (updated): + //
+ // + // Price is in pence (integer), divide by 100 = GBP + const articleRegex = /data-name="([^"]{10,200})"[^>]*data-product-price="\s*(\d+)\s*"[^>]*>[\s\S]{0,500}?href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi; let match; - while ((match = productRegex.exec(collapsed)) !== null) { - const label = match[1].trim(); - const url = match[2]; - - // aria-label contains "Product Name, £XX.XX" - // Split on last comma to separate name and price - const priceInLabel = label.match(/,\s*£\s*([\d,.]+)\s*$/); - const name = priceInLabel ? label.slice(0, label.lastIndexOf(",")).trim() : label; - let price = priceInLabel ? parseFloat(priceInLabel[1].replace(",", "")) : undefined; - - // Fallback: extract price from data-price-asc attribute on parent
  • - if (!price) { - const priceContext = collapsed.slice(Math.max(0, match.index - 500), match.index); - const dataPriceMatch = priceContext.match(/data-price-asc="(\d+)"/); - if (dataPriceMatch) price = parseFloat(dataPriceMatch[1]); - } + while ((match = articleRegex.exec(collapsed)) !== null) { + const name = match[1].trim(); + const priceRaw = parseInt(match[2], 10); + const url = match[3]; + // GBICS stores price in pence (integer) — e.g. 2395 = £23.95 OR £2,395.00 (full pounds)? + // Check by data-price-asc context: "data-price-asc=\"2395\"" with "£2,395.00" → price is in full GBP (no pence) + const price = priceRaw > 0 ? priceRaw : undefined; if (name.length < 10) continue; const reach = detectReach(name); - // Part number: first segment before " - " const partParts = name.split(/\s+-\s+/); const partNumber = partParts[0]?.trim().slice(0, 80) || url.split("/").filter(Boolean).pop() || ""; @@ -136,6 +128,29 @@ function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product }); } + // Fallback: aria-label pattern + if (products.length === 0) { + const ariaRegex = /aria-label="([^"]+£[^"]+)"\s+href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi; + while ((match = ariaRegex.exec(collapsed)) !== null) { + const label = match[1].trim(); + const url = match[2]; + const priceMatch = label.match(/£\s*([\d,.]+)/); + const name = label.split(",")[0]?.trim() || label; + const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined; + if (name.length < 10) continue; + const reach = detectReach(name); + products.push({ + partNumber: name.split(/\s+-\s+/)[0]?.trim().slice(0, 80) || "", + name, url, + price: price && price > 0 && price < 50000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + compatibleWith: extractCompatibleVendor(name), + }); + } + } + // Fallback: try "Now: £XX.XX" pattern near product links if (products.length === 0) { const altRegex = /href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"[^>]*>\s*([^<]{15,})<\/a>/gi; diff --git a/sql/025-blog-sll.sql b/sql/025-blog-sll.sql new file mode 100644 index 0000000..edc9361 --- /dev/null +++ b/sql/025-blog-sll.sql @@ -0,0 +1,127 @@ +-- Migration 025: Blog Self-Learning Loop (SLL v1.0) +-- Tracks LinkedIn engagement per post, extracts winning/losing patterns, +-- feeds insights back into the generation pipeline automatically. + +-- ───────────────────────────────────────────────────────── +-- Blog Performance: LinkedIn/platform engagement per post +-- ───────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS blog_performance ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + blog_id UUID REFERENCES blog_drafts(id) ON DELETE CASCADE, + platform TEXT DEFAULT 'linkedin', + + -- Raw engagement metrics (LinkedIn: saves > shares > comments > likes) + impressions INTEGER, + comments INTEGER DEFAULT 0, + shares INTEGER DEFAULT 0, + saves INTEGER DEFAULT 0, + likes INTEGER DEFAULT 0, + + -- SLL score formula: (comments×3) + (shares×2) + (saves×2), likes = 0 weight + engagement_score INTEGER GENERATED ALWAYS AS ( + COALESCE(comments,0)*3 + COALESCE(shares,0)*2 + COALESCE(saves,0)*2 + ) STORED, + + -- Snapshot of article at time of measurement (for pattern extraction) + hook_text TEXT, -- first ~120 chars of the LinkedIn post + blog_type TEXT, -- tutorial | market_alert | comparison | … + topic TEXT, -- topic tag used during generation + word_count INTEGER, + pipeline_version TEXT, + + -- Timing + posted_at TIMESTAMPTZ, + measured_at TIMESTAMPTZ DEFAULT NOW(), + notes TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_blog_perf_blog ON blog_performance(blog_id); +CREATE INDEX IF NOT EXISTS idx_blog_perf_score ON blog_performance(engagement_score DESC); +CREATE INDEX IF NOT EXISTS idx_blog_perf_type ON blog_performance(blog_type); +CREATE INDEX IF NOT EXISTS idx_blog_perf_posted ON blog_performance(posted_at DESC); + +-- ───────────────────────────────────────────────────────── +-- Learned Patterns: extracted winning / losing patterns +-- ───────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS blog_learned_patterns ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + pattern_type TEXT NOT NULL, -- hook | structure | topic | length | opening | verb_style + pattern_value TEXT NOT NULL, -- "short hook + contrast", "lab vs production", "single scenario" + performance_class TEXT NOT NULL, -- winner | loser + avg_engagement NUMERIC, -- average engagement_score across samples + sample_count INTEGER DEFAULT 1, + example_post_id UUID REFERENCES blog_drafts(id), + active BOOLEAN DEFAULT TRUE, + extracted_at TIMESTAMPTZ DEFAULT NOW(), + last_updated TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_blog_patterns_type ON blog_learned_patterns(pattern_type); +CREATE INDEX IF NOT EXISTS idx_blog_patterns_class ON blog_learned_patterns(performance_class); +CREATE INDEX IF NOT EXISTS idx_blog_patterns_active ON blog_learned_patterns(active) WHERE active = TRUE; + +-- ───────────────────────────────────────────────────────── +-- SLL State: current active learning snapshot (weekly) +-- ───────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS blog_sll_state ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + week_start DATE NOT NULL UNIQUE, + + -- Top-performing patterns (JSON arrays of strings) + winner_patterns JSONB DEFAULT '[]', + loser_patterns JSONB DEFAULT '[]', + + -- Topic performance + top_topics JSONB DEFAULT '[]', + avoid_topics JSONB DEFAULT '[]', + + -- Length optimization + optimal_length_min INTEGER, + optimal_length_max INTEGER, + + -- Hook patterns that convert + best_hook_patterns JSONB DEFAULT '[]', + + -- Meta + posts_analyzed INTEGER DEFAULT 0, + avg_engagement_score NUMERIC, + generated_at TIMESTAMPTZ DEFAULT NOW(), + generated_by TEXT DEFAULT 'sll-auto' +); + +-- ───────────────────────────────────────────────────────── +-- View: performance enriched with blog metadata +-- ───────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_blog_performance AS +SELECT + p.id, + p.blog_id, + d.title, + d.topic, + d.pipeline_version, + d.word_count, + p.platform, + p.impressions, + p.comments, + p.shares, + p.saves, + p.likes, + p.engagement_score, + p.hook_text, + p.blog_type, + p.posted_at, + p.measured_at, + -- Performance tier based on engagement score + CASE + WHEN p.engagement_score >= 20 THEN 'gold' + WHEN p.engagement_score >= 10 THEN 'silver' + WHEN p.engagement_score >= 4 THEN 'bronze' + ELSE 'miss' + END AS performance_tier +FROM blog_performance p +JOIN blog_drafts d ON d.id = p.blog_id; + +COMMENT ON TABLE blog_performance IS 'SLL v1.0: LinkedIn engagement tracking per blog post'; +COMMENT ON TABLE blog_learned_patterns IS 'SLL v1.0: Extracted winning/losing content patterns'; +COMMENT ON TABLE blog_sll_state IS 'SLL v1.0: Weekly learning state snapshot for pipeline injection';