diff --git a/packages/dashboard/index.html b/packages/dashboard/index.html index d43cc17..bb12052 100644 --- a/packages/dashboard/index.html +++ b/packages/dashboard/index.html @@ -1208,6 +1208,55 @@
Loading topics...
+ + +
+
+
+ 🧠 Self-Learning Loop (SLL v1.0) + loading… +
+
+ + +
+
+
Loading SLL data…
+ + +
+ Log LinkedIn engagement for a post: + +
+ +
+
@@ -1804,7 +1853,7 @@ function goToTab(tabName) { if (tabName === 'news') loadNews(1); if (tabName === 'vendors') loadVendors(); if (tabName === 'standards') loadStandardsList(); - if (tabName === 'blog') loadBlogDrafts(); + if (tabName === 'blog') { loadBlogDrafts(); loadSLLInsights(); } if (tabName === 'finder') document.getElementById('finder-switch-input').focus(); if (tabName === 'crawlers') loadCrawlerStatus(); if (tabName === 'procurement') loadProcurement(); @@ -2669,16 +2718,26 @@ async function openTxDetail(id) { var iVer = t.image_verified === true; var dVer = t.details_verified === true; var fVer = t.fully_verified === true; + // Competitor verified: at least 1 price from a non-Flexoptix vendor in last 30 days + var allPricesForBadge = (t.competitor_prices || []).filter(function(p) { return p.url && p.price > 0; }); + var cVer = allPricesForBadge.some(function(p) { + return p.vendor_name && p.vendor_name.toUpperCase().indexOf('FLEXOPTIX') === -1; + }); var verItems = []; if (pVer) verItems.push('βœ“ Price'); if (iVer) verItems.push('βœ“ Image'); if (dVer) verItems.push('βœ“ Details'); - if (fVer) { + if (cVer) verItems.push('βœ“ Competitor'); + else verItems.push('⟳ Competitor'); + // 100% VERIFIED requires all 4: Price + Image + Details + Competitor + var fullyVerified = fVer && cVer; + if (fullyVerified) { // Inside the green bar: all text must be white/light β€” not #2d6a4f (same as bg) var fvItems = []; if (pVer) fvItems.push('βœ“ Price'); if (iVer) fvItems.push('βœ“ Image'); if (dVer) fvItems.push('βœ“ Details'); + fvItems.push('βœ“ Competitor'); h += '
' + 'β˜… 100% VERIFIED' + '–' @@ -4003,7 +4062,7 @@ function pollBlogLlm(id, attempt) { var badge = document.querySelector('.ri[data-blog-id="' + id + '"] .blog-status-badge'); if (badge) { badge.className = 'b b-green blog-status-badge'; - badge.textContent = 'ready'; + badge.textContent = 'ready βœ“'; } api('/api/blog/' + id).then(function(data) { if (data.draft) { @@ -4016,7 +4075,7 @@ function pollBlogLlm(id, attempt) { var badge = document.querySelector('.ri[data-blog-id="' + id + '"] .blog-status-badge'); if (badge) { badge.className = 'b b-yellow blog-status-badge'; - badge.textContent = 'step ' + p.step + '/10'; + badge.textContent = 'step ' + p.step + '/14'; } pollBlogLlm(id, attempt + 1); } @@ -4206,7 +4265,7 @@ async function loadBlogDrafts() { buildDOM(el('blog-list'), drafts.map(function(d) { var p = progressMap[d.id] || {}; var isRunning = p.running === true; - // Status badge: generating β†’ step X/10, done β†’ ready, published β†’ published, review β†’ review + // Status badge: generating β†’ step X/N, review/fo-blog done β†’ ready βœ“, approved β†’ approved, published β†’ published, draft β†’ draft var statusLabel, statusClass; if (isRunning) { statusLabel = p.step ? 'step ' + p.step + '/' + (p.total || 14) : 'generating…'; @@ -4214,10 +4273,10 @@ async function loadBlogDrafts() { } else if (d.status === 'published') { statusLabel = 'published'; statusClass = 'b-green'; - } else if (d.status === 'review') { - statusLabel = 'review'; - statusClass = 'b-yellow'; - } else if (d.pipeline_steps_completed >= 10 || (d.generated_by || '').includes('fo-blog')) { + } else if (d.status === 'approved') { + statusLabel = 'approved βœ“'; + statusClass = 'b-green'; + } else if (d.status === 'review' || d.pipeline_steps_completed >= 10 || (d.generated_by || '').includes('fo-blog')) { statusLabel = 'ready βœ“'; statusClass = 'b-green'; } else { @@ -4335,9 +4394,10 @@ async function updateBlogStatus(id, status) { async function regenerateBlog(id) { showToast('Regenerating…', 'LLM pipeline wird neu gestartet'); try { + var token = window.loadToken ? window.loadToken() : ''; var data = await fetch(API + '/api/blog/' + id + '/regenerate', { method: 'POST', - headers: { 'Content-Type': 'application/json' } + headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + token } }).then(function(r) { return r.json(); }); if (data.success) { showToast('Gestartet', 'LLM lΓ€uft – Status wird aktualisiert'); @@ -4350,6 +4410,153 @@ async function regenerateBlog(id) { } catch(e) { showToast('Error', e.message, true); } } +// ───────────────────────────────────────────────────────────────── +// SLL v1.0 β€” Self-Learning Loop +// ───────────────────────────────────────────────────────────────── + +async function loadSLLInsights() { + var token = window.loadToken ? window.loadToken() : ''; + var el = document.getElementById('sll-insights-content'); + var badge = document.getElementById('sll-status-badge'); + if (!el) return; + try { + var r = await fetch(API + '/api/blog/sll/insights', { headers: { 'Authorization': 'Bearer ' + token } }); + var d = await r.json(); + if (!d.success) { el.innerHTML = 'Error loading SLL data'; return; } + + var stats = d.stats; + var ready = d.sll_ready; + badge.textContent = ready ? 'Active β€” ' + stats.total_posts + ' posts' : 'Needs data β€” ' + stats.total_posts + '/5 posts'; + badge.style.background = ready ? 'rgba(34,197,94,0.2)' : 'rgba(234,179,8,0.2)'; + badge.style.color = ready ? '#4ade80' : '#fbbf24'; + + var h = '
'; + h += '
' + (stats.best_score || 0) + '
Best Score
'; + h += '
' + (stats.tiers.gold || 0) + '
πŸ₯‡ Gold
'; + h += '
' + (stats.tiers.silver || 0) + '
πŸ₯ˆ Silver
'; + h += '
' + (stats.tiers.miss || 0) + '
Miss
'; + h += '
'; + + var winners = d.learned_patterns.winners || []; + var losers = d.learned_patterns.losers || []; + + if (winners.length > 0) { + h += '
βœ” WHAT WORKS
'; + h += '
'; + winners.forEach(function(p) { + h += '[' + p.pattern_type + '] ' + p.pattern_value + ''; + }); + h += '
'; + } + + if (losers.length > 0) { + h += '
βœ— WHAT FAILS
'; + h += '
'; + losers.forEach(function(p) { + h += '[' + p.pattern_type + '] ' + p.pattern_value + ''; + }); + h += '
'; + } + + if (winners.length === 0 && losers.length === 0) { + h += '
' + d.note + '
'; + } + + if (d.top_posts && d.top_posts.length > 0) { + h += '
TOP PERFORMERS: '; + h += d.top_posts.slice(0,3).map(function(p) { return '' + (p.title || '?').slice(0,40) + ' (' + p.engagement_score + ')'; }).join(' Β· '); + h += '
'; + } + + el.innerHTML = h; + + // Also populate blog select for performance form + populateSLLBlogSelect(); + } catch(e) { + el.innerHTML = 'SLL data unavailable'; + } +} + +async function populateSLLBlogSelect() { + var token = window.loadToken ? window.loadToken() : ''; + var sel = document.getElementById('sll-blog-select'); + if (!sel) return; + try { + var r = await fetch(API + '/api/blog?limit=50', { headers: { 'Authorization': 'Bearer ' + token } }); + var d = await r.json(); + var drafts = (d.drafts || d.data || []); + sel.innerHTML = ''; + drafts.forEach(function(b) { + var opt = document.createElement('option'); + opt.value = b.id; + opt.textContent = (b.title || 'Untitled').slice(0, 60) + ' (' + (b.status || '?') + ')'; + sel.appendChild(opt); + }); + } catch(e) { /* ignore */ } +} + +function showSLLPerformanceForm() { + var form = document.getElementById('sll-perf-form'); + if (!form) return; + form.style.display = form.style.display === 'none' ? 'block' : 'none'; + if (form.style.display === 'block') populateSLLBlogSelect(); +} + +async function submitSLLPerformance() { + var token = window.loadToken ? window.loadToken() : ''; + var blogId = document.getElementById('sll-blog-select').value; + if (!blogId) { showToast('Fehler', 'Bitte Blog-Post auswΓ€hlen', true); return; } + var comments = parseInt(document.getElementById('sll-comments').value) || 0; + var shares = parseInt(document.getElementById('sll-shares').value) || 0; + var saves = parseInt(document.getElementById('sll-saves').value) || 0; + var impressions = parseInt(document.getElementById('sll-impressions').value) || null; + + try { + var r = await fetch(API + '/api/blog/' + blogId + '/performance', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + token }, + body: JSON.stringify({ comments: comments, shares: shares, saves: saves, impressions: impressions }) + }); + var d = await r.json(); + if (d.success) { + var tier = d.tier; + var tierEmoji = tier === 'gold' ? 'πŸ₯‡' : tier === 'silver' ? 'πŸ₯ˆ' : tier === 'bronze' ? 'πŸ₯‰' : 'πŸ“‰'; + showToast('Gespeichert ' + tierEmoji, 'Score: ' + d.engagement_score + ' (' + tier + ')'); + document.getElementById('sll-perf-form').style.display = 'none'; + document.getElementById('sll-comments').value = '0'; + document.getElementById('sll-shares').value = '0'; + document.getElementById('sll-saves').value = '0'; + document.getElementById('sll-impressions').value = ''; + loadSLLInsights(); + } else { + showToast('Fehler', d.error || 'Unbekannter Fehler', true); + } + } catch(e) { showToast('Error', e.message, true); } +} + +async function sllAnalyze() { + var token = window.loadToken ? window.loadToken() : ''; + var btn = document.getElementById('sll-analyze-btn'); + if (btn) { btn.textContent = '⏳ Analyzing…'; btn.disabled = true; } + try { + var r = await fetch(API + '/api/blog/sll/analyze', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + token } + }); + var d = await r.json(); + if (d.success) { + showToast('SLL Analysis Complete', (d.posts_analyzed || 0) + ' posts Β· ' + (d.patterns_saved || 0) + ' patterns Β· "' + (d.key_insight || '') + '"'); + loadSLLInsights(); + } else { + showToast('Fehler', d.error || 'Analysis failed', true); + } + } catch(e) { + showToast('Error', e.message, true); + } finally { + if (btn) { btn.textContent = '⚑ Analyze Patterns'; btn.disabled = false; } + } +} + // TABLE SORTING function makeSortable(table) { if (!table) return; diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index ae9ae00..536b7c0 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -52,8 +52,6 @@ export async function createScheduler(): Promise { retryBackoff: true, expireInSeconds: 300, monitorStateIntervalSeconds: 60, - max: 4, // pg-boss internal connection pool - poolSize: 4, // alias used by some pg-boss versions }); boss.on("error", (error) => console.error("pg-boss error:", error)); @@ -137,20 +135,20 @@ export async function registerSchedules(boss: PgBoss): Promise { } // ══════════════════════════════════════════════════════════════════════ - // PLAYWRIGHT SCRAPERS β€” every 8h (resource-heavy, runs on Erik VPS) + // PLAYWRIGHT SCRAPERS β€” priority competitors every 2h, others every 4h // ══════════════════════════════════════════════════════════════════════ - // FS.com: 01:00, 09:00, 17:00 - await boss.schedule("scrape:pricing:fs", "0 1,9,17 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 }); + // FS.com: every 2h β€” primary competitor, highest data value + await boss.schedule("scrape:pricing:fs", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 }); - // 10Gtek: 01:20, 09:20, 17:20 - await boss.schedule("scrape:pricing:10gtek", "20 1,9,17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + // 10Gtek: every 2h offset by 20min + await boss.schedule("scrape:pricing:10gtek", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // ATGBICS: 01:50, 09:50, 17:50 - await boss.schedule("scrape:pricing:atgbics", "50 1,9,17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + // ATGBICS: every 4h (staggered) + await boss.schedule("scrape:pricing:atgbics", "50 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // ProLabs: 02:20, 10:20, 18:20 - await boss.schedule("scrape:pricing:prolabs", "20 2,10,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + // ProLabs: every 2h offset by 40min + await boss.schedule("scrape:pricing:prolabs", "40 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // FETCH/CHEERIO SCRAPERS β€” every 4h (lightweight, Pi-friendly)