diff --git a/sql/128-url-normalize-and-enrichment-worklist.sql b/sql/128-url-normalize-and-enrichment-worklist.sql new file mode 100644 index 0000000..adfa8ed --- /dev/null +++ b/sql/128-url-normalize-and-enrichment-worklist.sql @@ -0,0 +1,75 @@ +-- sql/128-url-normalize-and-enrichment-worklist.sql +-- Foundation for the datasheet-enrichment pipeline (Lever 3): +-- (1) Fix malformed double-prefixed URLs (a scraper concatenated the base host in +-- front of an already-absolute URL, e.g. "https://www.mouser.dehttps://www.mouser.de/…"). +-- 503 rows affected (details_source_url 373, product_page_url 83, image_url 47). +-- These cannot be fetched, so they block any enrichment run. +-- (2) A guard so the ingest bug cannot re-introduce them. +-- (3) v_datasheet_enrichment_worklist — the pipeline's input: every transceiver/switch +-- that needs a field filled from source, with its best source URL + type. +-- Idempotent; safe to re-run. The regex only matches a second scheme that appears +-- BEFORE the first path slash, so legitimate URLs with an encoded URL in the query string +-- (…?url=https://…) are never touched. + +create or replace function fn_strip_double_url_prefix(u text) returns text +language sql immutable as $$ + select case when u ~ '^https?://[^/]*https?://' + then regexp_replace(u, '^https?://[^/]*(https?://)', '\1') + else u end +$$; + +comment on function fn_strip_double_url_prefix(text) is + 'Removes a duplicated leading scheme+host from a URL, e.g. https://a.comhttps://a.com/x -> https://a.com/x. Only when the 2nd scheme precedes the first path slash (so ?url=https://… is safe).'; + +-- (1) one-time cleanup +update transceivers set + details_source_url = fn_strip_double_url_prefix(details_source_url), + product_page_url = fn_strip_double_url_prefix(product_page_url), + image_url = fn_strip_double_url_prefix(image_url) +where details_source_url ~ '^https?://[^/]*https?://' + or product_page_url ~ '^https?://[^/]*https?://' + or image_url ~ '^https?://[^/]*https?://'; + +-- (2) ingest guard +create or replace function transceiver_url_normalize() returns trigger +language plpgsql as $$ +begin + NEW.details_source_url := fn_strip_double_url_prefix(NEW.details_source_url); + NEW.product_page_url := fn_strip_double_url_prefix(NEW.product_page_url); + NEW.image_url := fn_strip_double_url_prefix(NEW.image_url); + NEW.datasheet_url := fn_strip_double_url_prefix(NEW.datasheet_url); + return NEW; +end $$; + +drop trigger if exists trg_transceiver_url_normalize on transceivers; +create trigger trg_transceiver_url_normalize + before insert or update on transceivers + for each row execute function transceiver_url_normalize(); + +-- (3) enrichment worklist — pipeline input +create or replace view v_datasheet_enrichment_worklist as +select 'switch'::text as entity, s.id as entity_id, v.name as vendor, s.model as item, + 'ports_config+max_speed'::text as need, + coalesce(s.datasheet_url, s.product_page_url) as source_url, + case when s.datasheet_url is not null then 'datasheet' + when s.product_page_url is not null then 'product_page' + else 'none' end as source_type +from switches s join vendors v on v.id = s.vendor_id +where (s.ports_config is null or s.ports_config = '{}'::jsonb) and s.max_speed_gbps is null +union all +select 'transceiver', t.id, v.name, t.part_number, + 'form_factor+speed', + coalesce(t.details_source_url, t.datasheet_url, t.product_page_url), + case when t.datasheet_url is not null then 'datasheet' + when t.details_source_url is not null then 'details_page' + when t.product_page_url is not null then 'product_page' + else 'none' end +from transceivers t join vendors v on v.id = t.vendor_id +where t.speed_gbps is null or t.speed_gbps <= 0; + +comment on view v_datasheet_enrichment_worklist is + 'Input for the datasheet-enrichment pipeline: entities whose ports_config/max_speed (switch) or form_factor/speed (transceiver) must be filled from source, with the best available source URL and its type. source_type=none means URL discovery is needed first.'; + +insert into _migrations (filename, applied_at) +select '128-url-normalize-and-enrichment-worklist.sql', now() +where not exists (select 1 from _migrations where filename = '128-url-normalize-and-enrichment-worklist.sql');