transceiver-db/sql/128-url-normalize-and-enrichment-worklist.sql

-- sql/128-url-normalize-and-enrichment-worklist.sql
-- Foundation for the datasheet-enrichment pipeline (Lever 3):
--   (1) Fix malformed double-prefixed URLs (a scraper concatenated the base host in
--       front of an already-absolute URL, e.g. "https://www.mouser.dehttps://www.mouser.de/…").
--       503 rows affected (details_source_url 373, product_page_url 83, image_url 47).
--       These cannot be fetched, so they block any enrichment run.
--   (2) A guard so the ingest bug cannot re-introduce them.
--   (3) v_datasheet_enrichment_worklist — the pipeline's input: every transceiver/switch
--       that needs a field filled from source, with its best source URL + type.
-- Idempotent; safe to re-run. The regex only matches a second scheme that appears
-- BEFORE the first path slash, so legitimate URLs with an encoded URL in the query string
-- (…?url=https://…) are never touched.

create or replace function fn_strip_double_url_prefix(u text) returns text
language sql immutable as $$
  select case when u ~ '^https?://[^/]*https?://'
              then regexp_replace(u, '^https?://[^/]*(https?://)', '\1')
              else u end
$$;

comment on function fn_strip_double_url_prefix(text) is
  'Removes a duplicated leading scheme+host from a URL, e.g. https://a.comhttps://a.com/x -> https://a.com/x. Only when the 2nd scheme precedes the first path slash (so ?url=https://… is safe).';

-- (1) one-time cleanup
update transceivers set
  details_source_url = fn_strip_double_url_prefix(details_source_url),
  product_page_url   = fn_strip_double_url_prefix(product_page_url),
  image_url          = fn_strip_double_url_prefix(image_url)
where details_source_url ~ '^https?://[^/]*https?://'
   or product_page_url  ~ '^https?://[^/]*https?://'
   or image_url         ~ '^https?://[^/]*https?://';

-- (2) ingest guard
create or replace function transceiver_url_normalize() returns trigger
language plpgsql as $$
begin
  NEW.details_source_url := fn_strip_double_url_prefix(NEW.details_source_url);
  NEW.product_page_url   := fn_strip_double_url_prefix(NEW.product_page_url);
  NEW.image_url          := fn_strip_double_url_prefix(NEW.image_url);
  NEW.datasheet_url      := fn_strip_double_url_prefix(NEW.datasheet_url);
  return NEW;
end $$;

drop trigger if exists trg_transceiver_url_normalize on transceivers;
create trigger trg_transceiver_url_normalize
  before insert or update on transceivers
  for each row execute function transceiver_url_normalize();

-- (3) enrichment worklist — pipeline input
create or replace view v_datasheet_enrichment_worklist as
select 'switch'::text as entity, s.id as entity_id, v.name as vendor, s.model as item,
       'ports_config+max_speed'::text as need,
       coalesce(s.datasheet_url, s.product_page_url) as source_url,
       case when s.datasheet_url is not null then 'datasheet'
            when s.product_page_url is not null then 'product_page'
            else 'none' end as source_type
from switches s join vendors v on v.id = s.vendor_id
where (s.ports_config is null or s.ports_config = '{}'::jsonb) and s.max_speed_gbps is null
union all
select 'transceiver', t.id, v.name, t.part_number,
       'form_factor+speed',
       coalesce(t.details_source_url, t.datasheet_url, t.product_page_url),
       case when t.datasheet_url is not null then 'datasheet'
            when t.details_source_url is not null then 'details_page'
            when t.product_page_url is not null then 'product_page'
            else 'none' end
from transceivers t join vendors v on v.id = t.vendor_id
where t.speed_gbps is null or t.speed_gbps <= 0;

comment on view v_datasheet_enrichment_worklist is
  'Input for the datasheet-enrichment pipeline: entities whose ports_config/max_speed (switch) or form_factor/speed (transceiver) must be filled from source, with the best available source URL and its type. source_type=none means URL discovery is needed first.';

insert into _migrations (filename, applied_at)
select '128-url-normalize-and-enrichment-worklist.sql', now()
where not exists (select 1 from _migrations where filename = '128-url-normalize-and-enrichment-worklist.sql');