rene.fichtmueller/transceiver-db
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-quality): fix malformed URLs + datasheet-enrichment worklist

Browse Source 
- fn_strip_double_url_prefix + one-time fix for 436 rows whose scraped URLs had a
  duplicated host prefix (https://host.comhttps://host.com/x) -> unfetchable.
  Ingest guard trg_transceiver_url_normalize prevents recurrence.
- v_datasheet_enrichment_worklist: pipeline input listing every switch/transceiver
  needing a field filled from source, with its best source URL + type.
This commit is contained in:
root 2026-07-06 09:26:01 +00:00
parent d9c37af642
commit 6e53cd4568
1 changed files with 75 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
sql/128-url-normalize-and-enrichment-worklist.sql Normal file
View File

@ -0,0 +1,75 @@
-- sql/128-url-normalize-and-enrichment-worklist.sql
-- Foundation for the datasheet-enrichment pipeline (Lever 3):
-- (1) Fix malformed double-prefixed URLs (a scraper concatenated the base host in
-- front of an already-absolute URL, e.g. "https://www.mouser.dehttps://www.mouser.de/…").
-- 503 rows affected (details_source_url 373, product_page_url 83, image_url 47).
-- These cannot be fetched, so they block any enrichment run.
-- (2) A guard so the ingest bug cannot re-introduce them.
-- (3) v_datasheet_enrichment_worklist — the pipeline's input: every transceiver/switch
-- that needs a field filled from source, with its best source URL + type.
-- Idempotent; safe to re-run. The regex only matches a second scheme that appears
-- BEFORE the first path slash, so legitimate URLs with an encoded URL in the query string
-- (…?url=https://…) are never touched.
create or replace function fn_strip_double_url_prefix(u text) returns text
language sql immutable as $$
select case when u ~ '^https?://[^/]*https?://'
then regexp_replace(u, '^https?://[^/]*(https?://)', '\1')
else u end
$$;
comment on function fn_strip_double_url_prefix(text) is
'Removes a duplicated leading scheme+host from a URL, e.g. https://a.comhttps://a.com/x -> https://a.com/x. Only when the 2nd scheme precedes the first path slash (so ?url=https://… is safe).';
-- (1) one-time cleanup
update transceivers set
details_source_url = fn_strip_double_url_prefix(details_source_url),
product_page_url = fn_strip_double_url_prefix(product_page_url),
image_url = fn_strip_double_url_prefix(image_url)
where details_source_url ~ '^https?://[^/]*https?://'
or product_page_url ~ '^https?://[^/]*https?://'
or image_url ~ '^https?://[^/]*https?://';
-- (2) ingest guard
create or replace function transceiver_url_normalize() returns trigger
language plpgsql as $$
begin
NEW.details_source_url := fn_strip_double_url_prefix(NEW.details_source_url);
NEW.product_page_url := fn_strip_double_url_prefix(NEW.product_page_url);
NEW.image_url := fn_strip_double_url_prefix(NEW.image_url);
NEW.datasheet_url := fn_strip_double_url_prefix(NEW.datasheet_url);
return NEW;
end $$;
drop trigger if exists trg_transceiver_url_normalize on transceivers;
create trigger trg_transceiver_url_normalize
before insert or update on transceivers
for each row execute function transceiver_url_normalize();
-- (3) enrichment worklist — pipeline input
create or replace view v_datasheet_enrichment_worklist as
select 'switch'::text as entity, s.id as entity_id, v.name as vendor, s.model as item,
'ports_config+max_speed'::text as need,
coalesce(s.datasheet_url, s.product_page_url) as source_url,
case when s.datasheet_url is not null then 'datasheet'
when s.product_page_url is not null then 'product_page'
else 'none' end as source_type
from switches s join vendors v on v.id = s.vendor_id
where (s.ports_config is null or s.ports_config = '{}'::jsonb) and s.max_speed_gbps is null
union all
select 'transceiver', t.id, v.name, t.part_number,
'form_factor+speed',
coalesce(t.details_source_url, t.datasheet_url, t.product_page_url),
case when t.datasheet_url is not null then 'datasheet'
when t.details_source_url is not null then 'details_page'
when t.product_page_url is not null then 'product_page'
else 'none' end
from transceivers t join vendors v on v.id = t.vendor_id
where t.speed_gbps is null or t.speed_gbps <= 0;
comment on view v_datasheet_enrichment_worklist is
'Input for the datasheet-enrichment pipeline: entities whose ports_config/max_speed (switch) or form_factor/speed (transceiver) must be filled from source, with the best available source URL and its type. source_type=none means URL discovery is needed first.';
insert into _migrations (filename, applied_at)
select '128-url-normalize-and-enrichment-worklist.sql', now()
where not exists (select 1 from _migrations where filename = '128-url-normalize-and-enrichment-worklist.sql');