transceiver-db/packages/scraper/src/utils/quarantine-non-transceivers.ts
2026-05-10 09:41:59 +02:00

171 lines
9.7 KiB
TypeScript

/**
* Non-Transceiver Quarantine
*
* Moves obvious accessories, switches, instruments, radio products and optical
* transport systems out of the active transceiver verification base.
*/
import { pool, recordVerificationEvidence } from "./db";
import { logger } from "./logger";
async function quarantine(): Promise<void> {
logger.info("=== Non-Transceiver Quarantine ===");
const result = await pool.query(`
UPDATE transceivers t
SET category = 'NonTransceiver',
price_verified = false,
image_verified = false,
details_verified = false,
competitor_verified = false,
fully_verified = false,
price_verified_at = NULL,
image_verified_at = NULL,
details_verified_at = NULL,
competitor_verified_at = NULL,
fully_verified_at = NULL,
updated_at = NOW()
FROM vendors v
WHERE v.id = t.vendor_id
AND (
v.name = 'GAO Tek'
OR (v.name = 'Flexoptix' AND COALESCE(t.product_page_url, '') ~ '^https://www\\.flexoptix\\.net/en/transceiver/?\\?')
OR (
v.name = 'Ascent Optics'
AND (
t.part_number LIKE '--%'
OR t.part_number ~* '(^Distance$|^Optical-Transceivers$|^Coherent-Transceivers$|^Fiber-Channel-Transceivers$|^LPO-Transceivers$|^Liquid-Cooling-Transceivers$|^PON-Transceivers$|^XGSPON$|Transceivers$|^[A-Z0-9+-]+-DD$|^QSFP112$|^QSFP112-DD$|^QSFP-DD800$|^OSFP224$|^OSFP-XD$)'
OR COALESCE(t.product_page_url, '') ~* '/category/'
)
)
OR COALESCE(t.product_page_url, '') ~* '(/c/[a-z0-9-]+-[0-9]+|supported-vendors|universal-dac-aoc|optical-patch-cables|universal-transceiver-our-voodoo|flexoptix\\.net/(en/)?transceiver/?$|direct-attach-cables|dynamic-components|arista\\.com/en/(products|solutions)/)'
OR t.part_number ~* '(AOC|DAC|DACBL|XDAC|DIRECT[ -]?ATTACH|TWINAX|BREAKOUT)'
OR COALESCE(t.product_page_url, '') ~* '(aoc|dac|direct-attach|active-optical|twinax|breakout)'
OR (v.name = 'Flexoptix' AND COALESCE(t.product_page_url, '') ~* '/stores/store/redirect/')
OR (v.name = 'FS.COM' AND (COALESCE(t.product_page_url, '') = '' OR COALESCE(t.product_page_url, '') !~* 'fs\\.com' OR COALESCE(t.product_page_url, '') ~* 'fs\\.com/c/'))
OR (
v.name = 'GBICS'
AND (
t.part_number ~* '(Cables?|Patch Leads?|Choose By|Cart with|Vendor|Speed|Network Adapters|Media Converters|All NVIDIA|ConnectX|Cisco Systems|Cisco Meraki|Cisco Viptela|Alcatel-Lucent|Brocade|WatchGuard|HP Procurve|AOC|DAC)'
OR t.part_number ~* '(^[0-9]+G (QSFP|OSFP|SFP)|I-Temp Tranceivers|Gigabit Ethernet$)'
OR COALESCE(t.product_page_url, '') ~* '(active-optical-cables|direct-attach|patch-cables|patch-leads|choose-by|custom-transceivers|network-adapters|media-converters|cart\\.php|nvidia-2|connectx|cisco|watchguard|hp-procurve|aoc-dac|mtp-mpo-cables)'
OR COALESCE(t.product_page_url, '') ~* '(100g-qsfp28/?$|100g-qsfp28-i-temp|10gb-industrial-temp|1g-sfp-i-temp|200g-qsfp56/?$|25g-sfp28-i-temp|400g-qsfp-dd/?$|400g-qsfp112/?$|40g-qsfp-i-temp|i-temp-tranceivers|osfp-[0-9]+g-gigabit|qsfp.*gigabit|400g-qsfp112-2)'
)
)
OR (
v.name = 'T&S Communication'
AND (
COALESCE(t.product_page_url, '') !~* '/products/'
OR COALESCE(t.product_page_url, '') ~* '(\\.pdf$|/products/fiber-optic-transceivers/?$)'
OR t.part_number ~* '(^Transceiver$|Product-Brochure|^[0-9]+G(/[0-9]+G(/[0-9]+G)?)?-Transceiver$)'
)
)
OR (
v.name = 'ShopFiber24'
AND (
t.part_number ~* '^(FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-.*|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver.*|.*DAC.*&.*AOC.*Kabel|.*DAC.*AOC.*Kabel)$'
OR t.part_number ~* '^FOCP-'
OR COALESCE(t.product_page_url, '') ~* '/(FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver|FOCP-|.*DAC.*AOC.*Kabel)/'
)
)
OR (v.name = 'Vcelink' AND t.part_number ~ '^[0-9]+$' AND COALESCE(t.product_page_url, '') = '')
OR t.part_number ~* '^(N/A|Change|wurde|Distance|FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver.*|.*DAC.*&.*AOC.*Kabel|.*DAC.*AOC.*Kabel)$'
OR t.category IN (
'Accessory',
'Adapter / Converter',
'Switch / Media Converter',
'Switch / Network Infrastructure',
'NIC / Adapter',
'Mux / Passive Optical',
'Product Family',
'Loopback / Test Module'
)
)
AND (
t.part_number ~* '(switch|walkie|radio|tester|meter|microscope|amplifier|oadm|demux|demultiplexer|multiplexer|mux|platform|transponder|transport[ -]system|solution|converter|adapter|cleaning|cassette|attenuator|telephone|sensor)'
OR COALESCE(t.product_page_url, '') ~* '(switch|walkie|radio|tester|meter|microscope|amplifier|oadm|demux|demultiplexer|multiplexer|mux|platform|transponder|transport-system|solution|converter|adapter|cleaning|cassette|attenuator|telephone|sensor)'
OR (v.name = 'Flexoptix' AND COALESCE(t.product_page_url, '') ~ '^https://www\\.flexoptix\\.net/en/transceiver/?\\?')
OR (
v.name = 'Ascent Optics'
AND (
t.part_number LIKE '--%'
OR t.part_number ~* '(^Distance$|^Optical-Transceivers$|^Coherent-Transceivers$|^Fiber-Channel-Transceivers$|^LPO-Transceivers$|^Liquid-Cooling-Transceivers$|^PON-Transceivers$|^XGSPON$|Transceivers$|^[A-Z0-9+-]+-DD$|^QSFP112$|^QSFP112-DD$|^QSFP-DD800$|^OSFP224$|^OSFP-XD$)'
OR COALESCE(t.product_page_url, '') ~* '/category/'
)
)
OR COALESCE(t.product_page_url, '') ~* '(/c/[a-z0-9-]+-[0-9]+|supported-vendors|universal-dac-aoc|optical-patch-cables|universal-transceiver-our-voodoo|flexoptix\\.net/(en/)?transceiver/?$|direct-attach-cables|dynamic-components|arista\\.com/en/(products|solutions)/)'
OR t.part_number ~* '(AOC|DAC|DACBL|XDAC|DIRECT[ -]?ATTACH|TWINAX|BREAKOUT)'
OR COALESCE(t.product_page_url, '') ~* '(aoc|dac|direct-attach|active-optical|twinax|breakout)'
OR (v.name = 'Flexoptix' AND COALESCE(t.product_page_url, '') ~* '/stores/store/redirect/')
OR (v.name = 'FS.COM' AND (COALESCE(t.product_page_url, '') = '' OR COALESCE(t.product_page_url, '') !~* 'fs\\.com' OR COALESCE(t.product_page_url, '') ~* 'fs\\.com/c/'))
OR (
v.name = 'GBICS'
AND (
t.part_number ~* '(Cables?|Patch Leads?|Choose By|Cart with|Vendor|Speed|Network Adapters|Media Converters|All NVIDIA|ConnectX|Cisco Systems|Cisco Meraki|Cisco Viptela|Alcatel-Lucent|Brocade|WatchGuard|HP Procurve|AOC|DAC)'
OR t.part_number ~* '(^[0-9]+G (QSFP|OSFP|SFP)|I-Temp Tranceivers|Gigabit Ethernet$)'
OR COALESCE(t.product_page_url, '') ~* '(active-optical-cables|direct-attach|patch-cables|patch-leads|choose-by|custom-transceivers|network-adapters|media-converters|cart\\.php|nvidia-2|connectx|cisco|watchguard|hp-procurve|aoc-dac|mtp-mpo-cables)'
OR COALESCE(t.product_page_url, '') ~* '(100g-qsfp28/?$|100g-qsfp28-i-temp|10gb-industrial-temp|1g-sfp-i-temp|200g-qsfp56/?$|25g-sfp28-i-temp|400g-qsfp-dd/?$|400g-qsfp112/?$|40g-qsfp-i-temp|i-temp-tranceivers|osfp-[0-9]+g-gigabit|qsfp.*gigabit|400g-qsfp112-2)'
)
)
OR (
v.name = 'T&S Communication'
AND (
COALESCE(t.product_page_url, '') !~* '/products/'
OR COALESCE(t.product_page_url, '') ~* '(\\.pdf$|/products/fiber-optic-transceivers/?$)'
OR t.part_number ~* '(^Transceiver$|Product-Brochure|^[0-9]+G(/[0-9]+G(/[0-9]+G)?)?-Transceiver$)'
)
)
OR (
v.name = 'ShopFiber24'
AND (
t.part_number ~* '^(FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-.*|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver.*|.*DAC.*&.*AOC.*Kabel|.*DAC.*AOC.*Kabel)$'
OR t.part_number ~* '^FOCP-'
OR COALESCE(t.product_page_url, '') ~* '/(FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver|FOCP-|.*DAC.*AOC.*Kabel)/'
)
)
OR (v.name = 'Vcelink' AND t.part_number ~ '^[0-9]+$' AND COALESCE(t.product_page_url, '') = '')
OR t.part_number ~* '^(N/A|Change|wurde|Distance|FO-TRANSCEIVER|Kompatible-Transceiver|Transceivermodule|Zu-den-|QSFP\\+-MSA|SFP\\+-MSA|Marken-Transceiver.*|.*DAC.*&.*AOC.*Kabel|.*DAC.*AOC.*Kabel)$'
OR t.category IN (
'Accessory',
'Adapter / Converter',
'Switch / Media Converter',
'Switch / Network Infrastructure',
'NIC / Adapter',
'Mux / Passive Optical',
'Product Family',
'Loopback / Test Module'
)
)
AND COALESCE(t.category, '') != 'NonTransceiver'
RETURNING t.id, t.part_number, v.name AS vendor_name, t.product_page_url
`);
for (const row of result.rows) {
await recordVerificationEvidence({
transceiverId: row.id,
verificationType: "artifact_quarantine",
sourceUrl: row.product_page_url || undefined,
evidenceValue: {
partNumber: row.part_number,
vendor: row.vendor_name,
reason: "matched deterministic non-transceiver artifact rule",
},
robotName: "verify:quarantine:non-transceivers",
confidence: 1,
});
}
logger.info("Non-transceiver quarantine complete", {
quarantined: result.rowCount ?? 0,
});
}
if (require.main === module) {
quarantine()
.then(() => pool.end())
.catch((err) => {
logger.error("Non-transceiver quarantine failed", { error: (err as Error).message });
pool.end();
process.exit(1);
});
}