1
2export const STOP = new Set(['a', 'an', 'the', 'of', 'for', 'and', 'or', 'with', 'without', 'in', 'on', 'to', 'by', 'from', 'other', 'than', 'its', 'thereof', 'kind', 'kinds', 'used', 'type', 'nesoi', 'not', 'n', 'e', 's', 'i', 'whether', 'like', 'my', 'our', 'product', 'products', 'item', 'items', 'hts', 'code', 'tariff', 'duty', 'rate', 'what', 'is', 'are']);
3
4export const digitsOf = (s) => String(s ?? '').replace(/\D/g, '');
5
6
7export function formatHts(d) {
8 d = digitsOf(d);
9 if (d.length <= 4) return d;
10 const parts = [d.slice(0, 4), d.slice(4, 6), d.slice(6, 8), d.slice(8, 10)].filter(Boolean);
11 return parts.join('.');
12}
13
14
15export function parseQuery(raw) {
16 const q = String(raw ?? '').trim().replace(/\s+/g, ' ');
17 if (!q) return { kind: 'empty' };
18 const m = q.match(/^(?:hts|htsus|hs|heading|subheading|code)?\s*:?\s*([0-9][0-9 .\-]*[0-9]|[0-9])$/i);
19 if (m) {
20 const d = digitsOf(m[1]);
21 if (d.length < 2 || d.length > 10) return { kind: 'invalid', q, error: `"${q}" is not an HTS code: use 2 to 10 digits, e.g. 8507.60.00.20 or 6109.10.` };
22 return { kind: 'code', q, digits: d };
23 }
24 if (q.length > 300) return { kind: 'invalid', q: q.slice(0, 100), error: 'Product descriptions must be 300 characters or less.' };
25 const toks = queryTokens(q);
26 if (!toks.length) return { kind: 'invalid', q, error: `"${q}" has no searchable words. Describe the product (e.g. "cotton t-shirts") or give an HTS code.` };
27 return { kind: 'text', q, tokens: toks };
28}
29
30const LEMMA = { electrical: 'electric', electrically: 'electric', jewellery: 'jewelry', aluminium: 'aluminum', colour: 'color', tshirt: 'tshirt', teeshirt: 'tshirt' };
31
32export const SYN = {
33 bicycle: ['cycle'], bike: ['cycle'], ebike: ['cycle'], scooter: ['scooter', 'cycle'], solar: ['photovoltaic'], phone: ['telephone'], smartphone: ['telephone'], cellphone: ['telephone'],
34 laptop: ['portable'], notebook: ['portable'], computer: ['computer', 'processing'], tv: ['television'], shoe: ['footwear'], sneaker: ['footwear'], boot: ['footwear'],
35 car: ['vehicle'], truck: ['vehicle'], sofa: ['seat'], couch: ['seat'], chair: ['seat'], headphone: ['earphone'], earbud: ['earphone', 'headphone'],
36 cable: ['conductor', 'wire'], charger: ['converter'], drone: ['unmanned'], led: ['diode'], mug: ['cup', 'tableware'], cup: ['tableware'], sunglass: ['spectacle'],
37 supplement: ['preparation'], vitamin: ['provitamin'], candy: ['confectionery', 'sugar'], chocolate: ['cocoa'], phonecase: ['case'], purse: ['handbag'], backpack: ['rucksack'],
38 hoodie: ['sweatshirt', 'pullover'], sweater: ['sweatshirt', 'pullover', 'jersey'], jean: ['trouser'], pant: ['trouser'], dress: ['dress'], sock: ['hosiery'], cosmetic: ['makeup', 'beauty'],
39 lipstick: ['lip'], perfume: ['perfume'], toy: ['toy'], game: ['game'], printer: ['printing'], tablet: ['portable'], speaker: ['loudspeaker'], fridge: ['refrigerator'], battery: ['battery'],
40};
41export function stem(w) {
42 if (LEMMA[w]) return LEMMA[w];
43 const b = stemBasic(w); return LEMMA[b] ?? b;
44}
45function stemBasic(w) {
46 if (w.length <= 3) return w;
47 if (/ies$/.test(w)) return `${w.slice(0, -3)}y`;
48 if (/(ss|us|is)$/.test(w)) return w;
49 if (/(sses|xes|ches|shes|zes)$/.test(w)) return w.slice(0, -2);
50 if (/toes$/.test(w)) return w.slice(0, -2);
51 if (/s$/.test(w)) return w.slice(0, -1);
52 return w;
53}
54
55const clean = (s) => String(s ?? '').toLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g, '').replace(/<[^>]+>/g, ' ');
56
57
58
59export function negTokens(s) {
60 const c = clean(s).replace(/\bwhether or not\b/g, ' '); const out = new Set();
61 for (const m of c.matchAll(/\b(?:not|other than|excluding|except)\s+(?:of\s+|a\s+|an\s+)?([a-z0-9-]+)/g)) out.add(stem(m[1].replace(/-/g, '')));
62 return out;
63}
64export function textTokens(s) {
65
66 const c = clean(s).replace(/\bwhether or not\b/g, ' ').replace(/\b(not|other than|excluding|except)\s+(?:of\s+|a\s+|an\s+)?[a-z0-9-]+/g, ' ');
67 const out = new Set();
68 for (const w of c.split(/[^a-z0-9]+/)) if (w && !STOP.has(w)) out.add(stem(w));
69 for (const m of c.matchAll(/[a-z0-9]+(?:-[a-z0-9]+)+/g)) out.add(stem(m[0].replace(/-/g, '')));
70 return out;
71}
72
73const PHRASES = [[/\be-?bikes?\b/g, 'electric cycle'], [/\b(sneakers?|trainers|running shoes?)\b/g, 'sports footwear'], [/\b(cell ?phones?|mobile phones?)\b/g, 'smartphone'],
74 [/\be-?scooters?\b/g, 'electric scooter'], [/\bflip[- ]?flops?\b/g, 'footwear thong'], [/\bhoodies?\b/g, 'sweatshirt'], [/\bpower ?banks?\b/g, 'lithium-ion battery'], [/\bvapes?\b/g, 'vaporizing'],
75 [/\b(laptops?|notebook computers?)\b/g, 'portable automatic data processing machines'], [/\b(?:wireless |bluetooth )?(earbuds?|headphones?|earphones?)\b/g, 'headphones earphones']];
76export function queryTokens(q) {
77 let c0 = clean(q); for (const [rx, rep] of PHRASES) c0 = c0.replace(rx, rep);
78 const c = c0.replace(/([a-z0-9])-([a-z0-9])/g, '$1$2');
79 const words = [...new Set(c.split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w) && !/^\d{1,2}$/.test(w)).map(stem))].slice(0, 12);
80 const raw = [...new Set(c.split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w) && !/^\d{1,2}$/.test(w)))].slice(0, 12);
81 return words.map((w, i) => {
82 const orig = raw[i] ? stemBasic(raw[i]) : w;
83 return [w, orig, ...new Set([...(SYN[w] ?? []), ...(SYN[orig] ?? [])].filter((x) => x !== w && x !== orig))];
84 });
85}
86
87const CH99 = /\b(99\d{2})\.(\d{2})\.(\d{2})(?:\.(\d{2}))?(?:\s*(?:-|–|through|to)\s*(99\d{2})\.(\d{2})\.(\d{2}))?/g;
88
89export function chapter99Refs(row) {
90 const texts = [[row.general, ['general']], [row.special, ['special']], [row.other, ['other']], ...(row.footnotes ?? []).map((f) => [f?.value, Array.isArray(f?.columns) ? f.columns : []])].filter(([t]) => t);
91 const out = []; const add = (code, cols) => { const ex = out.find((r) => r.code === code); if (ex) { for (const c of cols) if (!ex.columns.includes(c)) ex.columns.push(c); } else out.push({ code, columns: [...cols] }); };
92 for (const [t, cols] of texts) for (const m of String(t).matchAll(CH99)) {
93 add(`${m[1]}.${m[2]}.${m[3]}`, cols);
94 if (m[5]) add(`${m[5]}.${m[6]}.${m[7]}`, cols);
95 }
96 return out;
97}
98
99
100export function specialPrograms(special) {
101 const out = [];
102 for (const m of String(special ?? '').matchAll(/\(([^)]+)\)/g)) for (const p of m[1].split(',')) { const v = p.trim().replace(/\*$/, ''); if (/^[A-Z+*]{1,4}$/.test(v) && !out.includes(v)) out.push(v); }
103 return out;
104}
105
106const txt = (s) => String(s ?? '').replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
107
108
109export function buildIndex(rows) {
110 const items = []; const byCode = new Map(); const stack = [];
111 for (const r of rows) {
112 const indent = Number(r.indent) || 0;
113 const code = String(r.htsno ?? '').trim();
114 const d = digitsOf(code);
115 while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop();
116 if (indent === 0 && d.length === 4) stack.length = 0;
117 const parent = stack[stack.length - 1] ?? null;
118 const it = {
119 i: items.length, code, digits: d, indent, desc: txt(r.description), parent,
120 general: txt(r.general), special: txt(r.special), other: txt(r.other),
121 units: Array.isArray(r.units) ? r.units.filter(Boolean) : [], footnotes: (r.footnotes ?? []).map((f) => txt(f?.value)).filter(Boolean),
122 quota: txt(r.quotaQuantity) || null, refs: chapter99Refs(r), children: 0,
123 };
124 it.path = parent ? [...parent.path, it.desc] : [it.desc];
125 it.ancestorCode = d ? it : (parent?.ancestorCode ?? null);
126 it.own = textTokens(it.desc);
127 it.forCtx = new Set((clean(it.desc).match(/\b(?:for|used (?:in|with|on|for)|suitable for)\b[^;:]*/g) ?? []).flatMap((seg) => [...textTokens(seg.replace(/^\S+(?: \S+)?/, ''))]));
128 it.nonFor = new Set([...[...it.own].filter((t) => !it.forCtx.has(t)), ...(parent?.nonFor ?? [])]);
129 it.neg = new Set([...negTokens(it.desc), ...(parent?.neg ?? [])]);
130 it.all = new Set([...it.own, ...(parent?.all ?? [])].filter((t) => !it.neg.has(t)));
131
132 it.sub = d.length === 4 ? new Set() : new Set([...it.own, ...(parent?.sub ?? [])].filter((t) => !it.neg.has(t)));
133 if (parent) parent.children++;
134 items.push(it);
135 if (d && !byCode.has(d)) byCode.set(d, it);
136 stack.push(it);
137 }
138 const df = new Map();
139 for (const it of items) for (const t of it.own) df.set(t, (df.get(t) ?? 0) + 1);
140 for (const it of items) {
141 let h = it; while (h && h.digits.length !== 4) h = h.parent; it.head = h ?? null;
142 let src = it; while (src && !src.general && !src.special && !src.other) src = src.parent;
143 it.rateFrom = src ?? null;
144
145 let p = it.parent; const inh = [...it.refs]; while (p) { for (const r of p.refs) if (!inh.some((x) => x.code === r.code)) inh.push(r); p = p.parent; }
146 it.allRefs = inh;
147 }
148 return { items, byCode, df, n: items.length };
149}
150
151export function levelOf(d) { return d.length >= 10 ? 'statistical' : d.length === 8 ? 'tariff_line' : d.length === 6 ? 'subheading' : d.length === 4 ? 'heading' : d.length ? 'chapter' : 'header'; }
152
153
154export function lookupCode(index, digits, { includeChildren = true, max = 50 } = {}) {
155 const exact = index.byCode.get(digits) ?? null;
156 const out = [];
157 if (exact) out.push(exact);
158 if (includeChildren || !exact) {
159 for (const it of index.items) {
160 if (out.length >= max) break;
161 if (it === exact || !it.digits || !it.digits.startsWith(digits)) continue;
162 out.push(it);
163 }
164 }
165 return { exact, items: out.slice(0, max) };
166}
167
168
169
170function strength(g, set) { if (set.has(g[0]) || (g[1] && set.has(g[1]))) return 1; for (let i = 2; i < g.length; i++) if (set.has(g[i])) return 0.85; return 0; }
171
172export function searchText(index, tokens, { max = 10, includeChapter99 = false, statisticalOnly = false, relaxed = false } = {}) {
173 const scored = [];
174 const idf = (t) => Math.log(1 + (index.n ?? 1) / (1 + (index.df?.get(t) ?? 0)));
175 const weights = tokens.map((g) => Math.max(...g.map(idf)));
176 const wSum = weights.reduce((a, b) => a + b, 0) || 1;
177 const wantsParts = tokens.some((g) => g.some((t) => /^(part|accessory|accessorie|component|spare)$/.test(t)));
178 for (const it of index.items) {
179 if (!it.digits) continue;
180 const ch = it.digits.slice(0, 2);
181 if (!includeChapter99 && (ch === '98' || ch === '99')) continue;
182 if (statisticalOnly && it.digits.length < 10 && it.children) continue;
183 let hit = 0; let own = 0; let head = 0; let wHit = 0;
184 for (let k = 0; k < tokens.length; k++) {
185 const g = tokens[k];
186 const m = strength(g, it.all);
187 if (m) { hit++; wHit += weights[k] * m; }
188 own += strength(g, it.sub);
189 if (it.head) head += strength(g, it.head.own);
190 }
191 if (!hit) continue;
192 const cov = wHit / wSum;
193 if (tokens.length === 1 ? cov < 0.8 : (hit / tokens.length < (relaxed ? 0.34 : 0.5) || cov < (relaxed ? 0.3 : 0.5))) continue;
194 const leaf = it.children === 0 ? 1 : 0;
195 const rated = it.rateFrom ? 1 : 0;
196
197 const hn = tokens[tokens.length - 1];
198 const hnIn = strength(hn, it.all) > 0;
199 const hnOnlyFor = hnIn && !(strength(hn, it.nonFor) > 0);
200 const headNoun = tokens.length > 1 ? (hnOnlyFor ? -14 : hnIn ? 8 : -6) : 0;
201 const partsPenalty = !wantsParts && it.path.slice(1).some((p) => /\b(parts?|accessories)\b/i.test(p)) ? 12 : 0;
202 const score = Math.round((-partsPenalty + headNoun + cov * 60 + (head / tokens.length) * 15 + (own / tokens.length) * 10 + leaf * 8 + rated * 4 + (it.digits.length >= 10 ? 3 : 0)) * 10) / 10;
203 scored.push({ it, score, cov, own, leaf });
204 }
205 scored.sort((a, b) => b.score - a.score || b.it.digits.length - a.it.digits.length || a.it.i - b.it.i);
206
207 const per = new Map(); const perH = new Map(); const out = [];
208 for (const s of scored) {
209 const k = s.it.digits.slice(0, 6); const n = per.get(k) ?? 0;
210 const h = s.it.digits.slice(0, 4); const nh = perH.get(h) ?? 0;
211 if (n >= 3 || nh >= 5) continue; per.set(k, n + 1); perH.set(h, nh + 1); out.push(s);
212 if (out.length >= max) break;
213 }
214 if (!out.length && !relaxed && tokens.length > 1) return searchText(index, tokens, { max, includeChapter99, statisticalOnly, relaxed: true }).map((x) => ({ ...x, relaxed: true }));
215 return out;
216}
217
218
219export function closestCode(index, digits) {
220 for (let n = digits.length - 1; n >= 2; n--) { const it = index.byCode.get(digits.slice(0, n)); if (it) return it; }
221 return null;
222}
223
224
225export function toRow(it, index, { release, resolveRefs = true } = {}) {
226 const rf = it.rateFrom;
227 const refs = it.allRefs.slice(0, 12).map(({ code, columns }) => {
228 const r = index.byCode.get(digitsOf(code));
229 const appliesTo = columns.includes('other') && columns.length === 1 ? 'column2' : columns.includes('special') && columns.length === 1 ? 'special' : 'general';
230 return { htsNumber: code, appliesTo, description: resolveRefs && r ? r.desc.slice(0, 300) : null, rate: resolveRefs && r ? (r.general || r.rateFrom?.general || null) : null };
231 });
232 return {
233 htsNumber: it.code, htsDigits: it.digits, level: levelOf(it.digits), chapter: it.digits.slice(0, 2), heading: it.digits.slice(0, 4),
234 description: it.desc, fullDescription: it.path.filter(Boolean).join(' > '),
235 generalRate: rf?.general || null, specialRate: rf?.special || null, specialPrograms: specialPrograms(rf?.special), column2Rate: rf?.other || null,
236 rateSource: rf ? (rf === it ? 'this_line' : `inherited from ${rf.code || 'parent'}`) : 'none',
237 units: it.units, additionalDutyRefs: refs, footnotes: it.footnotes.slice(0, 10), quotaQuantity: it.quota,
238 hasChildren: it.children > 0,
239 htsRelease: release?.name ?? null, htsReleaseStart: release?.start ?? null,
240 sourceUrl: `https://hts.usitc.gov/search?query=${encodeURIComponent(it.code)}`,
241 };
242}
