1import { createHash } from 'node:crypto';
2import he from 'he';
3import { convert } from 'html-to-text';
4import type { Board, Job } from './types.js';
5export const hash = (s: string) => createHash('sha256').update(s).digest('hex');
6export const normalizedText = (s: string) => s.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase().replace(/ı/g, 'i').replace(/[^\p{L}\p{N}+#]+/gu, ' ').trim();
7export function plainText(value: unknown): string {
8 if (typeof value !== 'string') return '';
9
10 const html = /<\/?(?:p|div|h\d|ul|li|strong|span)\b/i.test(value) ? he.decode(value) : value;
11 return convert(html, { wordwrap: false, selectors: [{ selector: 'a', options: { ignoreHref: true } }, { selector: 'img', format: 'skip' }] }).trim();
12}
13export function isoDate(value: unknown): string | null {
14 if (typeof value !== 'string' && typeof value !== 'number') return null;
15 if (typeof value === 'string' && !/^\d{4}-\d{2}-\d{2}(?:T|$)/.test(value)) return null;
16 const date = new Date(value);
17 return Number.isFinite(date.getTime()) ? date.toISOString() : null;
18}
19export function canonicalUrl(value: string): string {
20 const u = new URL(value);
21 if (!['https:', 'http:'].includes(u.protocol) || u.username || u.password) throw new Error('Invalid application URL');
22 u.protocol = 'https:';
23 u.hash = '';
24 for (const key of [...u.searchParams.keys()]) if (/^(utm_|gh_src$|lever-source$|lever-origin$|source$|ref$|referrer$|fbclid$|gclid$)/i.test(key)) u.searchParams.delete(key);
25 u.searchParams.sort();
26 u.pathname = u.pathname.replace(/\/(application|apply)\/?$/, '').replace(/\/$/, '') || '/';
27 return u.toString();
28}
29const numberOrNull = (v: unknown): number | null => typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : null;
30function workplace(raw: unknown, locations: string[]): Pick<Job, 'remote' | 'workplaceType'> {
31 const explicit = normalizedText(String(raw ?? ''));
32 if (explicit === 'remote' || raw === true) return { remote: true, workplaceType: 'remote' };
33 if (explicit === 'hybrid') return { remote: false, workplaceType: 'hybrid' };
34 if (['onsite', 'on site'].includes(explicit)) return { remote: false, workplaceType: 'onsite' };
35 if (raw === false) return { remote: false, workplaceType: 'unknown' };
36 const text = normalizedText(locations.join(' '));
37 if (/\b(no remote|not remote|non remote|on site|onsite)\b/.test(text)) return { remote: false, workplaceType: 'onsite' };
38 if (/\bhybrid\b/.test(text)) return { remote: false, workplaceType: 'hybrid' };
39 if (/\b(remote|fully remote|work from home)\b/.test(text)) return { remote: true, workplaceType: 'remote' };
40 return { remote: null, workplaceType: 'unknown' };
41}
42function employment(value: unknown): string | null {
43 if (typeof value !== 'string' || !value.trim()) return null;
44 const v = normalizedText(value).replace(/ /g, '');
45 return ({ fulltime: 'Full-time', parttime: 'Part-time', contract: 'Contract', contractor: 'Contract', intern: 'Internship', internship: 'Internship', temporary: 'Temporary' } as Record<string, string>)[v] ?? value;
46}
47function salary(v: any): Pick<Job, 'salaryMin' | 'salaryMax' | 'currency' | 'salaryInterval'> {
48 let min = numberOrNull(v?.min ?? v?.minValue), max = numberOrNull(v?.max ?? v?.maxValue);
49 if (min !== null && max !== null && min > max) { min = null; max = null; }
50 const currency = v?.currency ?? v?.currencyCode;
51 const interval = normalizedText(String(v?.interval ?? ''));
52 return { salaryMin: min, salaryMax: max, currency: typeof currency === 'string' && /^[A-Z]{3}$/i.test(currency) ? currency.toUpperCase() : null,
53 salaryInterval: ({ year: 'year', yearly: 'year', annual: 'year', '1 year': 'year', month: 'month', monthly: 'month', '1 month': 'month', hour: 'hour', hourly: 'hour', '1 hour': 'hour', week: 'week', weekly: 'week' } as Record<string, string>)[interval] ?? (interval || null) };
54}
55const strings = (values: unknown[]): string[] => [...new Set(values.filter((v): v is string => typeof v === 'string' && !!v.trim()).map(s => s.trim()))];
56const countryFromAddress = (address: any) => address?.postalAddress?.addressCountry ?? address?.addressCountry;
57export function normalizeJob(board: Board, raw: any, now: string, enrich: boolean): Job | null {
58 let title: unknown, id: unknown, url: unknown, description = '', locations: string[] = [], countries: string[] = [];
59 let posted: unknown, updated: unknown, postedAtKind: Job['postedAtKind'] = 'unknown', remote: unknown, contract: unknown, compensation: any, requisition: unknown;
60 let company = board.company;
61 if (board.source === 'greenhouse') {
62 title = raw.title; id = raw.id; url = raw.absolute_url; description = plainText(raw.content);
63 locations = strings([raw.location?.name]); company = raw.company_name || company;
64 posted = raw.first_published; postedAtKind = 'published'; updated = raw.updated_at;
65 const metadata = Array.isArray(raw.metadata) ? raw.metadata : [];
66 contract = metadata.find((m: any) => /employment.?type/i.test(m.name))?.value;
67 remote = metadata.find((m: any) => /^(workplace type|workplaceType)$/i.test(m.name))?.value;
68 requisition = raw.requisition_id;
69 } else if (board.source === 'lever') {
70 title = raw.text; id = raw.id; url = raw.applyUrl ?? raw.hostedUrl;
71 description = [plainText(raw.description ?? raw.descriptionPlain), ...(raw.lists ?? []).map((l: any) => `${plainText(l.text)}\n${plainText(l.content)}`), plainText(raw.additional ?? raw.additionalPlain)].filter(Boolean).join('\n\n');
72 locations = strings([raw.categories?.location, ...(raw.categories?.allLocations ?? [])]); countries = strings([raw.country]);
73 posted = raw.createdAt; postedAtKind = 'created'; remote = raw.workplaceType; contract = raw.categories?.commitment; compensation = raw.salaryRange;
74 } else if (board.source === 'ashby') {
75 if (raw.isListed === false) return null;
76 title = raw.title; url = raw.applyUrl ?? raw.jobUrl; id = raw.id ?? raw.jobUrl?.split('/').filter(Boolean).at(-1);
77 description = raw.descriptionPlain || plainText(raw.descriptionHtml);
78 locations = strings([raw.location, ...(raw.secondaryLocations ?? []).map((l: any) => l.location)]);
79 countries = strings([countryFromAddress(raw.address), ...(raw.secondaryLocations ?? []).map((l: any) => countryFromAddress(l.address))]);
80 posted = raw.publishedAt; postedAtKind = 'lastPublished'; remote = raw.workplaceType ?? raw.isRemote; contract = raw.employmentType;
81 if (raw.shouldDisplayCompensationOnJobPostings !== false) compensation = raw.compensation?.summaryComponents?.find((c: any) => c.compensationType === 'Salary');
82 } else {
83 if (raw.canApply === false || raw.posted === false) return null;
84 title = raw.title; id = raw.jobReqId ?? raw.id; url = raw.externalUrl;
85 description = plainText(raw.jobDescription); requisition = raw.jobReqId;
86 locations = strings([raw.location, ...(raw.additionalLocations ?? []).map((l: any) => typeof l === 'string' ? l : l.descriptor)]);
87 countries = strings([raw.country?.descriptor, raw.jobRequisitionLocation?.country?.alpha2Code]);
88 posted = raw.startDate; postedAtKind = 'published'; remote = raw.remoteType?.descriptor ?? raw.remoteType; contract = raw.timeType;
89 }
90 if (typeof title !== 'string' || !title.trim() || id === undefined || id === null || typeof url !== 'string') return null;
91 try { canonicalUrl(url); } catch { return null; }
92 const postedAt = isoDate(posted);
93 const job: Job = {
94 title: title.trim(), company: String(company), location: locations.join(' | '), locations, countries,
95 ...workplace(remote, locations), employmentType: employment(contract), ...salary(compensation), description,
96 skills: [], experienceLevel: null, postedAt, postedAtKind: postedAt ? postedAtKind : 'unknown', updatedAt: isoDate(updated),
97 source: board.source, sourceBoard: board.url, applyUrl: url, jobId: String(id),
98 requisitionId: typeof requisition === 'string' ? requisition : null,
99 firstSeenAt: now, isNew: true, uniqueId: '', sources: [{ source: board.source, board: board.url, jobId: String(id), applyUrl: url }],
100 };
101 job.uniqueId = hash(`${board.source}|${board.url}|${id}`);
102 if (enrich) enrichJob(job);
103 return job;
104}
105function enrichJob(job: Job): void {
106 const text = `${job.title}\n${job.description}`;
107 const skills = ['Python', 'JavaScript', 'TypeScript', 'Java', 'C++', 'C#', 'SQL', 'PostgreSQL', 'React', 'Node.js', 'Docker', 'Kubernetes', 'AWS', 'Azure', 'GCP', 'OpenAI', 'n8n', 'Selenium', 'Playwright', 'Cypress', 'Terraform', 'LangChain'];
108 job.skills = skills.filter(skill => new RegExp(`(?<![\\w])${skill.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w])`, 'i').test(text));
109 const title = normalizedText(job.title);
110 if (/\b(intern|internship)\b/.test(title)) job.experienceLevel = 'Intern';
111 else if (/\b(junior|jr|entry level|graduate)\b/.test(title)) job.experienceLevel = 'Junior';
112 else if (/\b(principal|staff|lead)\b/.test(title)) job.experienceLevel = 'Lead';
113 else if (/\b(senior|sr)\b/.test(title)) job.experienceLevel = 'Senior';
114 else if (/\b(mid|intermediate)\b/.test(title)) job.experienceLevel = 'Mid';
115}
