1import { XMLParser } from 'fast-xml-parser';
2
3const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
4const UA = 'Mozilla/5.0 (compatible; ATSJobBoardScraper/1.0; +https://apify.com)';
5
6export class HttpError extends Error {
7 constructor(msg, status) { super(msg); this.status = status; }
8}
9
10export async function getJson(url, { timeoutMs = 60000, maxRetries = 3, fetchImpl = fetch, baseDelayMs = 1000 } = {}) {
11 let last;
12 for (let attempt = 0; attempt <= maxRetries; attempt++) {
13 const ctrl = new AbortController();
14 const t = setTimeout(() => ctrl.abort(), timeoutMs);
15 try {
16 const res = await fetchImpl(url, { signal: ctrl.signal, headers: { accept: 'application/json', 'user-agent': UA } });
17 if (res.status === 404) throw new HttpError('Board not found (404)', 404);
18 if (res.status === 429 || res.status >= 500) {
19 last = new HttpError(`HTTP ${res.status}`, res.status);
20 const ra = Number(res.headers?.get?.('retry-after'));
21 if (attempt < maxRetries) { await sleep(ra > 0 ? Math.min(ra * 1000, 30000) : baseDelayMs * 2 ** attempt); continue; }
22 throw last;
23 }
24 if (!res.ok) throw new HttpError(`HTTP ${res.status}`, res.status);
25 return await res.json();
26 } catch (e) {
27 if (e instanceof HttpError && (e.status < 500 && e.status !== 429)) throw e;
28 last = e.name === 'AbortError' ? new HttpError(`Timeout after ${timeoutMs / 1000}s`) : e;
29 if (attempt < maxRetries) { await sleep(baseDelayMs * 2 ** attempt); continue; }
30 } finally { clearTimeout(t); }
31 }
32 throw last;
33}
34
35export async function getText(url, opts = {}) {
36 const { fetchImpl = fetch, timeoutMs = 60000 } = opts;
37
38 return getJson(url, { ...opts, fetchImpl: async (u, init) => {
39 const res = await fetchImpl(u, init);
40 return { ok: res.ok, status: res.status, headers: res.headers, json: async () => res.text() };
41 }, timeoutMs });
42}
43
44export const ATS_LIST = ['greenhouse', 'lever', 'ashby', 'workable', 'smartrecruiters', 'recruitee', 'personio'];
45
46
47export function parseCompany(raw) {
48 const s = String(raw || '').trim();
49 if (!s) return null;
50 const m = s.match(/^(greenhouse|lever|ashby|workable|smartrecruiters|recruitee|personio)\s*:\s*([\w.-]+)$/i);
51 if (m) return { ats: m[1].toLowerCase(), slug: m[2] };
52 if (/^https?:\/\//i.test(s) || /\.(io|co|com|de)(\/|$)/i.test(s)) {
53 let u;
54 try { u = new URL(/^https?:/i.test(s) ? s : `https://${s}`); } catch { return null; }
55 const host = u.hostname.toLowerCase();
56 const seg = u.pathname.split('/').filter(Boolean);
57 if (host.endsWith('greenhouse.io')) {
58 const forParam = u.searchParams.get('for');
59 if (forParam) return { ats: 'greenhouse', slug: forParam };
60 if (host.startsWith('boards-api')) return { ats: 'greenhouse', slug: seg[2] };
61 if (seg[0] === 'embed') return null;
62 return seg[0] ? { ats: 'greenhouse', slug: seg[0] } : null;
63 }
64 if (host.endsWith('lever.co')) return seg[0] ? { ats: 'lever', slug: seg[0], region: host.includes('.eu.') ? 'eu' : undefined } : null;
65 if (host.endsWith('ashbyhq.com')) return seg[0] ? { ats: 'ashby', slug: decodeURIComponent(seg[0]) } : null;
66 if (host === 'apply.workable.com') return seg[0] && seg[0] !== 'j' ? { ats: 'workable', slug: seg[0] } : null;
67 if (host.endsWith('.workable.com') && !host.startsWith('www.')) return { ats: 'workable', slug: host.split('.')[0] };
68 if (host.endsWith('smartrecruiters.com')) {
69 if (host.startsWith('api.')) return seg[2] ? { ats: 'smartrecruiters', slug: seg[2] } : null;
70 return seg[0] ? { ats: 'smartrecruiters', slug: seg[0] } : null;
71 }
72 if (host.endsWith('.recruitee.com')) return { ats: 'recruitee', slug: host.split('.')[0] };
73 const pm = host.match(/^([\w-]+)\.jobs\.personio\.(de|com)$/);
74 if (pm) return { ats: 'personio', slug: pm[1], region: pm[2] };
75 return null;
76 }
77 if (/^[\w.-]+$/.test(s)) return { ats: null, slug: s };
78 return null;
79}
80
81export const endpoints = {
82 greenhouse: (slug) => `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(slug)}/jobs?content=true`,
83 lever: (slug, region) => `https://api.${region === 'eu' ? 'eu.' : ''}lever.co/v0/postings/${encodeURIComponent(slug)}?mode=json`,
84 ashby: (slug) => `https://api.ashbyhq.com/posting-api/job-board/${encodeURIComponent(slug)}?includeCompensation=true`,
85 workable: (slug) => `https://apply.workable.com/api/v1/widget/accounts/${encodeURIComponent(slug)}?details=true`,
86 smartrecruiters: (slug, offset = 0) => `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(slug)}/postings?limit=100&offset=${offset}`,
87 smartrecruitersDetail: (slug, id) => `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(slug)}/postings/${encodeURIComponent(id)}`,
88 recruitee: (slug) => `https://${encodeURIComponent(slug)}.recruitee.com/api/offers/`,
89 personio: (slug, tld = 'de') => `https://${encodeURIComponent(slug)}.jobs.personio.${tld}/xml`,
90};
91
92const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', '#39': "'" };
93export function decodeEntities(s) {
94 return String(s ?? '').replace(/&(#x[0-9a-f]+|#\d+|[a-z]+\d*);/gi, (m, e) => {
95 if (e[0] === '#') { const n = e[1].toLowerCase() === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); return Number.isFinite(n) ? String.fromCodePoint(n) : m; }
96 return ENTITIES[e.toLowerCase()] ?? m;
97 });
98}
99export function htmlToText(html) {
100 if (!html) return null;
101 const t = decodeEntities(String(html)
102 .replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, '')
103 .replace(/<br\s*\/?>/gi, '\n')
104 .replace(/<li[^>]*>/gi, '\n• ')
105 .replace(/<\/(p|div|h[1-6]|ul|ol|tr|section)>/gi, '\n')
106 .replace(/<[^>]+>/g, ''))
107 .replace(/[ \t\u00a0]+/g, ' ')
108 .replace(/ *\n */g, '\n')
109 .replace(/\n{3,}/g, '\n\n')
110 .trim();
111 return t || null;
112}
113const iso = (v) => {
114 if (v === null || v === undefined || v === '') return null;
115 const d = new Date(typeof v === 'number' ? v : String(v));
116 return Number.isNaN(d.getTime()) ? null : d.toISOString();
117};
118const clean = (s) => (s == null ? null : String(s).replace(/\s+/g, ' ').trim() || null);
119const normWorkplace = (w) => {
120 const s = String(w || '').toLowerCase().replace(/[-_\s]/g, '');
121 if (!s) return null;
122 if (s.includes('remote')) return 'remote';
123 if (s.includes('hybrid')) return 'hybrid';
124 if (s.includes('onsite') || s.includes('inoffice') || s.includes('office')) return 'onsite';
125 return s;
126};
127const INTERVALS = { year: 'year', month: 'month', week: 'week', day: 'day', hour: 'hour', '1 YEAR': 'year', '1 MONTH': 'month', '1 WEEK': 'week', '1 DAY': 'day', '1 HOUR': 'hour', 'per-year-salary': 'year', 'per-month-salary': 'month', 'per-week-salary': 'week', 'per-day-salary': 'day', 'per-hour-wage': 'hour' };
128
129export function normalizeGreenhouse(j, slug, { html = false } = {}) {
130 const meta = Object.fromEntries((j.metadata || []).filter((m) => m?.name).map((m) => [m.name.toLowerCase(), m.value]));
131 const workplace = normWorkplace(meta['workplace type'] || meta['location type'] || meta['remote'] || (/remote/i.test(j.location?.name || '') ? 'remote' : ''));
132 const contentHtml = j.content ? decodeEntities(j.content) : null;
133 const offices = (j.offices || []).map((o) => clean(o.location || o.name)).filter(Boolean);
134 return {
135 ats: 'greenhouse', company: slug, companyName: clean(j.company_name),
136 jobId: String(j.id), title: clean(j.title),
137 department: clean(j.departments?.[0]?.name), team: clean(j.departments?.[1]?.name),
138 location: clean(j.location?.name), allLocations: [...new Set([clean(j.location?.name), ...offices].filter(Boolean))],
139 workplaceType: workplace, isRemote: workplace === 'remote' || /remote/i.test(j.location?.name || ''),
140 employmentType: clean(meta['employment type'] || meta['employment_type'] || null),
141 postedAt: iso(j.first_published), updatedAt: iso(j.updated_at),
142 jobUrl: j.absolute_url || null, applyUrl: j.absolute_url || null,
143 salaryMin: null, salaryMax: null, salaryCurrency: null, salaryInterval: null, salaryText: null,
144 descriptionText: htmlToText(contentHtml),
145 ...(html ? { descriptionHtml: contentHtml } : {}),
146 };
147}
148
149export function normalizeLever(p, slug, { html = false } = {}) {
150 const c = p.categories || {};
151 const sr = p.salaryRange || null;
152 const workplace = normWorkplace(p.workplaceType);
153 const descHtml = [p.description, ...(p.lists || []).map((l) => `<h3>${l.text}</h3><ul>${l.content}</ul>`), p.additional].filter(Boolean).join('\n');
154 return {
155 ats: 'lever', company: slug, companyName: null,
156 jobId: p.id, title: clean(p.text),
157 department: clean(c.department), team: clean(c.team),
158 location: clean(c.location), allLocations: [...new Set([clean(c.location), ...(c.allLocations || []).map(clean)].filter(Boolean))],
159 workplaceType: workplace, isRemote: workplace === 'remote',
160 employmentType: clean(c.commitment),
161 postedAt: iso(p.createdAt), updatedAt: null,
162 jobUrl: p.hostedUrl || null, applyUrl: p.applyUrl || null,
163 salaryMin: sr?.min ?? null, salaryMax: sr?.max ?? null, salaryCurrency: sr?.currency ?? null,
164 salaryInterval: sr?.interval ? (INTERVALS[sr.interval] || sr.interval) : null,
165 salaryText: clean(p.salaryDescriptionPlain) || null,
166 descriptionText: htmlToText(descHtml),
167 ...(html ? { descriptionHtml: descHtml || null } : {}),
168 };
169}
170
171export function normalizeAshby(j, slug, { html = false } = {}) {
172 const comp = j.compensation || {};
173 const salary = (comp.summaryComponents || []).find((x) => x.compensationType === 'Salary') || null;
174 const workplace = normWorkplace(j.workplaceType) || (j.isRemote ? 'remote' : null);
175 return {
176 ats: 'ashby', company: slug, companyName: null,
177 jobId: j.id, title: clean(j.title),
178 department: clean(j.department), team: clean(j.team),
179 location: clean(j.location), allLocations: [...new Set([clean(j.location), ...(j.secondaryLocations || []).map((l) => clean(l.location))].filter(Boolean))],
180 workplaceType: workplace, isRemote: Boolean(j.isRemote) || workplace === 'remote',
181 employmentType: clean(j.employmentType),
182 postedAt: iso(j.publishedAt), updatedAt: null,
183 jobUrl: j.jobUrl || null, applyUrl: j.applyUrl || null,
184 salaryMin: salary?.minValue ?? null, salaryMax: salary?.maxValue ?? null, salaryCurrency: salary?.currencyCode ?? null,
185 salaryInterval: salary?.interval ? (INTERVALS[salary.interval] || salary.interval.toLowerCase()) : null,
186 salaryText: clean(comp.scrapeableCompensationSalarySummary || comp.compensationTierSummary),
187 descriptionText: j.descriptionPlain ? j.descriptionPlain.trim() : htmlToText(j.descriptionHtml),
188 ...(html ? { descriptionHtml: j.descriptionHtml || null } : {}),
189 };
190}
191
192
193export function normalizeWorkable(j, slug, companyName, { html = false } = {}) {
194 const locs = (j.locations || []).filter((l) => !l.hidden).map((l) => clean([l.city, l.region, l.country].filter(Boolean).join(', ')));
195 const primary = clean([j.city, j.state, j.country].filter(Boolean).join(', '));
196 return {
197 ats: 'workable', company: slug, companyName: clean(companyName),
198 jobId: j.shortcode, title: clean(j.title),
199 department: clean(j.department), team: clean(j.function),
200 location: primary, allLocations: [...new Set([primary, ...locs].filter(Boolean))],
201 workplaceType: j.telecommuting ? 'remote' : null, isRemote: Boolean(j.telecommuting),
202 employmentType: clean(j.employment_type),
203 postedAt: iso(j.published_on), updatedAt: null,
204 jobUrl: j.url || j.shortlink || null, applyUrl: j.application_url || null,
205 salaryMin: null, salaryMax: null, salaryCurrency: null, salaryInterval: null, salaryText: null,
206 descriptionText: htmlToText(j.description),
207 ...(html ? { descriptionHtml: j.description || null } : {}),
208 };
209}
210
211export function normalizeSmartRecruiters(p, slug) {
212 const loc = p.location || {};
213 const workplace = loc.remote ? 'remote' : loc.hybrid ? 'hybrid' : 'onsite';
214 return {
215 ats: 'smartrecruiters', company: slug, companyName: clean(p.company?.name),
216 jobId: String(p.id), title: clean(p.name),
217 department: clean(p.department?.label), team: clean(p.function?.label),
218 location: clean(loc.fullLocation || [loc.city, loc.region, loc.country].filter(Boolean).join(', ')), allLocations: [clean(loc.fullLocation)].filter(Boolean),
219 workplaceType: workplace, isRemote: Boolean(loc.remote),
220 employmentType: clean(p.typeOfEmployment?.label),
221 postedAt: iso(p.releasedDate), updatedAt: null,
222 jobUrl: `https://jobs.smartrecruiters.com/${encodeURIComponent(slug)}/${p.id}`, applyUrl: `https://jobs.smartrecruiters.com/${encodeURIComponent(slug)}/${p.id}`,
223 salaryMin: null, salaryMax: null, salaryCurrency: null, salaryInterval: null, salaryText: null,
224 descriptionText: null,
225 };
226}
227
228export function smartRecruitersDescription(detail) {
229 const sec = detail?.jobAd?.sections || {};
230 const parts = ['companyDescription', 'jobDescription', 'qualifications', 'additionalInformation']
231 .map((k) => sec[k]).filter((x) => x?.text).map((x) => `${x.title ? `<h3>${x.title}</h3>` : ''}${x.text}`);
232 const html = parts.join('\n');
233 return { html: html || null, text: htmlToText(html), postingUrl: detail?.postingUrl || null, applyUrl: detail?.applyUrl || null };
234}
235
236export function normalizeRecruitee(o, slug, { html = false } = {}) {
237 const workplace = o.remote ? 'remote' : o.hybrid ? 'hybrid' : o.on_site ? 'onsite' : null;
238 const sal = o.salary || {};
239 const descHtml = [o.description, o.requirements].filter(Boolean).join('\n');
240 const locs = (o.locations || []).map((l) => clean([l.city, l.state, l.country].filter(Boolean).join(', ')));
241 return {
242 ats: 'recruitee', company: slug, companyName: clean(o.company_name),
243 jobId: String(o.id), title: clean(o.title),
244 department: clean(o.department), team: null,
245 location: clean(o.location), allLocations: [...new Set([clean(o.location), ...locs].filter(Boolean))],
246 workplaceType: workplace, isRemote: Boolean(o.remote),
247 employmentType: clean(o.employment_type_code ? o.employment_type_code.replace(/_/g, ' ') : null),
248 postedAt: iso(o.published_at ? o.published_at.replace(' UTC', 'Z').replace(' ', 'T') : null), updatedAt: iso(o.updated_at ? o.updated_at.replace(' UTC', 'Z').replace(' ', 'T') : null),
249 jobUrl: o.careers_url || null, applyUrl: o.careers_apply_url || null,
250 salaryMin: sal.min ?? null, salaryMax: sal.max ?? null, salaryCurrency: sal.currency ?? null,
251 salaryInterval: sal.period ? (INTERVALS[sal.period] || sal.period) : null,
252 salaryText: sal.min || sal.max ? clean(`${sal.min ?? ''}–${sal.max ?? ''} ${sal.currency ?? ''} ${sal.period ? `per ${sal.period}` : ''}`) : null,
253 descriptionText: htmlToText(descHtml),
254 ...(html ? { descriptionHtml: descHtml || null } : {}),
255 };
256}
257
258const personioParser = new XMLParser({ ignoreAttributes: true, trimValues: true, parseTagValue: false, cdataPropName: false, isArray: (n) => ['position', 'jobDescription', 'office'].includes(n) });
259export function parsePersonioXml(xml) {
260 const doc = personioParser.parse(xml);
261 return [].concat(doc?.['workzag-jobs']?.position || []);
262}
263export function normalizePersonio(p, slug, tld = 'de', { html = false } = {}) {
264 const descs = [].concat(p.jobDescriptions?.jobDescription || []);
265 const descHtml = descs.map((d) => `${d.name ? `<h3>${d.name}</h3>` : ''}${typeof d.value === 'string' ? d.value : ''}`).join('\n');
266 const extra = [].concat(p.additionalOffices?.office || []).map(clean);
267 const all = [...new Set([clean(p.office), ...extra].filter(Boolean))];
268 const remote = all.some((l) => /remote/i.test(l));
269 return {
270 ats: 'personio', company: slug, companyName: clean(p.subcompany),
271 jobId: String(p.id), title: clean(decodeEntities(p.name)),
272 department: clean(p.department), team: clean(p.recruitingCategory),
273 location: clean(p.office), allLocations: all,
274 workplaceType: remote ? 'remote' : null, isRemote: remote,
275 employmentType: clean([p.employmentType, p.schedule].filter(Boolean).join(', ')),
276 postedAt: iso(p.createdAt), updatedAt: null,
277 jobUrl: `https://${slug}.jobs.personio.${tld}/job/${p.id}`, applyUrl: `https://${slug}.jobs.personio.${tld}/job/${p.id}#apply`,
278 salaryMin: null, salaryMax: null, salaryCurrency: null, salaryInterval: null, salaryText: null,
279 descriptionText: htmlToText(descHtml),
280 ...(html ? { descriptionHtml: descHtml || null } : {}),
281 };
282}
283
284
285export async function hydrateSmartRecruiters(items, { concurrency = 8, html = false, ...opts } = {}) {
286 const queue = items.filter((j) => j.ats === 'smartrecruiters' && j.descriptionText == null);
287 await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
288 while (queue.length) {
289 const j = queue.shift();
290 try {
291 const d = smartRecruitersDescription(await getJson(endpoints.smartrecruitersDetail(j.company, j.jobId), { maxRetries: 2, ...opts }));
292 j.descriptionText = d.text;
293 if (html) j.descriptionHtml = d.html;
294 if (d.postingUrl) j.jobUrl = d.postingUrl;
295 if (d.applyUrl) j.applyUrl = d.applyUrl;
296 } catch { }
297 }
298 }));
299 return items;
300}
301
302
303export async function fetchBoard({ ats, slug, region }, opts = {}) {
304 const { html = false } = opts;
305 if (ats === 'greenhouse') {
306 const d = await getJson(endpoints.greenhouse(slug), opts);
307 return { ats, slug, jobs: (d.jobs || []).map((j) => normalizeGreenhouse(j, slug, { html })) };
308 }
309 if (ats === 'lever') {
310 let d;
311 try { d = await getJson(endpoints.lever(slug, region), opts); } catch (e) {
312 if (e.status === 404 && region !== 'eu') { d = await getJson(endpoints.lever(slug, 'eu'), opts); region = 'eu'; } else throw e;
313 }
314 if (!Array.isArray(d)) throw new HttpError('Unexpected Lever response');
315 return { ats, slug, region, jobs: d.map((p) => normalizeLever(p, slug, { html })) };
316 }
317 if (ats === 'ashby') {
318 const d = await getJson(endpoints.ashby(slug), opts);
319 return { ats, slug, jobs: (d.jobs || []).filter((j) => j.isListed !== false).map((j) => normalizeAshby(j, slug, { html })) };
320 }
321 if (ats === 'workable') {
322 const d = await getJson(endpoints.workable(slug), opts);
323 return { ats, slug, jobs: (d.jobs || []).map((j) => normalizeWorkable(j, slug, d.name, { html })) };
324 }
325 if (ats === 'smartrecruiters') {
326 const jobs = [];
327 const maxPages = opts.maxListPages ?? 200;
328 for (let page = 0, offset = 0; page < maxPages; page++) {
329 const d = await getJson(endpoints.smartrecruiters(slug, offset), opts);
330 const content = d.content || [];
331 jobs.push(...content.map((p) => normalizeSmartRecruiters(p, slug)));
332 offset += content.length;
333 if (!content.length || offset >= (d.totalFound ?? 0)) break;
334 }
335 if (!jobs.length) throw new HttpError('Board not found or has no open jobs', 404);
336 return { ats, slug, jobs };
337 }
338 if (ats === 'recruitee') {
339 const d = await getJson(endpoints.recruitee(slug), opts);
340 return { ats, slug, jobs: (d.offers || []).map((o) => normalizeRecruitee(o, slug, { html })) };
341 }
342 if (ats === 'personio') {
343 let tld = region === 'com' ? 'com' : 'de';
344 let xml;
345 try { xml = await getText(endpoints.personio(slug, tld), opts); } catch (e) {
346 if (e.status === 404 && tld === 'de') { tld = 'com'; xml = await getText(endpoints.personio(slug, tld), opts); } else throw e;
347 }
348 if (!/<workzag-jobs/i.test(xml)) throw new HttpError('Board not found (not a Personio job feed)', 404);
349 return { ats, slug, region: tld, jobs: parsePersonioXml(xml).map((p) => normalizePersonio(p, slug, tld, { html })) };
350 }
351 throw new Error(`Unknown ATS ${ats}`);
352}
353
354
355export async function detectAndFetch(slug, opts = {}) {
356 const results = await Promise.allSettled(ATS_LIST.map((ats) => fetchBoard({ ats, slug }, { ...opts, maxRetries: 1 })));
357 const ok = results.filter((r) => r.status === 'fulfilled').map((r) => r.value).sort((a, b) => b.jobs.length - a.jobs.length);
358 if (!ok.length) {
359 const nonMissing = results.find((r) => r.status === 'rejected' && r.reason?.status !== 404);
360 throw nonMissing ? nonMissing.reason : new HttpError(`No job board found for "${slug}" on Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee or Personio`, 404);
361 }
362 return ok[0];
363}
