1
2
3
4
5
6
7
8
9
10
11
12
13
14
15import { createHash } from 'node:crypto';
16import { createInterface } from 'node:readline';
17import { Readable } from 'node:stream';
18import { createGunzip } from 'node:zlib';
19import { Actor, log } from 'apify';
20
21const USER_AGENT = 'foundry-actor/0.1 (Apify Actor; polite client; +https://apify.com/tabulary)';
22const MIN_INTERVAL_MS = 1000;
23const lastRequestAtByHost = new Map();
24
25
26
27async function politeGet(url, attempt = 1) {
28 const host = new URL(url).host;
29 const wait = (lastRequestAtByHost.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
30 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
31 lastRequestAtByHost.set(host, Date.now());
32
33 const res = await fetch(url, { headers: { 'user-agent': USER_AGENT, accept: 'application/json' } });
34 if (res.status === 429 || res.status >= 500) {
35 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
36 await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
37 return politeGet(url, attempt + 1);
38 }
39 let body = null;
40 try {
41 body = await res.json();
42 } catch {
43 body = null;
44 }
45 return { status: res.status, body };
46}
47
48async function politeGetText(url, attempt = 1) {
49 const host = new URL(url).host;
50 const wait = (lastRequestAtByHost.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
51 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
52 lastRequestAtByHost.set(host, Date.now());
53 const res = await fetch(url, { headers: { 'user-agent': USER_AGENT } });
54 if (res.status === 429 || res.status >= 500) {
55 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
56 await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
57 return politeGetText(url, attempt + 1);
58 }
59 return { status: res.status, text: res.ok ? await res.text() : null };
60}
61
62
63function xmlField(block, tag) {
64 const m = new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`).exec(block);
65 if (!m) return null;
66 const v = m[1].replace(/^<!\[CDATA\[([\s\S]*?)\]\]>$/, '$1').replace(/&/g, '&').replace(/</g, '<')
67 .replace(/>/g, '>').replace(/"/g, '"').replace(/'|'/g, "'").trim();
68 return v || null;
69}
70
71
72
73
74function stripHtml(html) {
75 if (!html) return null;
76 let text = String(html)
77 .replace(/</g, '<')
78 .replace(/>/g, '>')
79 .replace(/"/g, '"')
80 .replace(/'/g, "'")
81 .replace(/ /g, ' ');
82 text = text
83 .replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, ' ')
84 .replace(/<[^>]+>/g, ' ')
85 .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
86 .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
87 .replace(/&/g, '&')
88 .replace(/ /g, ' ')
89 .replace(/\s+/g, ' ')
90 .trim();
91 return text || null;
92}
93
94const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
95function maskEmails(text) {
96 return text ? text.replace(EMAIL_RE, '[email removed]') : text;
97}
98
99function normalizeEmploymentType(raw) {
100 if (!raw) return null;
101 const s = String(raw).toLowerCase();
102 if (s.includes('full')) return 'full_time';
103 if (s.includes('part')) return 'part_time';
104 if (s.includes('intern')) return 'internship';
105 if (s.includes('contract') || s.includes('freelance')) return 'contract';
106 if (s.includes('temp')) return 'temporary';
107 return 'other';
108}
109
110function toIso(value) {
111 if (!value) return null;
112 const d = new Date(value);
113 return Number.isNaN(d.getTime()) ? null : d.toISOString();
114}
115
116
117
118
119async function fetchGreenhouse(slug) {
120 const { status, body } = await politeGet(
121 `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(slug)}/jobs?content=true`,
122 );
123 if (status === 404) return { detected: false };
124 if (status !== 200 || !body) throw new Error(`Greenhouse: unexpected response for board "${slug}"`);
125 const records = (body.jobs ?? []).map((j) => {
126 const location = j.location?.name ?? null;
127 return {
128 company: slug,
129 company_name: j.company_name ?? null,
130 ats: 'greenhouse',
131 job_id: String(j.id),
132 title: j.title ?? null,
133 department: j.departments?.[0]?.name ?? null,
134 location,
135 remote: location ? /remote/i.test(location) : null,
136 employment_type: null,
137 posted_at: toIso(j.first_published),
138 updated_at: toIso(j.updated_at),
139 url: j.absolute_url ?? null,
140 apply_url: j.absolute_url ?? null,
141 description: stripHtml(j.content),
142 };
143 });
144 return { detected: true, records };
145}
146
147async function fetchLever(slug) {
148 const { status, body } = await politeGet(
149 `https://api.lever.co/v0/postings/${encodeURIComponent(slug)}?mode=json`,
150 );
151 if (status === 404) return { detected: false };
152 if (status !== 200 || !Array.isArray(body)) throw new Error(`Lever: unexpected response for board "${slug}"`);
153 const records = body.map((j) => {
154 const workplaceType = (j.workplaceType ?? '').toLowerCase();
155 return {
156 company: slug,
157 company_name: null,
158 ats: 'lever',
159 job_id: String(j.id),
160 title: j.text ?? null,
161 department: j.categories?.department ?? j.categories?.team ?? null,
162 location: j.categories?.location ?? null,
163 remote: workplaceType ? workplaceType === 'remote' : null,
164 employment_type: normalizeEmploymentType(j.categories?.commitment),
165 posted_at: toIso(j.createdAt),
166 updated_at: null,
167 url: j.hostedUrl ?? null,
168 apply_url: j.applyUrl ?? j.hostedUrl ?? null,
169 description: j.descriptionPlain?.trim() || stripHtml(j.description),
170 };
171 });
172 return { detected: true, records };
173}
174
175async function fetchAshby(slug) {
176 const { status, body } = await politeGet(
177 `https://api.ashbyhq.com/posting-api/job-board/${encodeURIComponent(slug)}`,
178 );
179 if (status === 404) return { detected: false };
180 if (status !== 200 || !body) throw new Error(`Ashby: unexpected response for board "${slug}"`);
181 const records = (body.jobs ?? []).map((j) => ({
182 company: slug,
183 company_name: null,
184 ats: 'ashby',
185 job_id: String(j.id),
186 title: j.title ?? null,
187 department: j.department ?? j.team ?? null,
188 location: j.location ?? null,
189 remote: typeof j.isRemote === 'boolean' ? j.isRemote : null,
190 employment_type: normalizeEmploymentType(j.employmentType),
191 posted_at: toIso(j.publishedAt),
192 updated_at: null,
193 url: j.jobUrl ?? null,
194 apply_url: j.applyUrl ?? j.jobUrl ?? null,
195 description: j.descriptionPlain?.trim() || stripHtml(j.descriptionHtml),
196 }));
197 return { detected: true, records };
198}
199
200async function fetchWorkable(slug) {
201 const { status, body } = await politeGet(
202 `https://apply.workable.com/api/v1/widget/accounts/${encodeURIComponent(slug)}?details=true`,
203 );
204 if (status === 404) return { detected: false };
205 if (status !== 200 || !body) throw new Error(`Workable: unexpected response for account "${slug}"`);
206 const records = (body.jobs ?? []).map((j) => {
207 const location = [j.city, j.state, j.country].filter(Boolean).join(', ') || null;
208 return {
209 company: slug,
210 company_name: body.name ?? null,
211 ats: 'workable',
212 job_id: j.shortcode ?? String(j.id ?? j.title),
213 title: j.title ?? null,
214 department: j.department ?? null,
215 location,
216 remote: typeof j.telecommuting === 'boolean' ? j.telecommuting : null,
217 employment_type: normalizeEmploymentType(j.employment_type),
218 posted_at: toIso(j.published_on),
219 updated_at: null,
220 url: j.url ?? j.shortlink ?? null,
221 apply_url: j.application_url ?? null,
222 description: stripHtml(j.description),
223 };
224 });
225 return { detected: true, records };
226}
227
228
229
230
231
232
233async function fetchRecruitee(slug) {
234 const { status, body } = await politeGet(
235 `https://${encodeURIComponent(slug)}.recruitee.com/api/offers/`,
236 );
237 if (status === 404) return { detected: false };
238 if (status !== 200 || !body) throw new Error(`Recruitee: unexpected response for company "${slug}"`);
239 const records = (body.offers ?? []).map((j) => {
240 const location = [j.city, j.state_name, j.country].filter(Boolean).join(', ') || null;
241 return {
242 company: slug,
243 company_name: j.company_name ?? null,
244 ats: 'recruitee',
245 job_id: String(j.id),
246 title: j.title ?? null,
247 department: j.department ?? null,
248 location,
249 remote: typeof j.remote === 'boolean' ? j.remote : null,
250 employment_type: normalizeEmploymentType(j.employment_type_code),
251 posted_at: toIso(j.published_at),
252 updated_at: toIso(j.updated_at),
253 url: j.careers_url ?? null,
254 apply_url: j.careers_apply_url ?? j.careers_url ?? null,
255 description: stripHtml(j.description),
256 };
257 });
258 return { detected: true, records };
259}
260
261
262
263const SR_DESCRIPTION_LIMIT = 100;
264async function smartRecruitersDescription(slug, id) {
265 const { status, body } = await politeGet(
266 `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(slug)}/postings/${encodeURIComponent(id)}`,
267 );
268 if (status !== 200 || !body?.jobAd?.sections) return null;
269 return stripHtml(Object.values(body.jobAd.sections).map((s) => s?.text ?? '').join(' '));
270}
271
272async function fetchSmartRecruiters(slug, { descriptions = true } = {}) {
273 const records = [];
274 for (let offset = 0; offset < 1000; offset += 100) {
275 const { status, body } = await politeGet(
276 `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(slug)}/postings?limit=100&offset=${offset}`,
277 );
278 if (status === 404) return { detected: false };
279 if (status !== 200 || !body) throw new Error(`SmartRecruiters: unexpected response for company "${slug}"`);
280 const content = body.content ?? [];
281
282 if (offset === 0 && content.length === 0) return { detected: false };
283 for (const j of content) {
284 const loc = j.location ?? {};
285 records.push({
286 company: slug,
287 company_name: j.company?.name ?? null,
288 ats: 'smartrecruiters',
289 job_id: String(j.id),
290 title: j.name ?? null,
291 department: j.department?.label ?? null,
292 location: [loc.city, loc.region, loc.country?.toUpperCase?.()].filter(Boolean).join(', ') || null,
293 remote: typeof loc.remote === 'boolean' ? loc.remote : null,
294 employment_type: normalizeEmploymentType(j.typeOfEmployment?.label),
295 posted_at: toIso(j.releasedDate),
296 updated_at: null,
297 url: `https://jobs.smartrecruiters.com/${slug}/${j.id}`,
298 apply_url: `https://jobs.smartrecruiters.com/${slug}/${j.id}`,
299 description: null,
300 });
301 }
302 if (content.length === 0 || offset + 100 >= (body.totalFound ?? 0)) break;
303 }
304 if (descriptions) {
305 if (records.length > SR_DESCRIPTION_LIMIT) {
306 log.info(`SmartRecruiters "${slug}": descriptions for the first ${SR_DESCRIPTION_LIMIT} of ${records.length} jobs only.`);
307 }
308 for (const r of records.slice(0, SR_DESCRIPTION_LIMIT)) r.description = await smartRecruitersDescription(slug, r.job_id);
309 }
310 return { detected: true, records };
311}
312
313async function fetchPersonio(slug) {
314 let tld = 'de';
315 let { status, text } = await politeGetText(`https://${encodeURIComponent(slug)}.jobs.personio.de/xml`);
316 if (status === 404) {
317 tld = 'com';
318 ({ status, text } = await politeGetText(`https://${encodeURIComponent(slug)}.jobs.personio.com/xml`));
319 }
320 if (status === 404 || (status === 200 && !text?.includes('<workzag-jobs'))) return { detected: false };
321 if (status !== 200 || !text) throw new Error(`Personio: unexpected response for company "${slug}"`);
322 const records = (text.match(/<position>[\s\S]*?<\/position>/g) ?? []).map((b) => {
323 const id = xmlField(b, 'id');
324 const office = xmlField(b, 'office');
325 const sections = b.match(/<value>[\s\S]*?<\/value>/g) ?? [];
326 return {
327 company: slug,
328 company_name: xmlField(b, 'subcompany'),
329 ats: 'personio',
330 job_id: id,
331 title: xmlField(b, 'name'),
332 department: xmlField(b, 'department'),
333 location: office,
334 remote: office ? /remote|home ?office/i.test(office) : null,
335 employment_type: normalizeEmploymentType(`${xmlField(b, 'schedule') ?? ''} ${xmlField(b, 'employmentType') ?? ''}`.trim()),
336 posted_at: toIso(xmlField(b, 'createdAt')),
337 updated_at: null,
338 url: `https://${slug}.jobs.personio.${tld}/job/${id}`,
339 apply_url: `https://${slug}.jobs.personio.${tld}/job/${id}`,
340 description: stripHtml(sections.map((s) => xmlField(s, 'value') ?? '').join(' ')),
341 };
342 }).filter((r) => r.job_id);
343 return { detected: true, records };
344}
345
346
347
348
349
350async function fetchJazzHR(slug) {
351 const { status, text } = await politeGetText(`https://app.jazz.co/feeds/export/jobs/${encodeURIComponent(slug)}`);
352 if (status === 404) return { detected: false };
353 if (status !== 200 || !text?.includes('<jobs>')) throw new Error(`JazzHR: unexpected response for company "${slug}"`);
354 const companyName = xmlField(text, 'company');
355 const records = (text.match(/<job>[\s\S]*?<\/job>/g) ?? [])
356 .filter((b) => (xmlField(b, 'status') ?? '').toLowerCase() === 'open')
357 .map((b) => {
358 const location = [xmlField(b, 'city'), xmlField(b, 'state'), xmlField(b, 'country')].filter(Boolean).join(', ') || null;
359 const url = xmlField(b, 'url');
360 return {
361 company: slug,
362 company_name: companyName,
363 ats: 'jazzhr',
364 job_id: xmlField(b, 'id'),
365 title: xmlField(b, 'title'),
366 department: xmlField(b, 'department'),
367 location,
368 remote: location ? /remote/i.test(location) : null,
369 employment_type: normalizeEmploymentType(xmlField(b, 'type')),
370 posted_at: null,
371 updated_at: null,
372 url,
373 apply_url: url,
374 description: stripHtml(xmlField(b, 'description')),
375 };
376 })
377 .filter((r) => r.job_id);
378 return { detected: true, records };
379}
380
381
382
383
384
385
386async function fetchPinpoint(slug) {
387 const { status, body } = await politeGet(
388 `https://${encodeURIComponent(slug)}.pinpointhq.com/postings.json`,
389 );
390 if (status === 404) return { detected: false };
391 if (status !== 200 || !Array.isArray(body)) throw new Error(`Pinpoint: unexpected response for company "${slug}"`);
392 const records = body.map((j) => {
393 const workplaceType = (j.workplace_type ?? '').toLowerCase();
394 const description = [j.description, j.key_responsibilities, j.skills_knowledge_expertise, j.benefits]
395 .filter(Boolean).join(' ');
396 return {
397 company: slug,
398 company_name: null,
399 ats: 'pinpoint',
400 job_id: String(j.id),
401 title: j.title ?? null,
402 department: j.job?.department?.name ?? null,
403 location: j.location?.name ?? null,
404 remote: workplaceType ? workplaceType === 'remote' : null,
405 employment_type: normalizeEmploymentType(j.employment_type_text ?? j.employment_type),
406 posted_at: null,
407 updated_at: null,
408 url: j.url ?? null,
409 apply_url: j.url ?? null,
410 description: stripHtml(description),
411 };
412 });
413 return { detected: true, records };
414}
415
416
417
418
419
420
421
422
423async function fetchComeet(uid, token) {
424 const { status, body } = await politeGet(
425 `https://www.comeet.co/careers-api/2.0/company/${encodeURIComponent(uid)}/positions?token=${encodeURIComponent(token)}&details=true`,
426 );
427 if (status === 400) {
428 throw new Error(`Comeet: company_uid "${uid}" and/or its token were rejected (HTTP 400). Copy both fresh from the company's own careers page (see README).`);
429 }
430 if (status !== 200 || !Array.isArray(body)) throw new Error(`Comeet: unexpected response for company_uid "${uid}"`);
431 const records = body.map((j) => {
432 const workplaceType = (j.workplace_type ?? '').toLowerCase();
433 const loc = j.location ?? {};
434 const location = [loc.city, loc.state, loc.country].filter(Boolean).join(', ') || loc.name || null;
435 const details = Array.isArray(j.details)
436 ? [...j.details].sort((a, b) => (a.order ?? 0) - (b.order ?? 0)).map((d) => d.value ?? '').join(' ')
437 : '';
438 const url = j.url_active_page ?? j.url_comeet_hosted_page ?? j.url_recruit_hosted_page ?? null;
439 return {
440 company: uid,
441 company_name: j.company_name ?? null,
442 ats: 'comeet',
443 job_id: j.uid ?? null,
444 title: j.name ?? null,
445 department: j.department ?? null,
446 location,
447 remote: workplaceType ? workplaceType.includes('remote') : (typeof loc.is_remote === 'boolean' ? loc.is_remote : null),
448 employment_type: normalizeEmploymentType(j.employment_type),
449 posted_at: null,
450 updated_at: toIso(j.time_updated),
451 url,
452 apply_url: url,
453 description: stripHtml(details),
454 };
455 }).filter((r) => r.job_id);
456 return { detected: true, records };
457}
458
459
460
461
462
463
464
465
466const MANATAL_PAGE_SIZE = 100;
467async function fetchManatal(slug) {
468 const records = [];
469 for (let page = 1; ; page++) {
470 const { status, body } = await politeGet(
471 `https://api.manatal.com/open/v3/career-page/${encodeURIComponent(slug)}/jobs/?page=${page}&page_size=${MANATAL_PAGE_SIZE}`,
472 );
473 if (status === 404) return page === 1 ? { detected: false } : { detected: true, records };
474 if (status !== 200 || !Array.isArray(body?.results)) throw new Error(`Manatal: unexpected response for company "${slug}"`);
475 for (const j of body.results) {
476 const url = `https://www.careers-page.com/${slug}/job/${j.hash}`;
477 records.push({
478 company: slug,
479 company_name: null,
480 ats: 'manatal',
481 job_id: String(j.id),
482 title: j.position_name ?? null,
483 department: j.organization_name ?? null,
484 location: j.location_display || [j.city, j.state, j.country].filter(Boolean).join(', ') || null,
485 remote: typeof j.is_remote === 'boolean' ? j.is_remote : null,
486 employment_type: normalizeEmploymentType(j.contract_details),
487 posted_at: null,
488 updated_at: null,
489 url,
490 apply_url: url,
491 description: stripHtml(j.description),
492 });
493 }
494 if (body.results.length < MANATAL_PAGE_SIZE || !body.next) break;
495 }
496 return { detected: true, records };
497}
498
499
500
501
502
503
504
505
506
507const HIREHIVE_PAGE_SIZE = 100;
508async function fetchHireHive(slug) {
509 const records = [];
510 for (let page = 1; ; page++) {
511 const { status, body } = await politeGet(
512 `https://${encodeURIComponent(slug)}.hirehive.com/api/v2/jobs?page=${page}&page_size=${HIREHIVE_PAGE_SIZE}`,
513 );
514 if (status === 404) return page === 1 ? { detected: false } : { detected: true, records };
515 if (status !== 200 || !Array.isArray(body?.items)) throw new Error(`HireHive: unexpected response for company "${slug}"`);
516 for (const j of body.items) {
517 const location = [j.location, j.state_code, j.country?.name].filter(Boolean).join(', ') || null;
518 records.push({
519 company: slug,
520 company_name: null,
521 ats: 'hirehive',
522 job_id: j.id ?? null,
523 title: j.title ?? null,
524 department: j.category?.name ?? null,
525 location,
526 remote: location ? /remote/i.test(location) : null,
527 employment_type: normalizeEmploymentType(j.type?.type ?? j.type?.name),
528 posted_at: toIso(j.published_date),
529 updated_at: null,
530 url: j.hosted_url ?? null,
531 apply_url: j.hosted_url ?? null,
532 description: j.description?.text?.trim() || stripHtml(j.description?.html),
533 });
534 }
535 if (!body.meta?.has_next_page) break;
536 }
537 return { detected: true, records: records.filter((r) => r.job_id) };
538}
539
540const DETECTORS = [
541 ['greenhouse', fetchGreenhouse],
542 ['lever', fetchLever],
543 ['ashby', fetchAshby],
544 ['workable', fetchWorkable],
545 ['recruitee', fetchRecruitee],
546 ['smartrecruiters', fetchSmartRecruiters],
547 ['personio', fetchPersonio],
548 ['jazzhr', fetchJazzHR],
549 ['pinpoint', fetchPinpoint],
550 ['manatal', fetchManatal],
551 ['hirehive', fetchHireHive],
552];
553
554
555
556
557function hostToAts(host) {
558 if (host.endsWith('greenhouse.io')) return 'greenhouse';
559 if (host === 'jobs.lever.co') return 'lever';
560 if (host === 'jobs.ashbyhq.com') return 'ashby';
561 if (host === 'apply.workable.com' || host.endsWith('.workable.com')) return 'workable';
562
563 if (host.endsWith('.recruitee.com') && host !== 'api.recruitee.com') return 'recruitee';
564 if (host === 'jobs.smartrecruiters.com' || host === 'careers.smartrecruiters.com') return 'smartrecruiters';
565 if (host.endsWith('.jobs.personio.de') || host.endsWith('.jobs.personio.com')) return 'personio';
566 if (host.endsWith('.applytojob.com')) return 'jazzhr';
567
568 if (host.endsWith('.pinpointhq.com') && host !== 'www.pinpointhq.com') return 'pinpoint';
569
570
571 if (host === 'careers-page.com' || host === 'www.careers-page.com') return 'manatal';
572 if (host.endsWith('.careers-page.com')) return 'manatal';
573
574 if (host.endsWith('.hirehive.com') && host !== 'www.hirehive.com') return 'hirehive';
575 return null;
576}
577
578
579
580
581function parseCompanyInput(raw) {
582 const trimmed = raw.trim();
583 if (/^comeet:/i.test(trimmed)) {
584 const rest = trimmed.slice('comeet:'.length);
585 const i = rest.indexOf(':');
586 return i === -1 ? { slug: null, ats: 'comeet', token: null } : { slug: rest.slice(0, i), ats: 'comeet', token: rest.slice(i + 1) };
587 }
588 if (!/^https?:\/\//i.test(trimmed)) return { slug: trimmed, ats: null };
589 let url;
590 try {
591 url = new URL(trimmed);
592 } catch {
593 return { slug: trimmed, ats: null };
594 }
595 if (/(^|\.)comeet\.co$/.test(url.host)) {
596 const m = /\/careers-api\/2\.0\/company\/([^/]+)\/positions/.exec(url.pathname);
597 const token = url.searchParams.get('token');
598 if (m && token) return { slug: m[1], ats: 'comeet', token };
599 }
600 const ats = hostToAts(url.host);
601 if (ats === 'recruitee' || ats === 'personio' || ats === 'jazzhr' || ats === 'pinpoint' || ats === 'hirehive' || (ats === 'workable' && url.host !== 'apply.workable.com')) {
602 return { slug: url.host.split('.')[0], ats };
603 }
604
605
606
607 if (ats === 'manatal' && url.host !== 'careers-page.com' && url.host !== 'www.careers-page.com') {
608 return { slug: url.host.split('.')[0], ats };
609 }
610 const firstSegment = url.pathname.split('/').filter(Boolean)[0];
611 return { slug: firstSegment ?? url.host, ats };
612}
613
614async function fetchCompanyJobs(rawInput) {
615 const { slug, ats, token } = parseCompanyInput(rawInput);
616 if (ats === 'comeet') {
617 if (!slug || !token) {
618 throw new Error(`"${rawInput}": Comeet needs both a company_uid and a token. Use "comeet:<company_uid>:<token>" or the full careers-api URL with its token (see README).`);
619 }
620 const { records } = await fetchComeet(slug, token);
621 return { ats: 'comeet', records };
622 }
623 if (!slug) throw new Error(`Could not read a board slug from "${rawInput}".`);
624 const order = ats ? [[ats, DETECTORS.find(([name]) => name === ats)[1]], ...DETECTORS.filter(([name]) => name !== ats)] : DETECTORS;
625 for (const [name, fetcher] of order) {
626 const result = await fetcher(slug);
627 if (result.detected) return { ats: name, records: result.records };
628 }
629 throw new Error(`"${rawInput}" was not found on Greenhouse, Lever, Ashby, Workable, Recruitee, SmartRecruiters, Personio, JazzHR, Pinpoint, Manatal or HireHive. For Comeet, pass "comeet:<company_uid>:<token>" (it can't be auto-detected from a bare name).`);
630}
631
632
633
634function remainingBudget() {
635 try {
636 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
637 return Number.isFinite(n) ? n : Infinity;
638 } catch {
639 return Infinity;
640 }
641}
642
643
644
645
646
647const INDEX_BASE = 'https://api.apify.com/v2/key-value-stores/EeS6AjFCcCZJuzQ4Y/records';
648
649
650const PLATFORMS = [...DETECTORS.map(([name]) => name), 'comeet'];
651
652
653async function* indexRows(quick = null) {
654 const res = await fetch(`${INDEX_BASE}/jobs-index.ndjson.gz`, { headers: { 'user-agent': USER_AGENT } });
655 if (!res.ok || !res.body) {
656 throw new Error(`The job index could not be loaded (HTTP ${res.status}). Try again in a few minutes, or list companies to fetch their boards directly.`);
657 }
658 const lines = createInterface({ input: Readable.fromWeb(res.body).pipe(createGunzip()), crlfDelay: Infinity });
659 for await (const line of lines) {
660 if (line && (!quick || quick(line.toLowerCase()))) yield JSON.parse(line);
661 }
662}
663
664async function indexMeta() {
665 try {
666 const res = await fetch(`${INDEX_BASE}/jobs-index-meta`, { headers: { 'user-agent': USER_AGENT } });
667 return res.ok ? await res.json() : null;
668 } catch {
669 return null;
670 }
671}
672
673
674const toRecord = (r) => ({
675 company: r.company, company_name: r.company_name ?? null, ats: r.ats, job_id: r.job_id,
676 title: r.title ?? null, department: r.department ?? null, location: r.location ?? null,
677 remote: r.remote ?? null, employment_type: r.employment_type ?? null, posted_at: r.posted_at ?? null,
678 updated_at: r.updated_at ?? null, url: r.url ?? null, apply_url: r.apply_url ?? null,
679 description: maskEmails(r.description ?? null),
680});
681
682
683
684async function addDescriptions(rows) {
685 const groups = new Map();
686 for (const r of rows) {
687 const key = `${r.ats}\t${r.company}`;
688 if (!groups.has(key)) groups.set(key, []);
689 groups.get(key).push(r);
690 }
691 const byPlatform = new Map();
692 for (const key of groups.keys()) {
693 const ats = key.split('\t')[0];
694 if (!byPlatform.has(ats)) byPlatform.set(ats, []);
695 byPlatform.get(ats).push(key);
696 }
697 log.info(`Fetching descriptions from ${groups.size} company board(s)...`);
698 await Promise.all([...byPlatform.entries()].map(async ([ats, keys]) => {
699 const fetcher = DETECTORS.find(([name]) => name === ats)[1];
700 for (const key of keys) {
701 const company = key.split('\t')[1];
702 if (ats === 'smartrecruiters') {
703 for (const r of groups.get(key)) {
704 try {
705 r.description = await smartRecruitersDescription(company, r.job_id);
706 } catch (err) {
707 log.warning(`Could not fetch a description for ${ats}/${company}/${r.job_id}: ${err.message}`);
708 }
709 }
710 continue;
711 }
712 try {
713 const result = await fetcher(company);
714 const byId = new Map((result.records ?? []).map((x) => [x.job_id, x.description]));
715 for (const r of groups.get(key)) r.description = byId.get(r.job_id) ?? null;
716 } catch (err) {
717 log.warning(`Could not fetch descriptions for ${ats}/${company}: ${err.message}`);
718 }
719 }
720 }));
721}
722
723await Actor.init();
724
725const input = (await Actor.getInput()) ?? {};
726const companies = Array.isArray(input.companies) ? input.companies.map((c) => String(c).trim()).filter(Boolean) : [];
727const onlyNewSinceLastRun = input.onlyNewSinceLastRun ?? false;
728const splitList = (v, name) => {
729 if (v === undefined || v === null) return [];
730 if (typeof v !== 'string') throw new Error(`Input "${name}" must be a string of comma-separated words.`);
731 return v.split(',').map((x) => x.trim().toLowerCase()).filter(Boolean);
732};
733const titleKeywords = splitList(input.titleKeywords, 'titleKeywords');
734const locationKeywords = splitList(input.locationKeywords, 'locationKeywords');
735const companyKeywords = splitList(input.companyKeywords, 'companyKeywords');
736const remoteOnly = input.remoteOnly === true;
737const postedWithinDays = input.postedWithinDays === undefined || input.postedWithinDays === null || input.postedWithinDays === ''
738 ? null : Number(input.postedWithinDays);
739if (postedWithinDays !== null && !(postedWithinDays >= 1 && postedWithinDays <= 365)) {
740 throw new Error('Input "postedWithinDays" must be a number from 1 to 365.');
741}
742const maxItems = Math.max(1, Math.min(Number(input.maxItems ?? 1000), 100000));
743const platforms = Array.isArray(input.platforms) && input.platforms.length ? input.platforms.map(String) : PLATFORMS;
744const unknownPlatforms = platforms.filter((p) => !PLATFORMS.includes(p));
745if (unknownPlatforms.length) throw new Error(`Unknown platform(s) ${unknownPlatforms.join(', ')}; use ${PLATFORMS.join(', ')}.`);
746const includeDescription = input.includeDescription === true;
747
748const filtering = titleKeywords.length > 0 || locationKeywords.length > 0 || remoteOnly;
749const matches = (r) => (!titleKeywords.length || titleKeywords.some((k) => (r.title ?? '').toLowerCase().includes(k)))
750 && (!locationKeywords.length || locationKeywords.some((k) => (r.location ?? '').toLowerCase().includes(k)))
751 && (!remoteOnly || r.remote === true);
752const cutoff = postedWithinDays ? Date.now() - postedWithinDays * 86_400_000 : null;
753const recent = (r) => !cutoff || (Date.parse(r.posted_at ?? r.updated_at ?? '') || 0) >= cutoff;
754const version = (r) => r.updated_at ?? r.posted_at ?? '1';
755
756const stateStore = onlyNewSinceLastRun ? await Actor.openKeyValueStore('multi-ats-job-postings-seen') : null;
757let saved = 0;
758
759if (companies.length === 0) {
760
761 const meta = await indexMeta();
762 if (meta) {
763 log.info(`Searching the job index built ${meta.built_at}: ${meta.jobs} open jobs at ${meta.boards_active} companies.`);
764 if (Date.now() - Date.parse(meta.built_at) > 3 * 86_400_000) {
765 log.warning('The job index is more than 3 days old, so the newest postings may be missing.');
766 }
767 }
768 const companyMatch = (r) => !companyKeywords.length
769 || companyKeywords.some((k) => `${r.company_name ?? ''} ${r.company}`.toLowerCase().includes(k));
770 const found = [];
771 const quick = titleKeywords.length ? (low) => titleKeywords.some((k) => low.includes(k)) : null;
772 for await (const r of indexRows(quick)) {
773 if (platforms.includes(r.ats) && companyMatch(r) && matches(r) && recent(r)) found.push(r);
774 }
775 found.sort((a, b) => (Date.parse(b.posted_at ?? '') || 0) - (Date.parse(a.posted_at ?? '') || 0));
776
777 const jobKey = (r) => `${r.ats}:${r.company}:${r.job_id}`;
778 let seenKey = null;
779 let seen = {};
780 let fresh = found;
781 if (onlyNewSinceLastRun) {
782 const sorted = (l) => [...l].sort();
783 const fingerprint = JSON.stringify({ mode: 'search', titleKeywords: sorted(titleKeywords),
784 locationKeywords: sorted(locationKeywords), companyKeywords: sorted(companyKeywords), remoteOnly,
785 platforms: sorted(platforms), postedWithinDays });
786 seenKey = `search-${createHash('sha256').update(fingerprint).digest('hex').slice(0, 40)}`;
787 seen = (await stateStore.getValue(seenKey)) ?? {};
788 fresh = found.filter((r) => seen[jobKey(r)] !== version(r));
789 }
790
791 const selected = fresh.slice(0, Math.min(maxItems, remainingBudget()));
792 if (includeDescription && selected.length) await addDescriptions(selected);
793 for (let i = 0; i < selected.length; i += 500) await Actor.pushData(selected.slice(i, i + 500).map(toRecord));
794 saved = selected.length;
795
796 if (onlyNewSinceLastRun) {
797
798
799 const savedKeys = new Set(selected.map(jobKey));
800 const nextSeen = {};
801 for (const r of found) {
802 const k = jobKey(r);
803 if (seen[k] === version(r) || savedKeys.has(k)) nextSeen[k] = version(r);
804 }
805 await stateStore.setValue(seenKey, nextSeen);
806 }
807 log.info(`${found.length} job(s) matched${onlyNewSinceLastRun ? `, ${fresh.length} new since the last run` : ''}; saved ${saved}.`);
808 if (found.length === 0) await Actor.setStatusMessage('No jobs in the index matched these filters.');
809 else if (onlyNewSinceLastRun && fresh.length === 0) await Actor.setStatusMessage('No new or updated jobs since the last run.');
810} else {
811
812 let stopped = false;
813 for (const rawInput of companies) {
814 if (stopped) break;
815 let matchedAts;
816 let records;
817 try {
818 ({ ats: matchedAts, records } = await fetchCompanyJobs(rawInput));
819 } catch (err) {
820 log.error(`Skipping "${rawInput}": ${err.message}`);
821 continue;
822 }
823 if (!platforms.includes(matchedAts)) {
824 log.info(`"${rawInput}": skipped, it is on ${matchedAts}, which is not in the selected platforms.`);
825 continue;
826 }
827 const allRecords = records;
828 if (records.length === 0) {
829 log.info(`"${rawInput}": matched ${matchedAts} but has 0 open jobs right now. A bare slug is checked against eleven platforms in a fixed order and a different company can own the same slug on another one; pass the full careers-page URL to be unambiguous.`);
830 }
831
832 records = records.map((r) => ({ ...r, description: maskEmails(r.description) }));
833 if (filtering) records = records.filter(matches);
834 records = records.filter(recent);
835
836
837
838
839 const memoryId = filtering
840 ? `${rawInput.toLowerCase()}|${JSON.stringify({ titleKeywords: [...titleKeywords].sort(), locationKeywords: [...locationKeywords].sort(), remoteOnly })}`
841 : rawInput.toLowerCase();
842 const seenKey = onlyNewSinceLastRun
843 ? `seen-${createHash('sha256').update(memoryId).digest('hex').slice(0, 40)}`
844 : null;
845 const seen = onlyNewSinceLastRun ? ((await stateStore.getValue(seenKey)) ?? {}) : {};
846 if (onlyNewSinceLastRun) records = records.filter((r) => seen[r.job_id] !== version(r));
847
848 const room = Math.min(remainingBudget(), maxItems - saved);
849 const batch = records.slice(0, Math.max(0, room));
850 if (records.length > 0 && batch.length === 0) {
851 log.info('Reached the maximum results or charge set for this run; stopping.');
852 stopped = true;
853 break;
854 }
855 if (batch.length > 0) await Actor.pushData(batch.map(toRecord));
856
857 if (onlyNewSinceLastRun) {
858
859
860 const savedIds = new Set(batch.map((r) => r.job_id));
861 const nextSeen = {};
862 for (const r of allRecords) {
863 if (seen[r.job_id] === version(r) || savedIds.has(r.job_id)) nextSeen[r.job_id] = version(r);
864 }
865 await stateStore.setValue(seenKey, nextSeen);
866 }
867 if (batch.length === 0) continue;
868 saved += batch.length;
869 log.info(`"${rawInput}": saved ${batch.length} of ${records.length} job(s).`);
870 if (saved >= maxItems) stopped = true;
871 }
872 log.info(`Saved ${saved} job posting(s) across ${companies.length} company input(s).`);
873}
874
875await Actor.exit();
