1
2
3
4
5
6
7
8
9
10
11
12
13import { createHash } from 'node:crypto';
14import { Actor, log } from 'apify';
15
16const USER_AGENT = 'foundry-actor/0.1 (Apify Actor; polite client; +https://apify.com/tabulary)';
17const MIN_INTERVAL_MS = 1000;
18const lastRequestAtByHost = new Map();
19
20
21
22async function politeGet(url, attempt = 1) {
23 const host = new URL(url).host;
24 const wait = (lastRequestAtByHost.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
25 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
26 lastRequestAtByHost.set(host, Date.now());
27
28 const res = await fetch(url, { headers: { 'user-agent': USER_AGENT, accept: 'application/json' } });
29 if (res.status === 429 || res.status >= 500) {
30 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
31 await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
32 return politeGet(url, attempt + 1);
33 }
34 let body = null;
35 try {
36 body = await res.json();
37 } catch {
38 body = null;
39 }
40 return { status: res.status, body };
41}
42
43
44
45
46function stripHtml(html) {
47 if (!html) return null;
48 let text = String(html)
49 .replace(/</g, '<')
50 .replace(/>/g, '>')
51 .replace(/"/g, '"')
52 .replace(/'/g, "'")
53 .replace(/ /g, ' ');
54 text = text
55 .replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, ' ')
56 .replace(/<[^>]+>/g, ' ')
57 .replace(/&/g, '&')
58 .replace(/ /g, ' ')
59 .replace(/\s+/g, ' ')
60 .trim();
61 return text || null;
62}
63
64const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
65function maskEmails(text) {
66 return text ? text.replace(EMAIL_RE, '[email removed]') : text;
67}
68
69function normalizeEmploymentType(raw) {
70 if (!raw) return null;
71 const s = String(raw).toLowerCase();
72 if (s.includes('full')) return 'full_time';
73 if (s.includes('part')) return 'part_time';
74 if (s.includes('intern')) return 'internship';
75 if (s.includes('contract') || s.includes('freelance')) return 'contract';
76 if (s.includes('temp')) return 'temporary';
77 return 'other';
78}
79
80function toIso(value) {
81 if (!value) return null;
82 const d = new Date(value);
83 return Number.isNaN(d.getTime()) ? null : d.toISOString();
84}
85
86
87
88
89async function fetchGreenhouse(slug) {
90 const { status, body } = await politeGet(
91 `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(slug)}/jobs?content=true`,
92 );
93 if (status === 404) return { detected: false };
94 if (status !== 200 || !body) throw new Error(`Greenhouse: unexpected response for board "${slug}"`);
95 const records = (body.jobs ?? []).map((j) => {
96 const location = j.location?.name ?? null;
97 return {
98 company: slug,
99 ats: 'greenhouse',
100 job_id: String(j.id),
101 title: j.title ?? null,
102 department: j.departments?.[0]?.name ?? null,
103 location,
104 remote: location ? /remote/i.test(location) : null,
105 employment_type: null,
106 posted_at: toIso(j.first_published),
107 updated_at: toIso(j.updated_at),
108 url: j.absolute_url ?? null,
109 apply_url: j.absolute_url ?? null,
110 description: stripHtml(j.content),
111 };
112 });
113 return { detected: true, records };
114}
115
116async function fetchLever(slug) {
117 const { status, body } = await politeGet(
118 `https://api.lever.co/v0/postings/${encodeURIComponent(slug)}?mode=json`,
119 );
120 if (status === 404) return { detected: false };
121 if (status !== 200 || !Array.isArray(body)) throw new Error(`Lever: unexpected response for board "${slug}"`);
122 const records = body.map((j) => {
123 const workplaceType = (j.workplaceType ?? '').toLowerCase();
124 return {
125 company: slug,
126 ats: 'lever',
127 job_id: String(j.id),
128 title: j.text ?? null,
129 department: j.categories?.department ?? j.categories?.team ?? null,
130 location: j.categories?.location ?? null,
131 remote: workplaceType ? workplaceType === 'remote' : null,
132 employment_type: normalizeEmploymentType(j.categories?.commitment),
133 posted_at: toIso(j.createdAt),
134 updated_at: null,
135 url: j.hostedUrl ?? null,
136 apply_url: j.applyUrl ?? j.hostedUrl ?? null,
137 description: j.descriptionPlain?.trim() || stripHtml(j.description),
138 };
139 });
140 return { detected: true, records };
141}
142
143async function fetchAshby(slug) {
144 const { status, body } = await politeGet(
145 `https://api.ashbyhq.com/posting-api/job-board/${encodeURIComponent(slug)}`,
146 );
147 if (status === 404) return { detected: false };
148 if (status !== 200 || !body) throw new Error(`Ashby: unexpected response for board "${slug}"`);
149 const records = (body.jobs ?? []).map((j) => ({
150 company: slug,
151 ats: 'ashby',
152 job_id: String(j.id),
153 title: j.title ?? null,
154 department: j.department ?? j.team ?? null,
155 location: j.location ?? null,
156 remote: typeof j.isRemote === 'boolean' ? j.isRemote : null,
157 employment_type: normalizeEmploymentType(j.employmentType),
158 posted_at: toIso(j.publishedAt),
159 updated_at: null,
160 url: j.jobUrl ?? null,
161 apply_url: j.applyUrl ?? j.jobUrl ?? null,
162 description: j.descriptionPlain?.trim() || stripHtml(j.descriptionHtml),
163 }));
164 return { detected: true, records };
165}
166
167async function fetchWorkable(slug) {
168 const { status, body } = await politeGet(
169 `https://apply.workable.com/api/v1/widget/accounts/${encodeURIComponent(slug)}?details=true`,
170 );
171 if (status === 404) return { detected: false };
172 if (status !== 200 || !body) throw new Error(`Workable: unexpected response for account "${slug}"`);
173 const records = (body.jobs ?? []).map((j) => {
174 const location = [j.city, j.state, j.country].filter(Boolean).join(', ') || null;
175 return {
176 company: slug,
177 ats: 'workable',
178 job_id: j.shortcode ?? String(j.id ?? j.title),
179 title: j.title ?? null,
180 department: j.department ?? null,
181 location,
182 remote: typeof j.telecommuting === 'boolean' ? j.telecommuting : null,
183 employment_type: normalizeEmploymentType(j.employment_type),
184 posted_at: toIso(j.published_on),
185 updated_at: null,
186 url: j.url ?? j.shortlink ?? null,
187 apply_url: j.application_url ?? null,
188 description: stripHtml(j.description),
189 };
190 });
191 return { detected: true, records };
192}
193
194const DETECTORS = [
195 ['greenhouse', fetchGreenhouse],
196 ['lever', fetchLever],
197 ['ashby', fetchAshby],
198 ['workable', fetchWorkable],
199];
200
201
202
203
204function hostToAts(host) {
205 if (host.endsWith('greenhouse.io')) return 'greenhouse';
206 if (host === 'jobs.lever.co') return 'lever';
207 if (host === 'jobs.ashbyhq.com') return 'ashby';
208 if (host === 'apply.workable.com' || host.endsWith('.workable.com')) return 'workable';
209 return null;
210}
211
212function parseCompanyInput(raw) {
213 const trimmed = raw.trim();
214 if (!/^https?:\/\//i.test(trimmed)) return { slug: trimmed, ats: null };
215 let url;
216 try {
217 url = new URL(trimmed);
218 } catch {
219 return { slug: trimmed, ats: null };
220 }
221 const ats = hostToAts(url.host);
222 if (ats === 'workable' && url.host !== 'apply.workable.com') {
223 return { slug: url.host.split('.')[0], ats };
224 }
225 const firstSegment = url.pathname.split('/').filter(Boolean)[0];
226 return { slug: firstSegment ?? url.host, ats };
227}
228
229async function fetchCompanyJobs(rawInput) {
230 const { slug, ats } = parseCompanyInput(rawInput);
231 if (!slug) throw new Error(`Could not read a board slug from "${rawInput}".`);
232 const order = ats ? [[ats, DETECTORS.find(([name]) => name === ats)[1]], ...DETECTORS.filter(([name]) => name !== ats)] : DETECTORS;
233 for (const [name, fetcher] of order) {
234 const result = await fetcher(slug);
235 if (result.detected) return { ats: name, records: result.records };
236 }
237 throw new Error(`"${rawInput}" was not found on Greenhouse, Lever, Ashby or Workable.`);
238}
239
240
241
242function remainingBudget() {
243 try {
244 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
245 return Number.isFinite(n) ? n : Infinity;
246 } catch {
247 return Infinity;
248 }
249}
250
251await Actor.init();
252
253const input = (await Actor.getInput()) ?? {};
254const companies = Array.isArray(input.companies) ? input.companies.map((c) => String(c).trim()).filter(Boolean) : [];
255const onlyNewSinceLastRun = input.onlyNewSinceLastRun ?? false;
256const splitList = (v, name) => {
257 if (v === undefined || v === null) return [];
258 if (typeof v !== 'string') throw new Error(`Input "${name}" must be a string of comma-separated words.`);
259 return v.split(',').map((x) => x.trim().toLowerCase()).filter(Boolean);
260};
261const titleKeywords = splitList(input.titleKeywords, 'titleKeywords');
262const locationKeywords = splitList(input.locationKeywords, 'locationKeywords');
263const remoteOnly = input.remoteOnly === true;
264const filtering = titleKeywords.length > 0 || locationKeywords.length > 0 || remoteOnly;
265const matches = (r) => (!titleKeywords.length || titleKeywords.some((k) => (r.title ?? '').toLowerCase().includes(k)))
266 && (!locationKeywords.length || locationKeywords.some((k) => (r.location ?? '').toLowerCase().includes(k)))
267 && (!remoteOnly || r.remote === true);
268if (companies.length === 0) throw new Error('Input "companies" must be a non-empty list of board slugs or careers-page URLs.');
269
270const stateStore = onlyNewSinceLastRun ? await Actor.openKeyValueStore('multi-ats-job-postings-seen') : null;
271
272let saved = 0;
273let skippedBudget = false;
274for (const rawInput of companies) {
275 if (skippedBudget) break;
276 let matchedAts;
277 let records;
278 try {
279 ({ ats: matchedAts, records } = await fetchCompanyJobs(rawInput));
280 } catch (err) {
281 log.error(`Skipping "${rawInput}": ${err.message}`);
282 continue;
283 }
284 const allRecords = records;
285 if (records.length === 0) {
286 log.info(`"${rawInput}": matched ${matchedAts} but has 0 open jobs right now. A bare slug is checked against four ATSs in a fixed order and a different company can own the same slug on another one; pass the full careers-page URL to be unambiguous.`);
287 }
288
289 records = records.map((r) => ({ ...r, description: maskEmails(r.description) }));
290 if (filtering) records = records.filter(matches);
291
292
293
294
295 const memoryId = filtering
296 ? `${rawInput.toLowerCase()}|${JSON.stringify({ titleKeywords: [...titleKeywords].sort(), locationKeywords: [...locationKeywords].sort(), remoteOnly })}`
297 : rawInput.toLowerCase();
298 const seenKey = onlyNewSinceLastRun
299 ? `seen-${createHash('sha256').update(memoryId).digest('hex').slice(0, 40)}`
300 : null;
301 const seen = onlyNewSinceLastRun ? ((await stateStore.getValue(seenKey)) ?? {}) : {};
302 const version = (r) => r.updated_at ?? r.posted_at ?? '1';
303 if (onlyNewSinceLastRun) records = records.filter((r) => seen[r.job_id] !== version(r));
304
305 const budget = remainingBudget();
306 const batch = records.slice(0, budget);
307 if (records.length > 0 && batch.length === 0) {
308 log.info('Reached the maximum charge set for this run; stopping.');
309 skippedBudget = true;
310 break;
311 }
312 if (batch.length > 0) await Actor.pushData(batch);
313
314 if (onlyNewSinceLastRun) {
315
316
317 const savedIds = new Set(batch.map((r) => r.job_id));
318 const nextSeen = {};
319 for (const r of allRecords) {
320 if (seen[r.job_id] === version(r) || savedIds.has(r.job_id)) nextSeen[r.job_id] = version(r);
321 }
322 await stateStore.setValue(seenKey, nextSeen);
323 }
324 if (batch.length === 0) continue;
325 saved += batch.length;
326 log.info(`"${rawInput}": saved ${batch.length} of ${records.length} job(s).`);
327}
328
329log.info(`Saved ${saved} job posting(s) across ${companies.length} company input(s).`);
330await Actor.exit();
