1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { parseSiteUrl, apiBase, siteUrl, parsePostedOn, maybeWithin, mapJob, pickSplitFacet, robotsAllows, FormatError, PAGE_SIZE, OFFSET_CAP } from './workday.js';
4
5const JOB_EVENT = 'job';
6const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
7class InputError extends Error {}
8const HOST_GAP = 120;
9
10await Actor.init();
11try {
12 const input = (await Actor.getInput()) ?? {};
13 const {
14 startUrls = [], searchText = '', titleKeywords = [], excludeTitleKeywords = [], locationKeywords = [],
15 postedWithinDays = null, remoteOnly = false,
16 includeDetails = true, includeDescription = true, includeDescriptionHtml = false,
17 maxJobsPerSite = 0, maxJobs = 0, splitLargeSites = true, emitStatusRows = true,
18 proxyConfiguration = { useApifyProxy: false }, maxRetries = 3, detailConcurrency = 4,
19 } = input;
20
21 const raw = (Array.isArray(startUrls) ? startUrls : String(startUrls).split(/[\n,]/))
22 .map((c) => (typeof c === 'object' && c ? c.url : c)).map((c) => String(c ?? '').trim()).filter(Boolean);
23 if (!raw.length) throw new InputError('Add at least one Workday career-site URL, e.g. https://nvidia.wd5.myworkdayjobs.com/NVIDIAExternalCareerSite');
24 const arr = (v) => (Array.isArray(v) ? v : String(v ?? '').split(',')).map((x) => String(x).trim().toLowerCase()).filter(Boolean);
25 const inc = arr(titleKeywords); const exc = arr(excludeTitleKeywords); const locs = arr(locationKeywords);
26 const maxDays = postedWithinDays === null || postedWithinDays === '' || postedWithinDays === undefined ? null : Number(postedWithinDays);
27 if (maxDays !== null && (!Number.isFinite(maxDays) || maxDays < 0)) throw new InputError('"Posted within (days)" must be a number of days, e.g. 7');
28 const perSite = Number(maxJobsPerSite) > 0 ? Math.floor(Number(maxJobsPerSite)) : Infinity;
29
30 const caps = [maxJobs, input.maxItems, input.limit, input.maxResults, process.env.ACTOR_MAX_PAID_DATASET_ITEMS].map(Number).filter((n) => Number.isFinite(n) && n > 0).map(Math.floor);
31 const totalCap = caps.length ? Math.min(...caps) : Infinity;
32 const needDetails = !!includeDetails || locs.length > 0 || !!remoteOnly;
33
34 const targets = []; const bad = []; const seen = new Set();
35 for (const r of raw) {
36 try { const t = parseSiteUrl(r); t.input = r; if (seen.has(t.key + JSON.stringify(t.facets) + t.searchText)) continue; seen.add(t.key + JSON.stringify(t.facets) + t.searchText); targets.push(t); }
37 catch (e) { bad.push({ input: r, status: 'invalid_url', error: e.message }); log.warning(e.message); }
38 }
39
40 const proxy = proxyConfiguration?.useApifyProxy || proxyConfiguration?.proxyUrls?.length ? await Actor.createProxyConfiguration(proxyConfiguration) : null;
41 const charging = Actor.getChargingManager();
42 const stats = { requests: 0, retries: 0, failedRequests: 0 };
43 const nextSlot = new Map();
44
45 async function http(url, { method = 'GET', json, accept = 'application/json' } = {}) {
46 const host = new URL(url).hostname; let lastErr;
47 for (let attempt = 0; attempt <= maxRetries; attempt++) {
48 const slot = Math.max(Date.now(), nextSlot.get(host) ?? 0); nextSlot.set(host, slot + HOST_GAP);
49 if (slot > Date.now()) await sleep(slot - Date.now());
50 stats.requests++;
51 try {
52 const res = await gotScraping({ url, method, json, proxyUrl: proxy ? await proxy.newUrl() : undefined, timeout: { request: 45000 }, throwHttpErrors: false, retry: { limit: 0 }, followRedirect: true,
53 headers: { accept, ...(json ? { 'content-type': 'application/json' } : {}) }, headerGeneratorOptions: { browsers: ['chrome'], devices: ['desktop'] } });
54 const code = res.statusCode;
55 if (code === 404 || code === 410 || code === 422) return { notFound: true, code };
56 if (code === 429 || code >= 500 || code === 403) { lastErr = new Error(`HTTP ${code} from ${host}`); if (attempt < maxRetries) { stats.retries++; await sleep(Math.min(2000 * 2 ** attempt, 20000) + Math.random() * 500); continue; } break; }
57 if (code >= 400) return { notFound: true, code };
58 if (accept !== 'application/json') return { text: res.body };
59 if (!/^\s*[[{]/.test(res.body)) return { notFound: true, code, nonJson: true };
60 try { return { body: JSON.parse(res.body) }; } catch { throw new FormatError(`Invalid JSON from ${host}`); }
61 } catch (e) {
62 if (e instanceof FormatError) throw e;
63 lastErr = e; if (attempt < maxRetries) { stats.retries++; await sleep(Math.min(2000 * 2 ** attempt, 20000) + Math.random() * 500); }
64 }
65 }
66 stats.failedRequests++; throw lastErr;
67 }
68
69 const robotsCache = new Map();
70 async function allowed(t, path) {
71 if (!robotsCache.has(t.host)) {
72 let txt = '';
73 try { const r = await http(`https://${t.host}/robots.txt`, { accept: 'text/plain' }); txt = r.text ?? ''; } catch { txt = ''; }
74 robotsCache.set(t.host, txt);
75 }
76 return robotsAllows(robotsCache.get(t.host), path);
77 }
78
79 let pushed = 0; let chargeLimitHit = false;
80 const stop = () => chargeLimitHit || pushed >= totalCap;
81 async function push(items) {
82 if (!items.length || stop()) return 0;
83 if (pushed + items.length > totalCap) items = items.slice(0, totalCap - pushed);
84 if (charging) {
85 const can = charging.calculateMaxEventChargeCountWithinLimit(JOB_EVENT);
86 if (Number.isFinite(can) && can < items.length) { items = items.slice(0, Math.max(0, can)); chargeLimitHit = true; log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); }
87 }
88 if (!items.length) return 0;
89 const r = await Actor.pushData(items, JOB_EVENT);
90 if (r?.eventChargeLimitReached) chargeLimitHit = true;
91 pushed += items.length; return items.length;
92 }
93
94 const scrapedAt = new Date().toISOString();
95 const summaryRows = [...bad];
96 let formatErrors = 0; let networkErrors = 0;
97
98 for (const t of targets) {
99 if (stop()) {
100 summaryRows.push({ input: t.input, careerSiteUrl: siteUrl(t), status: chargeLimitHit ? 'skipped_charge_limit' : 'skipped_max_jobs', jobsReported: 0, jobsListed: 0, jobsMatched: 0, jobsSaved: 0, splitBy: [], error: null });
101 continue;
102 }
103 const row = { input: t.input, careerSiteUrl: siteUrl(t), status: 'ok', jobsReported: 0, jobsListed: 0, jobsMatched: 0, jobsSaved: 0, splitBy: [], error: null };
104 try {
105 const base = apiBase(t); const apiPath = new URL(base).pathname;
106 if (!(await allowed(t, `${apiPath}/jobs`)) || !(await allowed(t, `/${t.site}/`))) { row.status = 'robots_disallowed'; row.error = `${t.host} robots.txt disallows this career site; skipped.`; log.warning(row.error); summaryRows.push(row); continue; }
107 const text = [t.searchText, searchText].filter(Boolean).join(' ').trim();
108 const postings = new Map();
109 const listPage = async (facets, offset) => {
110 const r = await http(`${base}/jobs`, { method: 'POST', json: { appliedFacets: facets, limit: PAGE_SIZE, offset, searchText: text } });
111 if (r.notFound) return null;
112 const b = r.body; if (!b || !Array.isArray(b.jobPostings)) throw new FormatError('Unexpected job list format');
113 return b;
114 };
115
116 const siteCap = Math.min(perSite, totalCap - pushed);
117 const noFilters = maxDays === null && !inc.length && !exc.length && !locs.length && !remoteOnly;
118 const enough = () => noFilters && postings.size >= siteCap * 3;
119
120 const skipSplit = noFilters && siteCap * 3 < OFFSET_CAP;
121 const crawl = async (facets, depth) => {
122 const first = await listPage(facets, 0);
123 if (!first) return false;
124 const total = Number(first.total) || 0;
125 if (depth === 0) row.jobsReported = total;
126 if (total >= OFFSET_CAP && splitLargeSites && !skipSplit && depth < 3) {
127 const f = pickSplitFacet(first.facets, facets, total);
128 if (f) {
129 row.splitBy.push(f.param); log.info(`${t.tenant}/${t.site}: ${total}+ jobs, over Workday's 2,000 page limit; splitting by "${f.param}" (${f.values.length} groups)`);
130 for (const v of f.values) { if (enough()) break; await crawl({ ...facets, [f.param]: [v.id] }, depth + 1); }
131 return true;
132 }
133 }
134 const add = (list) => { for (const p of list) if (p?.externalPath && !postings.has(p.externalPath)) postings.set(p.externalPath, p); };
135 add(first.jobPostings);
136 const last = Math.min(total, OFFSET_CAP);
137 for (let off = PAGE_SIZE; off < last; off += PAGE_SIZE) {
138 if (enough()) break;
139 const b = await listPage(facets, off); if (!b || !b.jobPostings.length) break;
140 add(b.jobPostings);
141 }
142 return true;
143 };
144 const found = await crawl({ ...t.facets }, 0);
145 if (!found) { row.status = 'not_found'; row.error = `No Workday career site at ${siteUrl(t)}. Copy the URL from the company's jobs page address bar.`; log.warning(row.error); summaryRows.push(row); continue; }
146 row.jobsListed = postings.size;
147 if (row.jobsReported >= OFFSET_CAP && !row.splitBy.length && !skipSplit) log.warning(`${t.tenant}/${t.site}: Workday only pages through 2,000 jobs and no filter was available to split by; saved the first 2,000. Add filters (e.g. a location) to get the rest.`);
148
149
150 const cands = [];
151 for (const p of postings.values()) {
152 const title = String(p.title ?? '').toLowerCase();
153 if (inc.length && !inc.some((k) => title.includes(k))) continue;
154 if (exc.some((k) => title.includes(k))) continue;
155 if (!maybeWithin(parsePostedOn(p.postedOn), maxDays)) continue;
156 cands.push(p);
157 }
158 const out = []; let idx = 0;
159 const worker = async () => {
160 while (idx < cands.length && out.length < perSite && !stop()) {
161 const p = cands[idx++]; let detail = null;
162 if (needDetails) {
163 const r = await http(`${base}${p.externalPath}`).catch((e) => { networkErrors++; log.debug(e.message); return null; });
164 if (!r || r.notFound) continue;
165 detail = r.body;
166 }
167 let job; try { job = mapJob(t, p, detail, { includeHtml: includeDescriptionHtml, includeDescription: includeDescription && needDetails, scrapedAt }); } catch (e) { formatErrors++; continue; }
168 if (maxDays !== null) {
169 if (job.postedDate) { const age = (Date.now() - Date.parse(job.postedDate + 'T00:00:00Z')) / 864e5; if (age > maxDays + 1) continue; }
170 else if (job.postedDaysAgo !== null && (job.postedDaysAgo > maxDays || (job.postedDaysAgoIsMinimum && maxDays < 30))) continue;
171 }
172 if (remoteOnly && !job.remote) continue;
173 if (locs.length) { const hay = [...job.allLocations, job.location, job.country].filter(Boolean).join(' | ').toLowerCase(); if (!locs.some((k) => hay.includes(k))) continue; }
174 if (out.length < perSite) out.push(job);
175 }
176 };
177 await Promise.all(Array.from({ length: Math.max(1, Math.min(8, Number(detailConcurrency) || 4)) }, worker));
178 row.jobsMatched = out.length;
179 if (!out.length) {
180 const f = [searchText && `searchText="${searchText}"`, inc.length && `titleKeywords=${JSON.stringify(inc)}`, exc.length && `excludeTitleKeywords=${JSON.stringify(exc)}`,
181 locs.length && `locationKeywords=${JSON.stringify(locs)}`, maxDays !== null && `postedWithinDays=${maxDays}`, remoteOnly && 'remoteOnly=true'].filter(Boolean).join(', ');
182 row.reason = row.jobsListed ? `${row.jobsListed} jobs listed, 0 matched your filters (${f || 'none'}). Loosen or remove filters.` : `The career site has no open jobs${f ? ` for ${f}` : ''} right now.`;
183 }
184 for (let i = 0; i < out.length && !stop(); i += 200) row.jobsSaved += await push(out.slice(i, i + 200));
185 log.info(`${t.tenant}/${t.site}: ${row.jobsReported} jobs reported, ${row.jobsListed} listed, ${row.jobsSaved} saved`);
186 } catch (e) {
187 if (e instanceof FormatError) { formatErrors++; row.status = 'format_error'; } else { networkErrors++; row.status = 'error'; }
188 row.error = e.message; log.warning(`${t.input}: ${e.message}`);
189 }
190 summaryRows.push(row);
191 }
192
193 const okSites = summaryRows.filter((r) => r.status === 'ok').length;
194
195 const empty = summaryRows.filter((r) => !r.jobsSaved && !String(r.status).startsWith('skipped'));
196 if (emitStatusRows !== false) {
197 for (const r of empty) await Actor.pushData({ status: r.status === 'ok' ? 'no_results' : r.status, input: r.input, careerSiteUrl: r.careerSiteUrl ?? null,
198 jobsListed: r.jobsListed ?? 0, jobsMatched: r.jobsMatched ?? 0, reason: r.reason ?? r.error ?? null, scrapedAt });
199 }
200 const stoppedReason = chargeLimitHit ? 'Reached the maximum cost you set for this run.' : pushed >= totalCap ? `Reached the job limit (${totalCap}). Raise maxJobs (0 = no limit) for more.` : null;
201 const zeroResultsReason = pushed ? null : (empty.map((r) => `${r.input}: ${r.reason ?? r.error ?? r.status}`).join(' | ') || null);
202 const summary = { sitesRequested: raw.length, sitesOk: okSites, jobsSaved: pushed, maxJobs: Number.isFinite(totalCap) ? totalCap : 0, stoppedReason, zeroResultsReason, chargeLimitReached: chargeLimitHit, formatErrors, networkErrors, stats, sites: summaryRows, finishedAt: new Date().toISOString() };
203 if (pushed) summary.nextSteps = [`Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${Actor.getEnv().defaultDatasetId}"}`, 'Also covers Greenhouse, Lever and Ashby boards: swillz/career-site-jobs-scraper.'];
204 await Actor.setValue('OUTPUT', summary);
205 if (!okSites && summaryRows.some((r) => r.status === 'format_error')) throw new Error('Workday response format changed; no jobs could be parsed. The actor needs an update.');
206 if (!okSites && targets.length && summaryRows.filter((r) => r.status === 'error').length === targets.length) throw new Error(`All ${targets.length} career sites failed to load (network errors). Try again or enable a proxy.`);
207 if (!okSites) await Actor.exit('0 results: no Workday career sites found for the URLs you entered. Copy the URL of the company\'s myworkdayjobs.com jobs page (see the README: "How to find a company\'s Workday URL").');
208 else if (!pushed) await Actor.exit(`0 results: ${zeroResultsReason}`);
209 else await Actor.exit(`Saved ${pushed} jobs from ${okSites} of ${raw.length} career sites.${stoppedReason ? ` Stopped: ${stoppedReason}` : ''}${empty.length ? ` ${empty.length} site(s) returned no jobs (see the status rows).` : ''}`);
210} catch (e) {
211 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Input error: ${e.message}`); }
212 else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
213}
