StepStone.dk Scraper — Denmark Jobs, No Proxy Needed

/**

• stepstone-dk-scraper — Unfenced Group
• StepStone Denmark job board scraper
• Architecture: HTTP-only · SSR Stash JSON extraction · No proxy
• Platform: Jobindex CMS (jix) — same engine as jobindex.dk
• Recon (2026-05): No REST/GraphQL/XHR JSON API found. All search data
• is server-side rendered into window.Stash — pure json-html method. */

import { Actor, log } from 'apify'; import { gotScraping } from 'got-scraping'; import { writeHealthSignal, assertMinResults } from './health.js';

// Write OUTPUT KVS key — required for the Apify Output tab to populate async function writeOutputKey(totalJobs) { const env = Actor.getEnv(); const datasetUrl = https://api.apify.com/v2/datasets/${env.defaultDatasetId}/items?clean=true&format=json; const payload = JSON.stringify({ totalJobs, datasetUrl }); try { const ctrl = new AbortController(); const tid = setTimeout(() => ctrl.abort(), 5_000); await fetch( https://api.apify.com/v2/key-value-stores/${env.defaultKeyValueStoreId}/records/OUTPUT?token=${env.token}, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: payload, signal: ctrl.signal } ); clearTimeout(tid); log.info([OUTPUT] Written — totalJobs=${totalJobs}); } catch (e) { try { await Actor.setValue('OUTPUT', { totalJobs, datasetUrl }); } catch { log.warning([OUTPUT] write failed: ${e.message}); } } }

const BASE_URL = 'https://www.stepstone.dk'; const SEARCH_URL = ${BASE_URL}/jobsoegning/; const DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';

const RETRYABLE = new Set([429, 500, 502, 503, 504]);

// ── Canonical withRetry ──────────────────────────────────────────────────────── async function withRetry(fn, label, maxAttempts = 3) { for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { const result = await fn(); if (result && RETRYABLE.has(result.status)) { if (attempt >= maxAttempts) return result; const ra = result.headers?.['retry-after']; let delay = Math.min(Math.pow(2, attempt) * 1000, 15000); if (ra) delay = Math.max(delay, parseInt(ra, 10) * 1000 || delay); if (result.status === 429) delay = Math.max(delay, 5000); log.warning([${label}] HTTP ${result.status} — retry ${attempt}/${maxAttempts} in ${delay}ms); await sleep(delay); continue; } return result; } catch (err) { if (attempt >= maxAttempts) { log.error([${label}] all ${maxAttempts} attempts failed: ${err.message}); return null; } const ra = err?.response?.headers?.['retry-after']; let delay = Math.min(Math.pow(2, attempt) * 1000, 15000); if (ra) delay = Math.max(delay, parseInt(ra, 10) * 1000 || delay); if (err?.response?.status === 429) delay = Math.max(delay, 5000); log.warning([${label}] attempt ${attempt}/${maxAttempts}: ${err.message} — retry in ${delay}ms); await sleep(delay); } } return null; }

// ── Gaussian sleep (§2.16) ───────────────────────────────────────────────────── function gaussianRandom(mean, std) { const u1 = Math.max(Math.random(), 1e-10); const u2 = Math.max(Math.random(), 1e-10); return mean + std * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2); } function sleep(ms) { return new Promise(r => setTimeout(r, Math.max(0, Math.round(ms)))); } function gaussianSleep(mean, std) { return sleep(gaussianRandom(mean, std)); }

// ── Stash extractor ──────────────────────────────────────────────────────────── function extractStash(html) { const idx = html.indexOf('Stash = {'); if (idx < 0) return null; const start = idx + 'Stash = '.length; let depth = 0; let i = start; while (i < html.length) { if (html[i] === '{') depth++; else if (html[i] === '}') { depth--; if (depth === 0) break; } i++; } try { return JSON.parse(html.slice(start, i + 1)); } catch (e) { log.error('Stash parse failed: ' + e.message); return null; } }

// ── Map raw result to output item ───────────────────────────────────────────── // v2.0 new fields: applyDeadline, applyDeadlineAsap, streetAddress, fullAddress, // workplaceCompany, workplaceCompanyWebsite, logoUrl, companyId, isPremiumAd, isLocal function mapJob(raw, searchQuery) { const addr = raw.addresses?.[0] || {}; const company = raw.companytext || raw.company?.name || null;

// workplace_company = the actual hiring company when the listing is via a recruiter
const workplaceCo = raw.workplace_company || {};

// Prefer share_url (canonical), fall back to vis-job pattern
const jobUrl = raw.share_url ||
    (raw.tid ? `${BASE_URL}/vis-job/${raw.tid}` : null);

return {
    jobId:                   raw.tid || null,
    title:                   raw.headline || null,
    company:                 company,
    companyId:               raw.company?.id || null,
    companyWebsite:          workplaceCo.homeurl || raw.company?.homeurl || null,
    workplaceCompany:        workplaceCo.name || null,
    workplaceCompanyWebsite: workplaceCo.homeurl || null,
    location:                raw.area || null,
    city:                    addr.city || null,
    zipCode:                 addr.zipcode || null,
    streetAddress:           addr.line || null,
    fullAddress:             addr.simple_string || null,
    latitude:                addr.coordinates?.latitude || null,
    longitude:               addr.coordinates?.longitude || null,
    datePosted:              raw.firstdate || null,
    dateExpires:             raw.lastdate || null,
    applyDeadline:           raw.apply_deadline || null,
    applyDeadlineAsap:       raw.apply_deadline_asap === true ? true : null,
    applyUrl:                raw.apply_url || null,
    jobUrl:                  jobUrl,
    logoUrl:                 raw.listlogo_url || raw.company?.logo || null,
    remoteWork:              raw.home_workplace === true ? true : null,
    isPremiumAd:             raw.has_spo || false,
    isLocal:                 raw.is_local || false,
    isArchived:              raw.is_archived || false,
    source:                  'stepstone.dk',
    searchQuery:             searchQuery || null,
};

}

// ── Main ───────────────────────────────────────────────────────────────────── await Actor.init();

const input = await Actor.getInput() || {}; if (input && input.maxItems !== undefined && input.maxResults === undefined) input.maxResults = input.maxItems; const { searchQuery= '', location = '', area = '', categoryId = null, employmentType = null, workingHours = null, employmentPlace = null, daysOld = null, maxResults = 100, startUrls = [], deduplicate = true, } = input;

// Deadline guard (§12.9) // ACTOR_TIMEOUT_AT is the timeout duration in seconds (not an absolute timestamp). // Compute absolute deadline as start time + duration, then subtract 25s safety margin. const _timeoutSecs = parseInt(process.env.ACTOR_TIMEOUT_AT || '0', 10); const deadline = _timeoutSecs > 0 ? Date.now() + _timeoutSecs * 1000 - 25000 : Infinity; const isNearDeadline = () => Date.now() >= deadline;

const dataset = await Actor.openDataset(); const kvStore = await Actor.openKeyValueStore();

// Dedup KVS (REST-safe pattern) const DEDUP_STORE_NAME = 'stepstone-dk-job-dedup'; let dedupStore = null; if (deduplicate) { try { dedupStore = await Actor.openKeyValueStore(DEDUP_STORE_NAME); } catch (e) { log.warning('Dedup store unavailable: ' + e.message); } }

const seenIds = new Set(); let totalSaved = 0;

// ── HTTP client (2026-07-13): gotScraping over RESIDENTIAL proxy by default. // The 2026-07-04 direct-connection build failed QA: StepStone DK intermittently // blocks the shared Apify datacenter/direct egress IP, returning a non-200 that // left __fetchHealthy false and threw FAILED on 0 results (3 days -> flagged). // Residential rotation avoids the shared-IP block; user proxyConfiguration // overrides. Non-200 responses now retry across fresh IPs before giving up so a // single transient block never produces a hard 0. const proxyConfiguration = input?.proxyConfiguration ? await Actor.createProxyConfiguration(input.proxyConfiguration) : await Actor.createProxyConfiguration({ groups: ['RESIDENTIAL'], countryCode: 'DK' }); const httpGet = async (url, params = {}) => { const u = new URL(url); for (const [k, val] of Object.entries(params)) u.searchParams.set(k, val); const MAX = 3; let resp; for (let attempt = 1; attempt <= MAX; attempt++) { resp = await gotScraping({ url: u.href, proxyUrl: proxyConfiguration ? await proxyConfiguration.newUrl() : undefined, headerGeneratorOptions: { browsers: [{ name: 'chrome', minVersion: 120 }], devices: ['desktop'], operatingSystems: ['windows'], locales: ['da-DK'], }, timeout: { request: 30000 }, retry: { limit: 0 }, throwHttpErrors: false, }); if (resp.statusCode === 200) { globalThis.__fetchHealthy = true; break; } if (attempt < MAX) { const delay = 1000 * attempt; log.warning([httpGet] HTTP ${resp.statusCode} on ${u.pathname} — retry ${attempt}/${MAX} in ${delay}ms (fresh IP)); await new Promise((r) => setTimeout(r, delay)); } } return { status: resp.statusCode, data: resp.body }; };

// ── startUrls mode ──────────────────────────────────────────────────────────── if (startUrls && startUrls.length > 0) { log.info(startUrls mode: ${startUrls.length} URL(s));

for (const startUrlObj of startUrls) {
    if (isNearDeadline() || totalSaved >= maxResults) break;

    const rawUrl = typeof startUrlObj === 'string' ? startUrlObj : startUrlObj.url;
    if (!rawUrl) continue;

    const parsedUrl = new URL(rawUrl);
    const urlParams = Object.fromEntries(parsedUrl.searchParams.entries());
    const baseSearchUrl = `${parsedUrl.origin}${parsedUrl.pathname}`;

    let page = parseInt(urlParams.page || '1', 10);
    let totalPages = 1;

    do {
        if (isNearDeadline() || totalSaved >= maxResults) break;

        const params = { ...urlParams, page };
        const resp = await withRetry(
            () => httpGet(baseSearchUrl, params),
            `startUrl-page-${page}`
        );

        if (!resp || resp.status !== 200) {
            log.warning(`startUrl page ${page} returned ${resp?.status}`);
            break;
        }

        globalThis.__fetchHealthy = true;   // clean fetch confirmed
        const stash = extractStash(resp.data);
        if (!stash) { log.warning('No Stash on startUrl page ' + page); break; }

        const store = stash['jobsearch/result_app']?.storeData;
        const searchResp = store?.searchResponse;
        if (!searchResp?.results?.length) { log.warning('No results on startUrl page ' + page); break; }

        totalPages = searchResp.total_pages || 1;
        const sq = Object.entries(store?.searchQuery || {}).map(([k, v]) => `${k}=${v}`).join('&');

        for (const raw of searchResp.results) {
            if (totalSaved >= maxResults) break;
            const id = raw.tid;
            if (!id) continue;
            if (seenIds.has(id)) continue;

            if (dedupStore) {
                const existing = await dedupStore.getValue(id).catch(() => null);
                if (existing) { log.debug(`[dedup] skip ${id}`); continue; }
            }

            seenIds.add(id);
            const item = mapJob(raw, sq);
            await dataset.pushData(item);
            if (dedupStore) await dedupStore.setValue(id, { seenAt: new Date().toISOString() }).catch(() => {});
            totalSaved++;
        }

        log.info(`startUrl page ${page}/${totalPages} — ${totalSaved} saved`);
        page++;
        if (page <= totalPages && totalSaved < maxResults) await gaussianSleep(300, 80);
    } while (page <= totalPages && totalSaved < maxResults && !isNearDeadline());
}

} else { // ── Standard search mode ────────────────────────────────────────────────── const params = {}; if (searchQuery) params.q = searchQuery; if (area) params.area = area; else if (location) params.area = location; if (categoryId) params.subid = categoryId; if (employmentType) params.employment_type = employmentType; if (workingHours) params.workinghours_type = workingHours; if (employmentPlace) params.employment_place = employmentPlace; if (daysOld !== null && daysOld !== undefined) { if (daysOld === 0) params.jobage = '0'; else if (daysOld > 0) params.jobage = String(daysOld); }

const displayQuery = searchQuery || '(alle job)';
log.info(`Search: "${displayQuery}" | area: ${area || location || 'all'} | maxResults: ${maxResults}`);

let page = 1;
let totalPages = 1;
let consecutiveEmpty = 0;

do {
    if (isNearDeadline() || totalSaved >= maxResults) break;

    const resp = await withRetry(
        () => httpGet(SEARCH_URL, { ...params, page }),
        `search-page-${page}`
    );

    if (!resp || resp.status !== 200) {
        log.warning(`Page ${page} returned ${resp?.status}`);
        break;
    }

    const stash = extractStash(resp.data);
    if (!stash) { log.warning('No Stash on page ' + page); break; }

    const store = stash['jobsearch/result_app']?.storeData;
    const searchResp = store?.searchResponse;
    if (!searchResp) { log.warning('No searchResponse on page ' + page); break; }

    const results = searchResp.results || [];
    totalPages = searchResp.total_pages || 1;

    if (page === 1) {
        log.info(`Found ${searchResp.hitcount} total jobs, ${totalPages} pages`);
    }

    if (results.length === 0) {
        consecutiveEmpty++;
        if (consecutiveEmpty >= 3) { log.warning('3 consecutive empty pages — stopping'); break; }
    } else {
        consecutiveEmpty = 0;
    }

    for (const raw of results) {
        if (totalSaved >= maxResults) break;
        const id = raw.tid;
        if (!id) continue;
        if (seenIds.has(id)) continue;

        if (dedupStore) {
            const existing = await dedupStore.getValue(id).catch(() => null);
            if (existing) { log.debug(`[dedup] skip ${id}`); continue; }
        }

        seenIds.add(id);
        const item = mapJob(raw, searchQuery);
        await dataset.pushData(item);
        if (dedupStore) await dedupStore.setValue(id, { seenAt: new Date().toISOString() }).catch(() => {});
        totalSaved++;
    }

    log.info(`Page ${page}/${totalPages} — ${results.length} results — ${totalSaved} saved total`);
    page++;

    if (page <= totalPages && totalSaved < maxResults && !isNearDeadline()) {
        await gaussianSleep(300, 80);
    }

} while (page <= totalPages && totalSaved < maxResults && !isNearDeadline());

}

log.info(Done — ${totalSaved} jobs saved);

// ── Health signal ────────────────────────────────────────────────────────────── const nearDeadline = isNearDeadline(); try { assertMinResults(totalSaved, 1, nearDeadline); } catch (e) { await writeHealthSignal(kvStore, 'WARN', e.message); throw e; }

// ── OUTPUT KVS key (§3.3) ────────────────────────────────────────────────────── await writeOutputKey(totalSaved); await writeHealthSignal(kvStore, 'OK', ${totalSaved} jobs saved);

await Actor.exit();