.actor

{
 "actorSpecification": 1,
 "name": "default-output",
 "title": "Live output",
 "description": "Auto format for dataset items",
 "version": "0.0.1",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "",
 "description": "",
 "views": {}
 }
 }
}

.actor/actor.json

import { Actor } from 'apify';
import { KeyValueStore, CheerioCrawler } from 'crawlee';
import { handleSearch } from './routes.js';

await Actor.init();

const input = await KeyValueStore.getInput();
const {
 startUrls = [],
 proxyConfiguration = {
 useApifyProxy: true,
 },
} = input;

const proxyConfig = await Actor.createProxyConfiguration(proxyConfiguration);

const crawler = new CheerioCrawler({
 proxyConfiguration: proxyConfig,
 async requestHandler(context) {
 return handleSearch(context, input);
 }
});

await crawler.run(startUrls);

// Exit successfully
await Actor.exit();

src/main.js

import { Dataset, log } from 'crawlee';

export const handleSearch = async (context, input) => {
 const { request, $, crawler } = context;
 const { url, userData } = request;
 const { page = 1 } = userData;
 const { resultsLimit = 0 } = input;

 const items = Array.from($('.result-item')).map((x) => {
 const source_url = $('a', x).attr('href');
 const job_title = $('.result-title', x).text().trim();
 const loc = Array.from($('.location-span strong', x)).map((a) => $(a).text().trim());
 const advertiser_name = loc?.[0];
 const advertiser_location = loc?.[1];
 const full_text = $('p.job-description', x).text().trim();
 // const = $('', x).text().trim();
 return {
 source_url,
 job_title,
 advertiser_name,
 advertiser_location,
 full_text,
 searchUrl: url
 };
 });

 if (!items?.length) {
 log.info(`[NO-DATA]: no jobs at ${url}`);
 return;
 }

 const itemsCounter = (page - 1) * 10;
 const resultsCounter = itemsCounter + items.length;

 await Dataset.pushData(items.slice(0, resultsLimit &amp;&amp; resultsCounter > resultsLimit ? resultsLimit - itemsCounter : undefined));

 const counter = parseInt($('span.amount-of-jobs.desktop-block > strong').text().replace('.', '')) || 0;

 if (!(resultsCounter >= counter) &amp;&amp; resultsLimit &amp;&amp; !(resultsCounter >= resultsLimit)) {
 const pagedUrl = new URL(url);
 pagedUrl.searchParams.set('pageNumber', page + 1);
 await crawler.requestQueue.addRequest({
 url: pagedUrl.toString(),
 userData: {
 page: page + 1
 }
 });
 } else {
 log.info(`[DONE]: ${resultsCounter} job(s) out of ${counter} at ${url}`);
 }
 }

src/routes.js

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
storage


.gitignore

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 &amp;&amp; npm install --omit=dev --omit=optional \
 &amp;&amp; echo "Installed NPM packages:" \
 &amp;&amp; (npm list --omit=dev --all || true) \
 &amp;&amp; echo "Node.js version:" \
 &amp;&amp; node --version \
 &amp;&amp; echo "NPM version:" \
 &amp;&amp; npm --version \
 &amp;&amp; rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent

Dockerfile

{
 "title": "Input schema",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "startUrls": {
 "title": "Add Vdab.be job categories URLs you want to scrape ",
 "type": "array",
 "description": "Add one or more job categories with optional sorting and filtering",
 "editor": "stringList",
 "placeholderValue": "URL",
 "prefill": ["https://www.vdab.be/vindeenjob/jobs/financieel?sort=1"],
 "patternValue": "https:\\/\\/(www\\.)?vdab\\.be\\/.+",
 "uniqueItems": true
 },
 "resultsLimit": {
 "title": "Max results",
 "type": "integer",
 "description": "How many jobs you want to scrape from each category URL",
 "editor": "number",
 "unit": "per page",
 "default": 100
 },
 "proxyConfiguration": {
 "title": "Proxy configuration",
 "type": "object",
 "description": "A proxy server is required to run this actor!",
 "prefill": {
 "useApifyProxy": true
 },
 "editor": "proxy",
 "sectionCaption": "Proxy configuration",
 "sectionDescription": "Select your proxy here."
 }
 }
}

INPUT_SCHEMA.json

# Vdab.be jobs scraper

Scrape jobs by category URL, sorting and filtering will be as in URL parameters.

## Input example
```
{
 "resultsLimit": 30,
 "startUrls": [
 "https://www.vdab.be/vindeenjob/jobs/financieel?sort=1"
 ]
}
```

## Output example
```
{
 "source_url": "https://www.vdab.be/vindeenjob/vacatures/65511481/accountant-mooi-pakket?jobdomein=JOBCAT06&amp;source=jobdomeinpagina&amp;sort=standaard&amp;sinds=9000&amp;limit=15&amp;expand=FILTERS&amp;medium=verbolia",
 "job_title": "Accountant - mooi pakket",
 "advertiser_name": "Le Grand &amp; Associates",
 "advertiser_location": "ANTWERPEN",
 "full_text": "Takenpakket\n\nJe staat in voor een correcte verwerking van de leveranciersfacturen van diverse entiteiten.Je werkt nauw samen met verschillende departementen.Verder neem je de credit collection voor jouw rekening.Ook sta je in voor de reconciliatie...",
 "searchUrl": "https://www.vdab.be/vindeenjob/jobs/financieel?sort=1"
}
```

README.md

{
 "name": "crawlee-cheerio-javascript",
 "version": "0.0.1",
 "type": "module",
 "description": "This is a boilerplate of an Apify actor.",
 "engines": {
 "node": ">=16.0.0"
 },
 "dependencies": {
 "apify": "^3.0.0",
 "crawlee": "^3.0.0"
 },
 "devDependencies": {
 "@apify/eslint-config": "^0.3.1",
 "eslint": "^8.20.0"
 },
 "scripts": {
 "start": "node src/main.js",
 "lint": "eslint ./src --ext .js,.jsx",
 "lint:fix": "eslint ./src --ext .js,.jsx --fix",
 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" &amp;&amp; exit 1"
 },
 "author": "It's not you it's me",
 "license": "ISC"
}

package.json

default-output

Live output

import { Actor } from 'apify';
import { KeyValueStore, CheerioCrawler } from 'crawlee';
import { handleSearch } from './routes.js';

await Actor.init();

const input = await KeyValueStore.getInput();
const {
    startUrls = [],
    proxyConfiguration = {
        useApifyProxy: true,
    },
} = input;

const proxyConfig = await Actor.createProxyConfiguration(proxyConfiguration);

const crawler = new CheerioCrawler({
    proxyConfiguration: proxyConfig,
    async requestHandler(context) {
        return handleSearch(context, input);
    }
});

await crawler.run(startUrls);

// Exit successfully
await Actor.exit();


import { Dataset, log } from 'crawlee';

export const handleSearch = async (context, input) => {
    const { request, $, crawler } = context;
    const { url, userData } = request;
    const { page = 1 } = userData;
    const { resultsLimit = 0 } = input;

    const items = Array.from($('.result-item')).map((x) => {
        const source_url = $('a', x).attr('href');
        const job_title = $('.result-title', x).text().trim();
        const loc = Array.from($('.location-span strong', x)).map((a) => $(a).text().trim());
        const advertiser_name = loc?.[0];
        const advertiser_location = loc?.[1];
        const full_text = $('p.job-description', x).text().trim();
        // const = $('', x).text().trim();
        return {
            source_url,
            job_title,
            advertiser_name,
            advertiser_location,
            full_text,
            searchUrl: url
        };
    });

    if (!items?.length) {
        log.info(`[NO-DATA]: no jobs at ${url}`);
        return;
    }

    const itemsCounter = (page - 1) * 10;
    const resultsCounter = itemsCounter + items.length;

    await Dataset.pushData(items.slice(0, resultsLimit && resultsCounter > resultsLimit ? resultsLimit - itemsCounter : undefined));

    const counter = parseInt($('span.amount-of-jobs.desktop-block > strong').text().replace('.', '')) || 0;

    if (!(resultsCounter >= counter) && resultsLimit && !(resultsCounter >= resultsLimit)) {
        const pagedUrl = new URL(url);
        pagedUrl.searchParams.set('pageNumber', page + 1);
        await crawler.requestQueue.addRequest({
            url: pagedUrl.toString(),
            userData: {
                page: page + 1
            }
        });
    } else {
        log.info(`[DONE]: ${resultsCounter} job(s) out of ${counter} at ${url}`);
    }
 }

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


A proxy server is required to run this actor!

Proxy configuration

How many jobs you want to scrape from each category URL

Max results

Add one or more job categories with optional sorting and filtering

Add Vdab.be job categories URLs you want to scrape 

Input schema

# Vdab.be jobs scraper

Scrape jobs by category URL, sorting and filtering will be as in URL parameters.

## Input example
```
{
  "resultsLimit": 30,
  "startUrls": [
    "https://www.vdab.be/vindeenjob/jobs/financieel?sort=1"
  ]
}
```

## Output example
```
{
  "source_url": "https://www.vdab.be/vindeenjob/vacatures/65511481/accountant-mooi-pakket?jobdomein=JOBCAT06&source=jobdomeinpagina&sort=standaard&sinds=9000&limit=15&expand=FILTERS&medium=verbolia",
  "job_title": "Accountant - mooi pakket",
  "advertiser_name": "Le Grand & Associates",
  "advertiser_location": "ANTWERPEN",
  "full_text": "Takenpakket\n\nJe staat in voor een correcte verwerking van de leveranciersfacturen van diverse entiteiten.Je werkt nauw samen met verschillende departementen.Verder neem je de credit collection voor jouw rekening.Ook sta je in voor de reconciliatie...",
  "searchUrl": "https://www.vdab.be/vindeenjob/jobs/financieel?sort=1"
}
```

crawlee-cheerio-javascript

Pick up any job categories from Belgium national job board https://www.vdab.be/jobs then choose sorting or filtering if you want and get list results for around 1 cent per 100 jobs!

Vdab-be jobs

Input example

Output example

Industries

Where next?