# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python-selenium:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

Scrapes Naver Blog using Selenium. Accept keywords to search for requested blogs.

naver-blog-scraper

Naver Blog Dataset

Naver Blog Scraper

.actor/actor.json

Insert your keywords below. The scraper will search for the keywords and return results.

End Date

Start Date

Keywords

Maximum depth

Maximum pages per keyword

Naver Blog Scraper Input

.actor/input_schema.json

src/__init__.py

import asyncio
import logging

from apify.log import ActorLogFormatter

from .main import main

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.INFO)
apify_client_logger.addHandler(handler)

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

asyncio.run(main())


src/__main__.py

from urllib.parse import urljoin, quote

from apify import Actor
from selenium import webdriver
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from lxml import html
import lxml.etree as etree
import datetime
import re
import math

# To run this Actor locally, you need to have the Selenium Chromedriver installed.
# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/
# When running on the Apify platform, it is already included in the Actor's Docker image.


def handle_search_result(dom):
    res = []
    ts = dom.xpath('//span[@class="title"]')
    for i in range(0, len(ts)):
        res.append({})
    
    for i, t in enumerate(ts):
        title = (t.text if t.text is not None else '') + ''.join(etree.tostring(e, encoding='utf-8').decode('utf-8') for e in t)
        res[i]["title"] = title
    
    urls = dom.xpath('//*[@id="content"]/section/div[2]//a[@class="desc_inner"]/@href')
    for i, url in enumerate(urls):
        res[i]["url"] = str(url)

    descs = dom.xpath('//a[@ng-bind-html="post.contents"]')
    for i, desc in enumerate(descs):
        description = (desc.text if desc.text is not None else '') + ''.join(etree.tostring(e, encoding='utf-8').decode('utf-8') for e in desc)
        res[i]["short_description"] = description
    
    a = dom.xpath('//em[@class="name_author"]')
    for i, author in enumerate(a):
        res[i]["author"] = author.text
    
    ds = dom.xpath('//span[@class="date"]')
    for i, date in enumerate(ds):
        d = date.text.split('.')
        year = int(d[0].strip())
        month = int(d[1].strip())
        day = int(d[2].strip())

        res[i]['pub_date'] = datetime.datetime(year, month, day)

    return res

def handle_blog_page(dom):
    unwanted = dom.xpath('//script|//style')
    for u in unwanted:
        u.drop_tree()
    
    full_text = dom.xpath('normalize-space(string(//div[@id="postListBody"]))')
    full_text = str(full_text)

    m = re.search(r'댓글\s+([0-9\,]+)', full_text) # comments
    if m is not None:
        num_of_comments = int(m.group(1).replace(',', ''))
    else:
        num_of_comments = 0

    m = re.search(r'공감\s+([0-9\,]+)', full_text) # likes
    if m is not None:
        num_of_likes = int(m.group(1).replace(',', ''))
    else:
        num_of_likes = 0

    return num_of_likes, num_of_comments, full_text

async def main():
    async with Actor:
        # Read the Actor input
        actor_input = await Actor.get_input() or {}
        keywords = actor_input.get('keywords', ["mcdonalds"])
        since = actor_input.get('date_start', '2023-01-01')
        until = actor_input.get('date_end', '2023-06-30')
        max_depth = actor_input.get('max_depth', 1)
        max_pages = actor_input.get('max_pages', 10)

        if not keywords:
            Actor.log.info('No keywords in actor input, exiting...')
            await Actor.exit()

        # Enqueue the starting URLs in the default request queue
        default_queue = await Actor.open_request_queue()
        for keyword in keywords:
            url = 'https://section.blog.naver.com/Search/Post.naver?pageNo=1&rangeType=PERIOD&orderBy=sim&startDate={}&endDate={}&keyword={}'.format(since, until, quote(keyword))
            Actor.log.info(f'Enqueuing {url} ...')
            await default_queue.add_request({ 'url': url, 'userData': { 'depth': 0, 'keyword': keyword, 'since': since, 'until': until, 'max_pages': max_pages }})

        # Launch a new Selenium Chrome WebDriver
        Actor.log.info('Launching Chrome WebDriver...')
        chrome_options = ChromeOptions()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--no-sandbox')
        chrome_options.add_argument('--disable-dev-shm-usage')
        driver = webdriver.Chrome(options=chrome_options)

        # Process the requests in the queue one by one
        while request := await default_queue.fetch_next_request():
            url = request['url']
            depth = request['userData']['depth']
            keyword = request['userData']['keyword']
            since = request['userData']['since']
            until = request['userData']['until']
            max_pages = request['userData']['max_pages']

            Actor.log.info(f'Scraping {url} ...')

            try:
                # Open the URL in the Selenium WebDriver
                driver.get(url)
                WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'search_number')))
                dom = html.fromstring(driver.page_source)

                if depth == 0:
                    # calculate how many pages are there
                    search_result = dom.xpath('//em[@class="search_number"]/text()')[0]
                    match = re.search(r'[0-9\,]+', search_result)
                    if match is not None:
                        num_of_result = int(match.group(0).replace(',', ''))
                        num_pages = math.ceil(num_of_result / 7)
                    else:
                        Actor.log.info('Unexpected error. Cannot detect number of pages, exiting...')
                        await Actor.exit()                 
                    
                    num_pages = min(num_pages, max_pages)
                    
                    for i in range(2, num_pages + 1):
                        link_url = 'https://section.blog.naver.com/Search/Post.naver?pageNo={}&rangeType=PERIOD&orderBy=sim&startDate={}&endDate={}&keyword={}'.format(i, since, until, quote(keyword))
                        Actor.log.info(f'Enqueuing {link_url} ...')
                        await default_queue.add_request({
                            'url': link_url,
                            'userData': {'depth': depth + 1, 'keyword': keyword, 'since': since, 'until': until, 'max_pages': max_pages },
                        })
                    
                blogs = handle_search_result(dom)

                for blog in blogs:
                    driver.get(blog["url"])
                    WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it("mainFrame"))
                    dom = html.fromstring(driver.page_source)
                    num_of_likes, num_of_comments, full_text = handle_blog_page(dom)
                    blog['full_text'] = full_text
                    blog['num_of_likes'] = num_of_likes
                    blog['num_of_comments'] = num_of_comments
                    blog['keyword'] = keyword

                # Push the result into the default dataset
                await Actor.push_data(blogs)
            except:
                Actor.log.exception(f'Cannot extract data from {url}.')
            finally:
                await default_queue.mark_request_as_handled(request)

        driver.quit()


src/main.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.1.1
selenium ~= 4.9.1
lxml ~= 4.9.3

Where next?