Cheerio Scraper

1from apify_client import ApifyClient
2
3# Initialize the ApifyClient with your Apify API token
4# Replace '<YOUR_API_TOKEN>' with your token.
5client = ApifyClient("<YOUR_API_TOKEN>")
6
7# Prepare the Actor input
8run_input = {
9    "startUrls": [{ "url": "https://crawlee.dev" }],
10    "globs": [{ "glob": "https://crawlee.dev/*/*" }],
11    "pseudoUrls": [],
12    "excludes": [{ "glob": "/**/*.{png,jpg,jpeg,pdf}" }],
13    "linkSelector": "a[href]",
14    "pageFunction": """async function pageFunction(context) {
15    const { $, request, log } = context;
16
17    // The \"$\" property contains the Cheerio object which is useful
18    // for querying DOM elements and extracting data from them.
19    const pageTitle = $('title').first().text();
20
21    // The \"request\" property contains various information about the web page loaded. 
22    const url = request.url;
23    
24    // Use \"log\" object to print information to actor log.
25    log.info('Page scraped', { url, pageTitle });
26
27    // Return an object with the data extracted from the page.
28    // It will be stored to the resulting dataset.
29    return {
30        url,
31        pageTitle
32    };
33}""",
34    "proxyConfiguration": { "useApifyProxy": True },
35    "initialCookies": [],
36    "additionalMimeTypes": [],
37    "preNavigationHooks": """// We need to return array of (possibly async) functions here.
38// The functions accept two arguments: the \"crawlingContext\" object
39// and \"requestAsBrowserOptions\" which are passed to the `requestAsBrowser()`
40// function the crawler calls to navigate..
41[
42    async (crawlingContext, requestAsBrowserOptions) => {
43        // ...
44    }
45]""",
46    "postNavigationHooks": """// We need to return array of (possibly async) functions here.
47// The functions accept a single argument: the \"crawlingContext\" object.
48[
49    async (crawlingContext) => {
50        // ...
51    },
52]""",
53    "customData": {},
54}
55
56# Run the Actor and wait for it to finish
57run = client.actor("apify/cheerio-scraper").call(run_input=run_input)
58
59# Fetch and print Actor results from the run's dataset (if there are any)
60print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
61for item in client.dataset(run["defaultDatasetId"]).iterate_items():
62    print(item)
63
64# 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/python/docs/quick-start