Web Scraper Task

1from apify_client import ApifyClient
2
3# Initialize the ApifyClient with your Apify API token
4client = ApifyClient("<YOUR_API_TOKEN>")
5
6# Prepare the Actor input
7run_input = {
8    "runMode": "DEVELOPMENT",
9    "startUrls": [{ "url": "https://crawlee.dev" }],
10    "linkSelector": "a[href]",
11    "globs": [{ "glob": "https://crawlee.dev/*/*" }],
12    "pseudoUrls": [],
13    "pageFunction": """// The function accepts a single argument: the \"context\" object.
14// For a complete list of its properties and functions,
15// see https://apify.com/apify/web-scraper#page-function 
16async function pageFunction(context) {
17    // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
18    // debugger; 
19
20    // jQuery is handy for finding DOM elements and extracting data from them.
21    // To use it, make sure to enable the \"Inject jQuery\" option.
22    const $ = context.jQuery;
23    const pageTitle = $('title').first().text();
24    const h1 = $('h1').first().text();
25    const first_h2 = $('h2').first().text();
26    const random_text_from_the_page = $('p').first().text();
27
28
29    // Print some information to actor log
30    context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
31
32    // Manually add a new page to the queue for scraping.
33   await context.enqueueRequest({ url: 'http://www.example.com' });
34
35    // Return an object with the data extracted from the page.
36    // It will be stored to the resulting dataset.
37    return {
38        url: context.request.url,
39        pageTitle,
40        h1,
41        first_h2,
42        random_text_from_the_page
43    };
44}""",
45    "proxyConfiguration": { "useApifyProxy": True },
46    "initialCookies": [],
47    "waitUntil": ["networkidle2"],
48    "preNavigationHooks": """// We need to return array of (possibly async) functions here.
49// The functions accept two arguments: the \"crawlingContext\" object
50// and \"gotoOptions\".
51[
52    async (crawlingContext, gotoOptions) => {
53        // ...
54    },
55]
56""",
57    "postNavigationHooks": """// We need to return array of (possibly async) functions here.
58// The functions accept a single argument: the \"crawlingContext\" object.
59[
60    async (crawlingContext) => {
61        // ...
62    },
63]""",
64    "breakpointLocation": "NONE",
65    "customData": {},
66}
67
68# Run the Actor and wait for it to finish
69run = client.actor("undrtkr984/web-scraper-task").call(run_input=run_input)
70
71# Fetch and print Actor results from the run's dataset (if there are any)
72print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
73for item in client.dataset(run["defaultDatasetId"]).iterate_items():
74    print(item)
75
76# 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/python/docs/quick-start