Web Scraper

1from apify_client import ApifyClient
2
3# Initialize the ApifyClient with your Apify API token
4# Replace '<YOUR_API_TOKEN>' with your token.
5client = ApifyClient("<YOUR_API_TOKEN>")
6
7# Prepare the Actor input
8run_input = {
9    "runMode": "DEVELOPMENT",
10    "startUrls": [{ "url": "https://crawlee.dev" }],
11    "linkSelector": "a[href]",
12    "globs": [{ "glob": "https://crawlee.dev/*/*" }],
13    "pseudoUrls": [],
14    "excludes": [{ "glob": "/**/*.{png,jpg,jpeg,pdf}" }],
15    "pageFunction": """// The function accepts a single argument: the \"context\" object.
16// For a complete list of its properties and functions,
17// see https://apify.com/apify/web-scraper#page-function 
18async function pageFunction(context) {
19    // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
20    // debugger; 
21
22    // jQuery is handy for finding DOM elements and extracting data from them.
23    // To use it, make sure to enable the \"Inject jQuery\" option.
24    const $ = context.jQuery;
25    const pageTitle = $('title').first().text();
26    const h1 = $('h1').first().text();
27    const first_h2 = $('h2').first().text();
28    const random_text_from_the_page = $('p').first().text();
29
30
31    // Print some information to actor log
32    context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
33
34    // Manually add a new page to the queue for scraping.
35   await context.enqueueRequest({ url: 'http://www.example.com' });
36
37    // Return an object with the data extracted from the page.
38    // It will be stored to the resulting dataset.
39    return {
40        url: context.request.url,
41        pageTitle,
42        h1,
43        first_h2,
44        random_text_from_the_page
45    };
46}""",
47    "proxyConfiguration": { "useApifyProxy": True },
48    "initialCookies": [],
49    "waitUntil": ["networkidle2"],
50    "preNavigationHooks": """// We need to return array of (possibly async) functions here.
51// The functions accept two arguments: the \"crawlingContext\" object
52// and \"gotoOptions\".
53[
54    async (crawlingContext, gotoOptions) => {
55        // ...
56    },
57]
58""",
59    "postNavigationHooks": """// We need to return array of (possibly async) functions here.
60// The functions accept a single argument: the \"crawlingContext\" object.
61[
62    async (crawlingContext) => {
63        // ...
64    },
65]""",
66    "breakpointLocation": "NONE",
67    "customData": {},
68}
69
70# Run the Actor and wait for it to finish
71run = client.actor("apify/web-scraper").call(run_input=run_input)
72
73# Fetch and print Actor results from the run's dataset (if there are any)
74print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
75for item in client.dataset(run["defaultDatasetId"]).iterate_items():
76    print(item)
77
78# 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/python/docs/quick-start