BeautifulSoup Scraper

1from apify_client import ApifyClient
2
3# Initialize the ApifyClient with your Apify API token
4client = ApifyClient("<YOUR_API_TOKEN>")
5
6# Prepare the Actor input
7run_input = {
8    "startUrls": [{ "url": "https://crawlee.dev" }],
9    "maxCrawlingDepth": 1,
10    "requestTimeout": 10,
11    "linkSelector": "a[href]",
12    "linkPatterns": [".*crawlee\\.dev.*"],
13    "pageFunction": """from typing import Any
14
15# See the context section in readme to find out what fields you can access 
16# https://apify.com/vdusek/beautifulsoup-scraper#context    
17def page_function(context: Context) -> Any:
18    url = context.request['url']
19    title = context.soup.title.string if context.soup.title else None
20    return {'url': url, 'title': title}
21""",
22    "soupFeatures": "html.parser",
23    "proxyConfiguration": { "useApifyProxy": True },
24}
25
26# Run the Actor and wait for it to finish
27run = client.actor("apify/beautifulsoup-scraper").call(run_input=run_input)
28
29# Fetch and print Actor results from the run's dataset (if there are any)
30print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
31for item in client.dataset(run["defaultDatasetId"]).iterate_items():
32    print(item)
33
34# 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/python/docs/quick-start