1from apify_client import ApifyClient
2
3
4
5client = ApifyClient("<YOUR_API_TOKEN>")
6
7
8run_input = {
9 "preset": "seo-friendly",
10 "sitemaps": ["https://example.com/sitemap.xml"],
11 "host": "example.com",
12 "rules": [
13 {
14 "userAgents": ["*"],
15 "allow": ["/"],
16 "disallow": [
17 "/admin/",
18 "/private/",
19 ],
20 "crawlDelay": 1,
21 "comment": "Default rules for all bots",
22 },
23 {
24 "userAgents": ["Googlebot"],
25 "allow": ["/"],
26 "disallow": [],
27 "comment": "Give Googlebot full access",
28 },
29 ],
30 "presetConflictMode": "append",
31 "sites": [
32 {
33 "label": "Marketing site",
34 "preset": "seo-friendly",
35 "sitemaps": ["https://example.com/sitemap.xml"],
36 "host": "example.com",
37 },
38 {
39 "label": "Staging site",
40 "preset": "staging-private",
41 "sitemaps": [],
42 "host": "staging.example.com",
43 },
44 ],
45 "validateRules": True,
46 "includeTimestamp": True,
47 "includeGeneratorComment": True,
48}
49
50
51run = client.actor("maximedupre/robots-txt-generator").call(run_input=run_input)
52
53
54print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
55for item in client.dataset(run["defaultDatasetId"]).iterate_items():
56 print(item)
57
58
