1import { ApifyClient } from 'apify-client';
2
3
4
5const client = new ApifyClient({
6 token: '<YOUR_API_TOKEN>',
7});
8
9
10const input = {
11 "startUrls": [
12 {
13 "url": "https://docs.apify.com"
14 }
15 ],
16 "maxCrawlDepth": 1,
17 "maxCrawlPages": 3,
18 "useSitemaps": false,
19 "useLlmsTxt": false,
20 "respectRobotsTxtFile": true,
21 "crawlOnlyPagesChangedSince": "",
22 "includeUrlGlobs": [],
23 "excludeUrlGlobs": [],
24 "includeUrlRegexps": [],
25 "excludeUrlRegexps": [],
26 "keepUrlFragments": false,
27 "ignoreCanonicalUrl": false,
28 "outputFormat": "markdown",
29 "htmlTransformer": "auto",
30 "includeImages": false,
31 "chunkMarkdown": false,
32 "chunkMaxChars": 2000,
33 "chunkOverlapChars": 200,
34 "keepElementsCssSelector": "",
35 "removeElementsCssSelector": "",
36 "parsePdfText": true,
37 "clickElementsCssSelector": "",
38 "takeScreenshots": false,
39 "customHttpHeaders": [],
40 "storeSkippedUrls": false
41};
42
43
44const run = await client.actor("vexaloom-labs/domain-scoped-site-crawler-for-markdown-datasets").call(input);
45
46
47console.log('Results from dataset');
48console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
49const { items } = await client.dataset(run.defaultDatasetId).listItems();
50items.forEach((item) => {
51 console.dir(item);
52});
53
54
