Smart Article Extractor

1{
2  "openapi": "3.0.1",
3  "info": {
4    "version": "1.0",
5    "x-build-id": "d9P6aRQgRTMibYSs2"
6  },
7  "servers": [
8    {
9      "url": "https://api.apify.com/v2"
10    }
11  ],
12  "paths": {
13    "/acts/lukaskrivka~article-extractor-smart/run-sync-get-dataset-items": {
14      "post": {
15        "operationId": "run-sync-get-dataset-items-lukaskrivka-article-extractor-smart",
16        "x-openai-isConsequential": false,
17        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
18        "tags": [
19          "Run Actor"
20        ],
21        "requestBody": {
22          "required": true,
23          "content": {
24            "application/json": {
25              "schema": {
26                "$ref": "#/components/schemas/inputSchema"
27              }
28            }
29          }
30        },
31        "parameters": [
32          {
33            "name": "token",
34            "in": "query",
35            "required": true,
36            "schema": {
37              "type": "string"
38            },
39            "description": "Enter your Apify token here"
40          }
41        ],
42        "responses": {
43          "200": {
44            "description": "OK"
45          }
46        }
47      }
48    },
49    "/acts/lukaskrivka~article-extractor-smart/runs": {
50      "post": {
51        "operationId": "runs-sync-lukaskrivka-article-extractor-smart",
52        "x-openai-isConsequential": false,
53        "summary": "Executes an Actor and returns information about the initiated run in response.",
54        "tags": [
55          "Run Actor"
56        ],
57        "requestBody": {
58          "required": true,
59          "content": {
60            "application/json": {
61              "schema": {
62                "$ref": "#/components/schemas/inputSchema"
63              }
64            }
65          }
66        },
67        "parameters": [
68          {
69            "name": "token",
70            "in": "query",
71            "required": true,
72            "schema": {
73              "type": "string"
74            },
75            "description": "Enter your Apify token here"
76          }
77        ],
78        "responses": {
79          "200": {
80            "description": "OK",
81            "content": {
82              "application/json": {
83                "schema": {
84                  "$ref": "#/components/schemas/runsResponseSchema"
85                }
86              }
87            }
88          }
89        }
90      }
91    },
92    "/acts/lukaskrivka~article-extractor-smart/run-sync": {
93      "post": {
94        "operationId": "run-sync-lukaskrivka-article-extractor-smart",
95        "x-openai-isConsequential": false,
96        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
97        "tags": [
98          "Run Actor"
99        ],
100        "requestBody": {
101          "required": true,
102          "content": {
103            "application/json": {
104              "schema": {
105                "$ref": "#/components/schemas/inputSchema"
106              }
107            }
108          }
109        },
110        "parameters": [
111          {
112            "name": "token",
113            "in": "query",
114            "required": true,
115            "schema": {
116              "type": "string"
117            },
118            "description": "Enter your Apify token here"
119          }
120        ],
121        "responses": {
122          "200": {
123            "description": "OK"
124          }
125        }
126      }
127    }
128  },
129  "components": {
130    "schemas": {
131      "inputSchema": {
132        "type": "object",
133        "properties": {
134          "startUrls": {
135            "title": "Website/category URLs",
136            "type": "array",
137            "description": "These could be the main page URL or any category/subpage URL, e.g. https://www.bbc.com/. Article pages are detected and crawled from these. If you prefer to use direct article URLs, use `articleUrls` input instead",
138            "items": {
139              "type": "object",
140              "required": [
141                "url"
142              ],
143              "properties": {
144                "url": {
145                  "type": "string",
146                  "title": "URL of a web page",
147                  "format": "uri"
148                }
149              }
150            }
151          },
152          "articleUrls": {
153            "title": "Article URLs",
154            "type": "array",
155            "description": "These are direct URLs for the articles to be extracted, e.g. https://www.bbc.com/news/uk-62836057. No extra pages are crawled from article pages.",
156            "items": {
157              "type": "object",
158              "required": [
159                "url"
160              ],
161              "properties": {
162                "url": {
163                  "type": "string",
164                  "title": "URL of a web page",
165                  "format": "uri"
166                }
167              }
168            }
169          },
170          "onlyNewArticles": {
171            "title": "Only new articles (only for small runs)",
172            "type": "boolean",
173            "description": "This option is only viable for smaller runs. If you plan to use this on a large scale, use the 'Only new articles (saved per domain)' option below instead. If this function is selected, the extractor will only scrape new articles each time you run it. (Scraped URLs are saved in a dataset named `articles-state`, and are compared with new ones.)",
174            "default": false
175          },
176          "onlyNewArticlesPerDomain": {
177            "title": "Only new articles (saved per domain, preferable)",
178            "type": "boolean",
179            "description": "If this function is selected, the extractor will only scrape only new articles each time you run it. (Scraped articles are saved in one dataset, named 'ARTICLES-SCRAPED-domain', per each domain, and compared with new ones.)",
180            "default": false
181          },
182          "onlyInsideArticles": {
183            "title": "Only inside domain articles",
184            "type": "boolean",
185            "description": "If this function is selected, the extractor will only scrape articles that are on the domain from where they are linked. If the domain presents links to articles on different domains, those articles will not be scraped, e.g. https://www.bbc.com/ vs. https://www.bbc.co.uk/.",
186            "default": true
187          },
188          "enqueueFromArticles": {
189            "title": "Enqueue articles from articles",
190            "type": "boolean",
191            "description": "Normally, the scraper only extracts articles from category pages. This option allows the scraper to also extract articles linked within articles.",
192            "default": false
193          },
194          "crawlWholeSubdomain": {
195            "title": "Crawl whole subdomain (same base as Start URL)",
196            "type": "boolean",
197            "description": "Automatically enqueue categories and articles from whole subdomain with the same path. E.g. if Start URL is https://apify.com/store, it will enqueue all pages starting with https://apify.com/store",
198            "default": false
199          },
200          "onlySubdomainArticles": {
201            "title": "Limit articles to only from subdomain",
202            "type": "boolean",
203            "description": "Only loads articles which URL begins with the same path as Start URL. E.g. if Start URL is https://apify.com/store, it will only load articles starting with https://apify.com/store",
204            "default": false
205          },
206          "scanSitemaps": {
207            "title": "Find articles in sitemaps (caution)",
208            "type": "boolean",
209            "description": "We recommend using `Sitemap URLs` instead. \n If this function is selected, the extractor will scan different sitemaps from the initial article URL. Keep in mind that this option can lead to the loading of a huge amount of (sometimes old) articles, in which case the time and cost of the scrape will increase.",
210            "default": false
211          },
212          "sitemapUrls": {
213            "title": "Sitemap URLs (safer)",
214            "type": "array",
215            "description": "You can provide selected sitemap URLs that include the articles you need to extract.",
216            "items": {
217              "type": "object",
218              "required": [
219                "url"
220              ],
221              "properties": {
222                "url": {
223                  "type": "string",
224                  "title": "URL of a web page",
225                  "format": "uri"
226                }
227              }
228            }
229          },
230          "saveHtml": {
231            "title": "Save full HTML",
232            "type": "boolean",
233            "description": "If this function is selected, the scraper will save the full HTML of the article page, but this will make the data less readable."
234          },
235          "saveHtmlAsLink": {
236            "title": "Save full HTML (only as link to it)",
237            "type": "boolean",
238            "description": "If this function is selected, the scraper will save the full HTML of the article page as a URL to keep the dataset clean and small."
239          },
240          "saveSnapshots": {
241            "title": "Save screenshots of article pages (browser only)",
242            "type": "boolean",
243            "description": "Stores a screenshot for each article page to Key-Value Store and provides that as screenshotUrl. Useful for debugging.",
244            "default": false
245          },
246          "useGoogleBotHeaders": {
247            "title": "Use Googlebot headers",
248            "type": "boolean",
249            "description": "This option will allow you to bypass protection and paywalls on some websites. Use with caution as it might lead to getting blocked.",
250            "default": false
251          },
252          "minWords": {
253            "title": "Minimum words",
254            "type": "integer",
255            "description": "The article needs to contain at least this number of words to be extracted",
256            "default": 150
257          },
258          "dateFrom": {
259            "title": "Extract articles from [date]",
260            "type": "string",
261            "description": "Only articles from this day on will be scraped. If empty, all articles will be scraped. Format is YYYY-MM-DD, e.g. 2019-12-31, or number type e.g. 1 week or 20 days"
262          },
263          "onlyArticlesForLastDays": {
264            "title": "Only articles for last X days",
265            "type": "integer",
266            "description": "Only get posts that were published in the last X days from time the scraping starts. Use either this or the absolute date."
267          },
268          "mustHaveDate": {
269            "title": "Must have date",
270            "type": "boolean",
271            "description": "If checked, the article must have a date of release to be extracted.",
272            "default": true
273          },
274          "isUrlArticleDefinition": {
275            "title": "Is the URL an article?",
276            "type": "object",
277            "description": "Here you can input JSON settings to define what URLs should be considered articles by the scraper. If any of them is `true`, then the link will be opened and the article extracted."
278          },
279          "pseudoUrls": {
280            "title": "Pseudo URLs",
281            "type": "array",
282            "description": "This function can be used to enqueue more pages, i.e. include more links like pagination or categories. This doesn't work for articles, as they are recognized by the recognition system.",
283            "items": {
284              "type": "object",
285              "required": [
286                "url"
287              ],
288              "properties": {
289                "url": {
290                  "type": "string",
291                  "title": "URL of a web page",
292                  "format": "uri"
293                }
294              }
295            }
296          },
297          "linkSelector": {
298            "title": "Link selector",
299            "type": "string",
300            "description": "You can limit the <a> tags whose links will be enqueued. This field is empty by default. Add `a.some-class` to activate it"
301          },
302          "maxDepth": {
303            "title": "Max depth",
304            "type": "integer",
305            "description": "Maximum depth of crawling, i.e. how many times the scraper picks up a link to other webpages. Level 0 refers to the start URLs, 1 are the first level links, and so on. This is only valid for pseudo URLs"
306          },
307          "maxPagesPerCrawl": {
308            "title": "Max pages per crawl",
309            "type": "integer",
310            "description": "Maximum number of total pages crawled. It includes the home page, pagination pages, invalid articles, and so on. The crawler will stop automatically after reaching this number."
311          },
312          "maxArticlesPerCrawl": {
313            "title": "Max articles per crawl",
314            "type": "integer",
315            "description": "Maximum number of valid articles scraped. The crawler will stop automatically after reaching this number."
316          },
317          "maxArticlesPerStartUrl": {
318            "title": "Max articles per start URL",
319            "type": "integer",
320            "description": "Maximum number of articles scraped per start URL."
321          },
322          "maxConcurrency": {
323            "title": "Max concurrency",
324            "type": "integer",
325            "description": "You can limit the speed of the scraper to avoid getting blocked."
326          },
327          "proxyConfiguration": {
328            "title": "Proxy configuration",
329            "type": "object",
330            "description": "Proxy configuration"
331          },
332          "overrideProxyGroup": {
333            "title": "Override proxy group",
334            "type": "string",
335            "description": "If you want to override the default proxy group, you can specify it here. This is useful if you want to use a different proxy group for each crawler."
336          },
337          "useBrowser": {
338            "title": "Use browser (Puppeteer)",
339            "type": "boolean",
340            "description": "This option is more expensive, but it allows you to evaluate JavaScript and wait for dynamically loaded data.",
341            "default": false
342          },
343          "pageWaitMs": {
344            "title": "Wait on each page (ms)",
345            "type": "integer",
346            "description": "How many milliseconds to wait on each page before extracting data"
347          },
348          "navigationWaitUntil": {
349            "title": "Wait until navigation event is finished",
350            "enum": [
351              "load",
352              "domcontentloaded",
353              "networkidle0",
354              "networkidle2"
355            ],
356            "type": "string",
357            "description": "What to wait until the navigation is finished. `domcontentloaded` happens when initial HTML loads and is fastest. `load` happens when JS is executed and it is default. `networkidle0`, `networkidle2` wait for background network but cannot cause infinite loading.",
358            "default": "load"
359          },
360          "pageWaitSelectorCategory": {
361            "title": "Wait for selector on each category page",
362            "type": "string",
363            "description": "For what selector to wait on each page before extracting data"
364          },
365          "pageWaitSelectorArticle": {
366            "title": "Wait for selector on each article page",
367            "type": "string",
368            "description": "For what selector to wait on each page before extracting data"
369          },
370          "scrollToBottom": {
371            "title": "Scroll to bottom of the page (infinite scroll)",
372            "type": "boolean",
373            "description": "Scroll to the botton of the page, loading dynamic articles."
374          },
375          "scrollToBottomButtonSelector": {
376            "title": "Scroll to bottom button selector",
377            "type": "string",
378            "description": "CSS selector for a button to load more articles"
379          },
380          "scrollToBottomMaxSecs": {
381            "title": "Scroll to bottom max seconds",
382            "type": "integer",
383            "description": "Limit for how long the scrolling can run so it does no go infinite."
384          },
385          "extendOutputFunction": {
386            "title": "Extend output function",
387            "type": "string",
388            "description": "This function allows you to merge your custom extraction with the default one. You can only return an object from this function. This object will be merged/overwritten with the default output for each article."
389          },
390          "stopAfterCUs": {
391            "title": "Limit CU consumption",
392            "type": "integer",
393            "description": "The scraper will stop running after reaching this number of compute units."
394          },
395          "notificationEmails": {
396            "title": "Emails address for notifications",
397            "type": "array",
398            "description": "Notifications will be sent to these email addresses.",
399            "items": {
400              "type": "string"
401            }
402          },
403          "notifyAfterCUs": {
404            "title": "Notify after [number] CUs",
405            "type": "integer",
406            "description": "The scraper will send notifications to the provided email when it reaches this number of CUs."
407          },
408          "notifyAfterCUsPeriodically": {
409            "title": "Notify every [number] CUs",
410            "type": "integer",
411            "description": "The scraper will send notifications to the provided email every time this number of CUs is reached since the last notification."
412          }
413        }
414      },
415      "runsResponseSchema": {
416        "type": "object",
417        "properties": {
418          "data": {
419            "type": "object",
420            "properties": {
421              "id": {
422                "type": "string"
423              },
424              "actId": {
425                "type": "string"
426              },
427              "userId": {
428                "type": "string"
429              },
430              "startedAt": {
431                "type": "string",
432                "format": "date-time",
433                "example": "2025-01-08T00:00:00.000Z"
434              },
435              "finishedAt": {
436                "type": "string",
437                "format": "date-time",
438                "example": "2025-01-08T00:00:00.000Z"
439              },
440              "status": {
441                "type": "string",
442                "example": "READY"
443              },
444              "meta": {
445                "type": "object",
446                "properties": {
447                  "origin": {
448                    "type": "string",
449                    "example": "API"
450                  },
451                  "userAgent": {
452                    "type": "string"
453                  }
454                }
455              },
456              "stats": {
457                "type": "object",
458                "properties": {
459                  "inputBodyLen": {
460                    "type": "integer",
461                    "example": 2000
462                  },
463                  "rebootCount": {
464                    "type": "integer",
465                    "example": 0
466                  },
467                  "restartCount": {
468                    "type": "integer",
469                    "example": 0
470                  },
471                  "resurrectCount": {
472                    "type": "integer",
473                    "example": 0
474                  },
475                  "computeUnits": {
476                    "type": "integer",
477                    "example": 0
478                  }
479                }
480              },
481              "options": {
482                "type": "object",
483                "properties": {
484                  "build": {
485                    "type": "string",
486                    "example": "latest"
487                  },
488                  "timeoutSecs": {
489                    "type": "integer",
490                    "example": 300
491                  },
492                  "memoryMbytes": {
493                    "type": "integer",
494                    "example": 1024
495                  },
496                  "diskMbytes": {
497                    "type": "integer",
498                    "example": 2048
499                  }
500                }
501              },
502              "buildId": {
503                "type": "string"
504              },
505              "defaultKeyValueStoreId": {
506                "type": "string"
507              },
508              "defaultDatasetId": {
509                "type": "string"
510              },
511              "defaultRequestQueueId": {
512                "type": "string"
513              },
514              "buildNumber": {
515                "type": "string",
516                "example": "1.0.0"
517              },
518              "containerUrl": {
519                "type": "string"
520              },
521              "usage": {
522                "type": "object",
523                "properties": {
524                  "ACTOR_COMPUTE_UNITS": {
525                    "type": "integer",
526                    "example": 0
527                  },
528                  "DATASET_READS": {
529                    "type": "integer",
530                    "example": 0
531                  },
532                  "DATASET_WRITES": {
533                    "type": "integer",
534                    "example": 0
535                  },
536                  "KEY_VALUE_STORE_READS": {
537                    "type": "integer",
538                    "example": 0
539                  },
540                  "KEY_VALUE_STORE_WRITES": {
541                    "type": "integer",
542                    "example": 1
543                  },
544                  "KEY_VALUE_STORE_LISTS": {
545                    "type": "integer",
546                    "example": 0
547                  },
548                  "REQUEST_QUEUE_READS": {
549                    "type": "integer",
550                    "example": 0
551                  },
552                  "REQUEST_QUEUE_WRITES": {
553                    "type": "integer",
554                    "example": 0
555                  },
556                  "DATA_TRANSFER_INTERNAL_GBYTES": {
557                    "type": "integer",
558                    "example": 0
559                  },
560                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
561                    "type": "integer",
562                    "example": 0
563                  },
564                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
565                    "type": "integer",
566                    "example": 0
567                  },
568                  "PROXY_SERPS": {
569                    "type": "integer",
570                    "example": 0
571                  }
572                }
573              },
574              "usageTotalUsd": {
575                "type": "number",
576                "example": 0.00005
577              },
578              "usageUsd": {
579                "type": "object",
580                "properties": {
581                  "ACTOR_COMPUTE_UNITS": {
582                    "type": "integer",
583                    "example": 0
584                  },
585                  "DATASET_READS": {
586                    "type": "integer",
587                    "example": 0
588                  },
589                  "DATASET_WRITES": {
590                    "type": "integer",
591                    "example": 0
592                  },
593                  "KEY_VALUE_STORE_READS": {
594                    "type": "integer",
595                    "example": 0
596                  },
597                  "KEY_VALUE_STORE_WRITES": {
598                    "type": "number",
599                    "example": 0.00005
600                  },
601                  "KEY_VALUE_STORE_LISTS": {
602                    "type": "integer",
603                    "example": 0
604                  },
605                  "REQUEST_QUEUE_READS": {
606                    "type": "integer",
607                    "example": 0
608                  },
609                  "REQUEST_QUEUE_WRITES": {
610                    "type": "integer",
611                    "example": 0
612                  },
613                  "DATA_TRANSFER_INTERNAL_GBYTES": {
614                    "type": "integer",
615                    "example": 0
616                  },
617                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
618                    "type": "integer",
619                    "example": 0
620                  },
621                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
622                    "type": "integer",
623                    "example": 0
624                  },
625                  "PROXY_SERPS": {
626                    "type": "integer",
627                    "example": 0
628                  }
629                }
630              }
631            }
632          }
633        }
634      }
635    }
636  }
637}