Web Scraper Experimental Debug
Try for free
No credit card required
Go to Store
Web Scraper Experimental Debug
mtrunkat/web-scraper-experimental-dbgr
Try for free
No credit card required
Experimental version of Apify Web Scraper with Chrome debugger integrated
You can access the Web Scraper Experimental Debug programmatically from your own applications by using the Apify API. You can choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in Integrations settings in Apify Console.
1{
2 "openapi": "3.0.1",
3 "info": {
4 "version": "0.1",
5 "x-build-id": "Q3sb7KLmaZ3L46Zyg"
6 },
7 "servers": [
8 {
9 "url": "https://api.apify.com/v2"
10 }
11 ],
12 "paths": {
13 "/acts/mtrunkat~web-scraper-experimental-dbgr/run-sync-get-dataset-items": {
14 "post": {
15 "operationId": "run-sync-get-dataset-items-mtrunkat-web-scraper-experimental-dbgr",
16 "x-openai-isConsequential": false,
17 "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
18 "tags": [
19 "Run Actor"
20 ],
21 "requestBody": {
22 "required": true,
23 "content": {
24 "application/json": {
25 "schema": {
26 "$ref": "#/components/schemas/inputSchema"
27 }
28 }
29 }
30 },
31 "parameters": [
32 {
33 "name": "token",
34 "in": "query",
35 "required": true,
36 "schema": {
37 "type": "string"
38 },
39 "description": "Enter your Apify token here"
40 }
41 ],
42 "responses": {
43 "200": {
44 "description": "OK"
45 }
46 }
47 }
48 },
49 "/acts/mtrunkat~web-scraper-experimental-dbgr/runs": {
50 "post": {
51 "operationId": "runs-sync-mtrunkat-web-scraper-experimental-dbgr",
52 "x-openai-isConsequential": false,
53 "summary": "Executes an Actor and returns information about the initiated run in response.",
54 "tags": [
55 "Run Actor"
56 ],
57 "requestBody": {
58 "required": true,
59 "content": {
60 "application/json": {
61 "schema": {
62 "$ref": "#/components/schemas/inputSchema"
63 }
64 }
65 }
66 },
67 "parameters": [
68 {
69 "name": "token",
70 "in": "query",
71 "required": true,
72 "schema": {
73 "type": "string"
74 },
75 "description": "Enter your Apify token here"
76 }
77 ],
78 "responses": {
79 "200": {
80 "description": "OK",
81 "content": {
82 "application/json": {
83 "schema": {
84 "$ref": "#/components/schemas/runsResponseSchema"
85 }
86 }
87 }
88 }
89 }
90 }
91 },
92 "/acts/mtrunkat~web-scraper-experimental-dbgr/run-sync": {
93 "post": {
94 "operationId": "run-sync-mtrunkat-web-scraper-experimental-dbgr",
95 "x-openai-isConsequential": false,
96 "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
97 "tags": [
98 "Run Actor"
99 ],
100 "requestBody": {
101 "required": true,
102 "content": {
103 "application/json": {
104 "schema": {
105 "$ref": "#/components/schemas/inputSchema"
106 }
107 }
108 }
109 },
110 "parameters": [
111 {
112 "name": "token",
113 "in": "query",
114 "required": true,
115 "schema": {
116 "type": "string"
117 },
118 "description": "Enter your Apify token here"
119 }
120 ],
121 "responses": {
122 "200": {
123 "description": "OK"
124 }
125 }
126 }
127 }
128 },
129 "components": {
130 "schemas": {
131 "inputSchema": {
132 "type": "object",
133 "required": [
134 "startUrls",
135 "pageFunction"
136 ],
137 "properties": {
138 "startUrls": {
139 "title": "Start URLs",
140 "type": "array",
141 "description": "URLs to start with",
142 "items": {
143 "type": "object",
144 "required": [
145 "url"
146 ],
147 "properties": {
148 "url": {
149 "type": "string",
150 "title": "URL of a web page",
151 "format": "uri"
152 }
153 }
154 }
155 },
156 "useRequestQueue": {
157 "title": "Use request queue",
158 "type": "boolean",
159 "description": "Request queue enables recursive crawling and the use of Pseudo-URLs, Link selector and <code>context.enqueueRequest()</code>.",
160 "default": true
161 },
162 "pseudoUrls": {
163 "title": "Pseudo-URLs",
164 "type": "array",
165 "description": "Pseudo-URLs to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Pseudo-URLs will cause the scraper to enqueue all links matched by the Link selector.",
166 "default": [],
167 "items": {
168 "type": "object",
169 "required": [
170 "purl"
171 ],
172 "properties": {
173 "purl": {
174 "type": "string",
175 "title": "Pseudo-URL of a web page"
176 }
177 }
178 }
179 },
180 "linkSelector": {
181 "title": "Link selector",
182 "type": "string",
183 "description": "CSS selector matching elements with 'href' attributes that should be enqueued. To enqueue urls from <code><div class=\"my-class\" href=...></code> tags, you would enter <strong>div.my-class</strong>. Leave empty to ignore all links."
184 },
185 "keepUrlFragments": {
186 "title": "Keep URL fragments",
187 "type": "boolean",
188 "description": "URL fragments (the parts of URL after a <code>#</code>) are not considered when the scraper determines whether a URL has already been visited. This means that when adding URLs such as <code>https://example.com/#foo</code> and <code>https://example.com/#bar</code>, only the first will be visited. Turn this option on to tell the scraper to visit both.",
189 "default": false
190 },
191 "pageFunction": {
192 "title": "Page function",
193 "type": "string",
194 "description": "Function executed for each request"
195 },
196 "injectJQuery": {
197 "title": "jQuery",
198 "type": "boolean",
199 "description": "The jQuery library will be injected into each page. If the page already uses jQuery, conflicts may arise.",
200 "default": true
201 },
202 "injectUnderscore": {
203 "title": "Underscore",
204 "type": "boolean",
205 "description": "The Underscore.js library will be injected into each page. If the page already uses Underscore.js (or other libraries that attach to '_', such as Lodash), conflicts may arise.",
206 "default": false
207 },
208 "proxyConfiguration": {
209 "title": "Proxy configuration",
210 "type": "object",
211 "description": "Choose to use no proxy, Apify Proxy, or provide custom proxy URLs.",
212 "default": {}
213 },
214 "initialCookies": {
215 "title": "Initial cookies",
216 "type": "array",
217 "description": "The provided cookies will be pre-set to all pages the scraper opens.",
218 "default": []
219 },
220 "useChrome": {
221 "title": "Use Chrome",
222 "type": "boolean",
223 "description": "The scraper will use a real Chrome browser instead of a Chromium masking as Chrome. Using this option may help with bypassing certain anti-scraping protections, but risks that the scraper will be unstable or not work at all.",
224 "default": false
225 },
226 "useStealth": {
227 "title": "Use Stealth",
228 "type": "boolean",
229 "description": "The scraper will apply various browser emulation techniques to match a real user as closely as possible. This feature works best in conjunction with the Use Chrome option and also carries the risk of making the scraper unstable.",
230 "default": false
231 },
232 "ignoreSslErrors": {
233 "title": "Ignore SSL errors",
234 "type": "boolean",
235 "description": "Scraper will ignore SSL certificate errors.",
236 "default": false
237 },
238 "ignoreCorsAndCsp": {
239 "title": "Ignore CORS and CSP",
240 "type": "boolean",
241 "description": "Scraper will ignore CSP (content security policy) and CORS (cross origin resource sharing) settings of visited pages and requested domains. This enables you to freely use XHR/Fetch to make HTTP requests from the scraper.",
242 "default": false
243 },
244 "downloadMedia": {
245 "title": "Download media",
246 "type": "boolean",
247 "description": "Scraper will download media such as images, fonts, videos and sounds. Disabling this may speed up the scrape, but certain websites could stop working correctly.",
248 "default": true
249 },
250 "downloadCss": {
251 "title": "Download CSS",
252 "type": "boolean",
253 "description": "Scraper will download CSS stylesheets. Disabling this may speed up the scrape, but certain websites could stop working correctly.",
254 "default": true
255 },
256 "maxRequestRetries": {
257 "title": "Max request retries",
258 "minimum": 0,
259 "type": "integer",
260 "description": "Maximum number of times the request for the page will be retried in case of an error. Setting it to 0 means that the request will be attempted once and will not be retried if it fails.",
261 "default": 3
262 },
263 "maxPagesPerCrawl": {
264 "title": "Max pages per run",
265 "minimum": 0,
266 "type": "integer",
267 "description": "Maximum number of pages that the scraper will open. 0 means unlimited.",
268 "default": 0
269 },
270 "maxResultsPerCrawl": {
271 "title": "Max result records",
272 "minimum": 0,
273 "type": "integer",
274 "description": "Maximum number of results that will be saved to dataset. The scraper will terminate afterwards. 0 means unlimited.",
275 "default": 0
276 },
277 "maxCrawlingDepth": {
278 "title": "Max crawling depth",
279 "minimum": 0,
280 "type": "integer",
281 "description": "Defines how many links away from the StartURLs will the scraper descend. 0 means unlimited.",
282 "default": 0
283 },
284 "maxConcurrency": {
285 "title": "Max concurrency",
286 "minimum": 1,
287 "type": "integer",
288 "description": "Defines how many pages can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. Use this option to set a hard limit.",
289 "default": 50
290 },
291 "pageLoadTimeoutSecs": {
292 "title": "Page load timeout",
293 "minimum": 1,
294 "maximum": 360,
295 "type": "integer",
296 "description": "Maximum time the scraper will allow a web page to load in seconds.",
297 "default": 60
298 },
299 "pageFunctionTimeoutSecs": {
300 "title": "Page function timeout",
301 "minimum": 1,
302 "maximum": 360,
303 "type": "integer",
304 "description": "Maximum time the scraper will wait for the page function to execute in seconds.",
305 "default": 60
306 },
307 "waitUntil": {
308 "title": "Navigation wait until",
309 "type": "array",
310 "description": "The scraper will wait until the selected events are triggered in the page before executing the page function. Available events are <code>domcontentloaded</code>, <code>load</code>, <code>networkidle2</code> and <code>networkidle0</code>. <a href=\"https://pptr.dev/#?product=Puppeteer&show=api-pagegotourl-options\" target=\"_blank\">See Puppeteer docs</a>.",
311 "default": [
312 "networkidle2"
313 ]
314 },
315 "debugLog": {
316 "title": "Debug log",
317 "type": "boolean",
318 "description": "Debug messages will be included in the log. Use <code>context.log.debug('message')</code> to log your own debug messages.",
319 "default": false
320 },
321 "browserLog": {
322 "title": "Browser log",
323 "type": "boolean",
324 "description": "Console messages from the Browser will be included in the log. This may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.",
325 "default": false
326 },
327 "chromeDebugger": {
328 "title": "Chrome debugger [experimental]",
329 "type": "boolean",
330 "description": "Experimental implementation of Chrome debugger. In this mode the scraper will run with single browser on concurrency 1. You can place <code>debugger;</code> into your page function to set up a debugger breakpoint.",
331 "default": false
332 },
333 "customData": {
334 "title": "Custom data",
335 "type": "object",
336 "description": "This object will be available on pageFunction's context as customData.",
337 "default": {}
338 }
339 }
340 },
341 "runsResponseSchema": {
342 "type": "object",
343 "properties": {
344 "data": {
345 "type": "object",
346 "properties": {
347 "id": {
348 "type": "string"
349 },
350 "actId": {
351 "type": "string"
352 },
353 "userId": {
354 "type": "string"
355 },
356 "startedAt": {
357 "type": "string",
358 "format": "date-time",
359 "example": "2025-01-08T00:00:00.000Z"
360 },
361 "finishedAt": {
362 "type": "string",
363 "format": "date-time",
364 "example": "2025-01-08T00:00:00.000Z"
365 },
366 "status": {
367 "type": "string",
368 "example": "READY"
369 },
370 "meta": {
371 "type": "object",
372 "properties": {
373 "origin": {
374 "type": "string",
375 "example": "API"
376 },
377 "userAgent": {
378 "type": "string"
379 }
380 }
381 },
382 "stats": {
383 "type": "object",
384 "properties": {
385 "inputBodyLen": {
386 "type": "integer",
387 "example": 2000
388 },
389 "rebootCount": {
390 "type": "integer",
391 "example": 0
392 },
393 "restartCount": {
394 "type": "integer",
395 "example": 0
396 },
397 "resurrectCount": {
398 "type": "integer",
399 "example": 0
400 },
401 "computeUnits": {
402 "type": "integer",
403 "example": 0
404 }
405 }
406 },
407 "options": {
408 "type": "object",
409 "properties": {
410 "build": {
411 "type": "string",
412 "example": "latest"
413 },
414 "timeoutSecs": {
415 "type": "integer",
416 "example": 300
417 },
418 "memoryMbytes": {
419 "type": "integer",
420 "example": 1024
421 },
422 "diskMbytes": {
423 "type": "integer",
424 "example": 2048
425 }
426 }
427 },
428 "buildId": {
429 "type": "string"
430 },
431 "defaultKeyValueStoreId": {
432 "type": "string"
433 },
434 "defaultDatasetId": {
435 "type": "string"
436 },
437 "defaultRequestQueueId": {
438 "type": "string"
439 },
440 "buildNumber": {
441 "type": "string",
442 "example": "1.0.0"
443 },
444 "containerUrl": {
445 "type": "string"
446 },
447 "usage": {
448 "type": "object",
449 "properties": {
450 "ACTOR_COMPUTE_UNITS": {
451 "type": "integer",
452 "example": 0
453 },
454 "DATASET_READS": {
455 "type": "integer",
456 "example": 0
457 },
458 "DATASET_WRITES": {
459 "type": "integer",
460 "example": 0
461 },
462 "KEY_VALUE_STORE_READS": {
463 "type": "integer",
464 "example": 0
465 },
466 "KEY_VALUE_STORE_WRITES": {
467 "type": "integer",
468 "example": 1
469 },
470 "KEY_VALUE_STORE_LISTS": {
471 "type": "integer",
472 "example": 0
473 },
474 "REQUEST_QUEUE_READS": {
475 "type": "integer",
476 "example": 0
477 },
478 "REQUEST_QUEUE_WRITES": {
479 "type": "integer",
480 "example": 0
481 },
482 "DATA_TRANSFER_INTERNAL_GBYTES": {
483 "type": "integer",
484 "example": 0
485 },
486 "DATA_TRANSFER_EXTERNAL_GBYTES": {
487 "type": "integer",
488 "example": 0
489 },
490 "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
491 "type": "integer",
492 "example": 0
493 },
494 "PROXY_SERPS": {
495 "type": "integer",
496 "example": 0
497 }
498 }
499 },
500 "usageTotalUsd": {
501 "type": "number",
502 "example": 0.00005
503 },
504 "usageUsd": {
505 "type": "object",
506 "properties": {
507 "ACTOR_COMPUTE_UNITS": {
508 "type": "integer",
509 "example": 0
510 },
511 "DATASET_READS": {
512 "type": "integer",
513 "example": 0
514 },
515 "DATASET_WRITES": {
516 "type": "integer",
517 "example": 0
518 },
519 "KEY_VALUE_STORE_READS": {
520 "type": "integer",
521 "example": 0
522 },
523 "KEY_VALUE_STORE_WRITES": {
524 "type": "number",
525 "example": 0.00005
526 },
527 "KEY_VALUE_STORE_LISTS": {
528 "type": "integer",
529 "example": 0
530 },
531 "REQUEST_QUEUE_READS": {
532 "type": "integer",
533 "example": 0
534 },
535 "REQUEST_QUEUE_WRITES": {
536 "type": "integer",
537 "example": 0
538 },
539 "DATA_TRANSFER_INTERNAL_GBYTES": {
540 "type": "integer",
541 "example": 0
542 },
543 "DATA_TRANSFER_EXTERNAL_GBYTES": {
544 "type": "integer",
545 "example": 0
546 },
547 "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
548 "type": "integer",
549 "example": 0
550 },
551 "PROXY_SERPS": {
552 "type": "integer",
553 "example": 0
554 }
555 }
556 }
557 }
558 }
559 }
560 }
561 }
562 }
563}Web Scraper Experimental Debug OpenAPI definition
OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Web Scraper Experimental Debug from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients:
Developer
Marek TrunkátMaintained by Community
Actor Metrics
3 monthly users
-
3 bookmarks
>99% runs succeeded
Created in Oct 2019
Modified 4 years ago
Categories
