ScrapyFy
Go to Store
This Actor is deprecated
This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?
See alternative ActorsScrapyFy
jupri/scrapyfy
Scrapy Runner
You can access the ScrapyFy programmatically from your own applications by using the Apify API. You can choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in Integrations settings in Apify Console.
1{
2 "openapi": "3.0.1",
3 "info": {
4 "version": "0.0",
5 "x-build-id": "fpbGm3h1jlzvVaqMf"
6 },
7 "servers": [
8 {
9 "url": "https://api.apify.com/v2"
10 }
11 ],
12 "paths": {
13 "/acts/jupri~scrapyfy/run-sync-get-dataset-items": {
14 "post": {
15 "operationId": "run-sync-get-dataset-items-jupri-scrapyfy",
16 "x-openai-isConsequential": false,
17 "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
18 "tags": [
19 "Run Actor"
20 ],
21 "requestBody": {
22 "required": true,
23 "content": {
24 "application/json": {
25 "schema": {
26 "$ref": "#/components/schemas/inputSchema"
27 }
28 }
29 }
30 },
31 "parameters": [
32 {
33 "name": "token",
34 "in": "query",
35 "required": true,
36 "schema": {
37 "type": "string"
38 },
39 "description": "Enter your Apify token here"
40 }
41 ],
42 "responses": {
43 "200": {
44 "description": "OK"
45 }
46 }
47 }
48 },
49 "/acts/jupri~scrapyfy/runs": {
50 "post": {
51 "operationId": "runs-sync-jupri-scrapyfy",
52 "x-openai-isConsequential": false,
53 "summary": "Executes an Actor and returns information about the initiated run in response.",
54 "tags": [
55 "Run Actor"
56 ],
57 "requestBody": {
58 "required": true,
59 "content": {
60 "application/json": {
61 "schema": {
62 "$ref": "#/components/schemas/inputSchema"
63 }
64 }
65 }
66 },
67 "parameters": [
68 {
69 "name": "token",
70 "in": "query",
71 "required": true,
72 "schema": {
73 "type": "string"
74 },
75 "description": "Enter your Apify token here"
76 }
77 ],
78 "responses": {
79 "200": {
80 "description": "OK",
81 "content": {
82 "application/json": {
83 "schema": {
84 "$ref": "#/components/schemas/runsResponseSchema"
85 }
86 }
87 }
88 }
89 }
90 }
91 },
92 "/acts/jupri~scrapyfy/run-sync": {
93 "post": {
94 "operationId": "run-sync-jupri-scrapyfy",
95 "x-openai-isConsequential": false,
96 "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
97 "tags": [
98 "Run Actor"
99 ],
100 "requestBody": {
101 "required": true,
102 "content": {
103 "application/json": {
104 "schema": {
105 "$ref": "#/components/schemas/inputSchema"
106 }
107 }
108 }
109 },
110 "parameters": [
111 {
112 "name": "token",
113 "in": "query",
114 "required": true,
115 "schema": {
116 "type": "string"
117 },
118 "description": "Enter your Apify token here"
119 }
120 ],
121 "responses": {
122 "200": {
123 "description": "OK"
124 }
125 }
126 }
127 }
128 },
129 "components": {
130 "schemas": {
131 "inputSchema": {
132 "type": "object",
133 "required": [
134 "spiders_code"
135 ],
136 "properties": {
137 "spiders_code": {
138 "title": "Spiders Code",
139 "type": "string",
140 "description": "Define your spiders <i>(multiple spiders can be specified)</i>."
141 },
142 "AJAXCRAWL_ENABLED": {
143 "title": "AJAXCRAWL_ENABLED",
144 "type": "boolean",
145 "description": "",
146 "default": false
147 },
148 "ASYNCIO_EVENT_LOOP": {
149 "title": "ASYNCIO_EVENT_LOOP",
150 "type": "string",
151 "description": ""
152 },
153 "AUTOTHROTTLE_DEBUG": {
154 "title": "AUTOTHROTTLE_DEBUG",
155 "type": "boolean",
156 "description": "",
157 "default": false
158 },
159 "AUTOTHROTTLE_ENABLED": {
160 "title": "AUTOTHROTTLE_ENABLED",
161 "type": "boolean",
162 "description": "",
163 "default": false
164 },
165 "AUTOTHROTTLE_MAX_DELAY": {
166 "title": "AUTOTHROTTLE_MAX_DELAY",
167 "type": "integer",
168 "description": "",
169 "default": 60
170 },
171 "AUTOTHROTTLE_START_DELAY": {
172 "title": "AUTOTHROTTLE_START_DELAY",
173 "type": "integer",
174 "description": "",
175 "default": 5
176 },
177 "AUTOTHROTTLE_TARGET_CONCURRENCY": {
178 "title": "AUTOTHROTTLE_TARGET_CONCURRENCY",
179 "type": "integer",
180 "description": "",
181 "default": 1
182 },
183 "BOT_NAME": {
184 "title": "BOT_NAME",
185 "type": "string",
186 "description": "",
187 "default": "scrapybot"
188 },
189 "CLOSESPIDER_ERRORCOUNT": {
190 "title": "CLOSESPIDER_ERRORCOUNT",
191 "type": "integer",
192 "description": "",
193 "default": 0
194 },
195 "CLOSESPIDER_ITEMCOUNT": {
196 "title": "CLOSESPIDER_ITEMCOUNT",
197 "type": "integer",
198 "description": "",
199 "default": 0
200 },
201 "CLOSESPIDER_PAGECOUNT": {
202 "title": "CLOSESPIDER_PAGECOUNT",
203 "type": "integer",
204 "description": "",
205 "default": 0
206 },
207 "CLOSESPIDER_TIMEOUT": {
208 "title": "CLOSESPIDER_TIMEOUT",
209 "type": "integer",
210 "description": "",
211 "default": 0
212 },
213 "COMMANDS_MODULE": {
214 "title": "COMMANDS_MODULE",
215 "type": "string",
216 "description": "",
217 "default": ""
218 },
219 "COMPRESSION_ENABLED": {
220 "title": "COMPRESSION_ENABLED",
221 "type": "boolean",
222 "description": "",
223 "default": true
224 },
225 "CONCURRENT_ITEMS": {
226 "title": "CONCURRENT_ITEMS",
227 "type": "integer",
228 "description": "",
229 "default": 100
230 },
231 "CONCURRENT_REQUESTS": {
232 "title": "CONCURRENT_REQUESTS",
233 "type": "integer",
234 "description": "",
235 "default": 16
236 },
237 "CONCURRENT_REQUESTS_PER_DOMAIN": {
238 "title": "CONCURRENT_REQUESTS_PER_DOMAIN",
239 "type": "integer",
240 "description": "",
241 "default": 8
242 },
243 "CONCURRENT_REQUESTS_PER_IP": {
244 "title": "CONCURRENT_REQUESTS_PER_IP",
245 "type": "integer",
246 "description": "",
247 "default": 0
248 },
249 "COOKIES_DEBUG": {
250 "title": "COOKIES_DEBUG",
251 "type": "boolean",
252 "description": "",
253 "default": false
254 },
255 "COOKIES_ENABLED": {
256 "title": "COOKIES_ENABLED",
257 "type": "boolean",
258 "description": "",
259 "default": true
260 },
261 "DEFAULT_ITEM_CLASS": {
262 "title": "DEFAULT_ITEM_CLASS",
263 "type": "string",
264 "description": "",
265 "default": "scrapy.item.Item"
266 },
267 "DEFAULT_REQUEST_HEADERS": {
268 "title": "DEFAULT_REQUEST_HEADERS",
269 "type": "object",
270 "description": ""
271 },
272 "DEPTH_LIMIT": {
273 "title": "DEPTH_LIMIT",
274 "type": "integer",
275 "description": "",
276 "default": 0
277 },
278 "DEPTH_PRIORITY": {
279 "title": "DEPTH_PRIORITY",
280 "type": "integer",
281 "description": "",
282 "default": 0
283 },
284 "DEPTH_STATS_VERBOSE": {
285 "title": "DEPTH_STATS_VERBOSE",
286 "type": "boolean",
287 "description": "",
288 "default": false
289 },
290 "DNSCACHE_ENABLED": {
291 "title": "DNSCACHE_ENABLED",
292 "type": "boolean",
293 "description": "",
294 "default": true
295 },
296 "DNSCACHE_SIZE": {
297 "title": "DNSCACHE_SIZE",
298 "type": "integer",
299 "description": "",
300 "default": 10000
301 },
302 "DNS_RESOLVER": {
303 "title": "DNS_RESOLVER",
304 "type": "string",
305 "description": "",
306 "default": "scrapy.resolver.CachingThreadedResolver"
307 },
308 "DNS_TIMEOUT": {
309 "title": "DNS_TIMEOUT",
310 "type": "integer",
311 "description": "",
312 "default": 60
313 },
314 "DOWNLOADER": {
315 "title": "DOWNLOADER",
316 "type": "string",
317 "description": "",
318 "default": "scrapy.core.downloader.Downloader"
319 },
320 "DOWNLOADER_CLIENTCONTEXTFACTORY": {
321 "title": "DOWNLOADER_CLIENTCONTEXTFACTORY",
322 "type": "string",
323 "description": "",
324 "default": "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
325 },
326 "DOWNLOADER_CLIENT_TLS_CIPHERS": {
327 "title": "DOWNLOADER_CLIENT_TLS_CIPHERS",
328 "type": "string",
329 "description": "",
330 "default": "DEFAULT"
331 },
332 "DOWNLOADER_CLIENT_TLS_METHOD": {
333 "title": "DOWNLOADER_CLIENT_TLS_METHOD",
334 "type": "string",
335 "description": "",
336 "default": "TLS"
337 },
338 "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": {
339 "title": "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING",
340 "type": "boolean",
341 "description": "",
342 "default": false
343 },
344 "DOWNLOADER_HTTPCLIENTFACTORY": {
345 "title": "DOWNLOADER_HTTPCLIENTFACTORY",
346 "type": "string",
347 "description": "",
348 "default": "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
349 },
350 "DOWNLOADER_MIDDLEWARES": {
351 "title": "DOWNLOADER_MIDDLEWARES",
352 "type": "object",
353 "description": ""
354 },
355 "DOWNLOADER_MIDDLEWARES_BASE": {
356 "title": "DOWNLOADER_MIDDLEWARES_BASE",
357 "type": "object",
358 "description": ""
359 },
360 "DOWNLOADER_STATS": {
361 "title": "DOWNLOADER_STATS",
362 "type": "boolean",
363 "description": "",
364 "default": true
365 },
366 "DOWNLOAD_DELAY": {
367 "title": "DOWNLOAD_DELAY",
368 "type": "integer",
369 "description": "",
370 "default": 0
371 },
372 "DOWNLOAD_FAIL_ON_DATALOSS": {
373 "title": "DOWNLOAD_FAIL_ON_DATALOSS",
374 "type": "boolean",
375 "description": "",
376 "default": true
377 },
378 "DOWNLOAD_HANDLERS": {
379 "title": "DOWNLOAD_HANDLERS",
380 "type": "object",
381 "description": ""
382 },
383 "DOWNLOAD_HANDLERS_BASE": {
384 "title": "DOWNLOAD_HANDLERS_BASE",
385 "type": "object",
386 "description": ""
387 },
388 "DOWNLOAD_MAXSIZE": {
389 "title": "DOWNLOAD_MAXSIZE",
390 "type": "integer",
391 "description": "",
392 "default": 1073741824
393 },
394 "DOWNLOAD_TIMEOUT": {
395 "title": "DOWNLOAD_TIMEOUT",
396 "type": "integer",
397 "description": "",
398 "default": 180
399 },
400 "DOWNLOAD_WARNSIZE": {
401 "title": "DOWNLOAD_WARNSIZE",
402 "type": "integer",
403 "description": "",
404 "default": 33554432
405 },
406 "DUPEFILTER_CLASS": {
407 "title": "DUPEFILTER_CLASS",
408 "type": "string",
409 "description": "",
410 "default": "scrapy.dupefilters.RFPDupeFilter"
411 },
412 "EDITOR": {
413 "title": "EDITOR",
414 "type": "string",
415 "description": "",
416 "default": "vi"
417 },
418 "EXTENSIONS": {
419 "title": "EXTENSIONS",
420 "type": "object",
421 "description": ""
422 },
423 "EXTENSIONS_BASE": {
424 "title": "EXTENSIONS_BASE",
425 "type": "object",
426 "description": ""
427 },
428 "FEEDS": {
429 "title": "FEEDS",
430 "type": "object",
431 "description": ""
432 },
433 "FEED_EXPORTERS": {
434 "title": "FEED_EXPORTERS",
435 "type": "object",
436 "description": ""
437 },
438 "FEED_EXPORTERS_BASE": {
439 "title": "FEED_EXPORTERS_BASE",
440 "type": "object",
441 "description": ""
442 },
443 "FEED_EXPORT_BATCH_ITEM_COUNT": {
444 "title": "FEED_EXPORT_BATCH_ITEM_COUNT",
445 "type": "integer",
446 "description": "",
447 "default": 0
448 },
449 "FEED_EXPORT_ENCODING": {
450 "title": "FEED_EXPORT_ENCODING",
451 "type": "string",
452 "description": ""
453 },
454 "FEED_EXPORT_FIELDS": {
455 "title": "FEED_EXPORT_FIELDS",
456 "type": "string",
457 "description": ""
458 },
459 "FEED_EXPORT_INDENT": {
460 "title": "FEED_EXPORT_INDENT",
461 "type": "integer",
462 "description": "",
463 "default": 0
464 },
465 "FEED_STORAGES": {
466 "title": "FEED_STORAGES",
467 "type": "object",
468 "description": ""
469 },
470 "FEED_STORAGES_BASE": {
471 "title": "FEED_STORAGES_BASE",
472 "type": "object",
473 "description": ""
474 },
475 "FEED_STORAGE_FTP_ACTIVE": {
476 "title": "FEED_STORAGE_FTP_ACTIVE",
477 "type": "boolean",
478 "description": "",
479 "default": false
480 },
481 "FEED_STORAGE_GCS_ACL": {
482 "title": "FEED_STORAGE_GCS_ACL",
483 "type": "string",
484 "description": "",
485 "default": ""
486 },
487 "FEED_STORAGE_S3_ACL": {
488 "title": "FEED_STORAGE_S3_ACL",
489 "type": "string",
490 "description": "",
491 "default": ""
492 },
493 "FEED_STORE_EMPTY": {
494 "title": "FEED_STORE_EMPTY",
495 "type": "boolean",
496 "description": "",
497 "default": false
498 },
499 "FEED_TEMPDIR": {
500 "title": "FEED_TEMPDIR",
501 "type": "string",
502 "description": ""
503 },
504 "FEED_URI_PARAMS": {
505 "title": "FEED_URI_PARAMS",
506 "type": "string",
507 "description": ""
508 },
509 "FILES_STORE_GCS_ACL": {
510 "title": "FILES_STORE_GCS_ACL",
511 "type": "string",
512 "description": "",
513 "default": ""
514 },
515 "FILES_STORE_S3_ACL": {
516 "title": "FILES_STORE_S3_ACL",
517 "type": "string",
518 "description": "",
519 "default": "private"
520 },
521 "FTP_PASSIVE_MODE": {
522 "title": "FTP_PASSIVE_MODE",
523 "type": "boolean",
524 "description": "",
525 "default": true
526 },
527 "FTP_PASSWORD": {
528 "title": "FTP_PASSWORD",
529 "type": "string",
530 "description": "",
531 "default": "guest"
532 },
533 "FTP_USER": {
534 "title": "FTP_USER",
535 "type": "string",
536 "description": "",
537 "default": "anonymous"
538 },
539 "GCS_PROJECT_ID": {
540 "title": "GCS_PROJECT_ID",
541 "type": "string",
542 "description": ""
543 },
544 "HTTPCACHE_ALWAYS_STORE": {
545 "title": "HTTPCACHE_ALWAYS_STORE",
546 "type": "boolean",
547 "description": "",
548 "default": false
549 },
550 "HTTPCACHE_DBM_MODULE": {
551 "title": "HTTPCACHE_DBM_MODULE",
552 "type": "string",
553 "description": "",
554 "default": "dbm"
555 },
556 "HTTPCACHE_DIR": {
557 "title": "HTTPCACHE_DIR",
558 "type": "string",
559 "description": "",
560 "default": "httpcache"
561 },
562 "HTTPCACHE_ENABLED": {
563 "title": "HTTPCACHE_ENABLED",
564 "type": "boolean",
565 "description": "",
566 "default": false
567 },
568 "HTTPCACHE_EXPIRATION_SECS": {
569 "title": "HTTPCACHE_EXPIRATION_SECS",
570 "type": "integer",
571 "description": "",
572 "default": 0
573 },
574 "HTTPCACHE_GZIP": {
575 "title": "HTTPCACHE_GZIP",
576 "type": "boolean",
577 "description": "",
578 "default": false
579 },
580 "HTTPCACHE_IGNORE_HTTP_CODES": {
581 "title": "HTTPCACHE_IGNORE_HTTP_CODES",
582 "type": "array",
583 "description": "",
584 "items": {
585 "type": "string"
586 }
587 },
588 "HTTPCACHE_IGNORE_MISSING": {
589 "title": "HTTPCACHE_IGNORE_MISSING",
590 "type": "boolean",
591 "description": "",
592 "default": false
593 },
594 "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS": {
595 "title": "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS",
596 "type": "array",
597 "description": "",
598 "items": {
599 "type": "string"
600 }
601 },
602 "HTTPCACHE_IGNORE_SCHEMES": {
603 "title": "HTTPCACHE_IGNORE_SCHEMES",
604 "type": "array",
605 "description": "",
606 "items": {
607 "type": "string"
608 }
609 },
610 "HTTPCACHE_POLICY": {
611 "title": "HTTPCACHE_POLICY",
612 "type": "string",
613 "description": "",
614 "default": "scrapy.extensions.httpcache.DummyPolicy"
615 },
616 "HTTPCACHE_STORAGE": {
617 "title": "HTTPCACHE_STORAGE",
618 "type": "string",
619 "description": "",
620 "default": "scrapy.extensions.httpcache.FilesystemCacheStorage"
621 },
622 "HTTPPROXY_AUTH_ENCODING": {
623 "title": "HTTPPROXY_AUTH_ENCODING",
624 "type": "string",
625 "description": "",
626 "default": "latin-1"
627 },
628 "HTTPPROXY_ENABLED": {
629 "title": "HTTPPROXY_ENABLED",
630 "type": "boolean",
631 "description": "",
632 "default": true
633 },
634 "IMAGES_STORE_GCS_ACL": {
635 "title": "IMAGES_STORE_GCS_ACL",
636 "type": "string",
637 "description": "",
638 "default": ""
639 },
640 "IMAGES_STORE_S3_ACL": {
641 "title": "IMAGES_STORE_S3_ACL",
642 "type": "string",
643 "description": "",
644 "default": "private"
645 },
646 "ITEM_PIPELINES": {
647 "title": "ITEM_PIPELINES",
648 "type": "object",
649 "description": ""
650 },
651 "ITEM_PIPELINES_BASE": {
652 "title": "ITEM_PIPELINES_BASE",
653 "type": "object",
654 "description": ""
655 },
656 "ITEM_PROCESSOR": {
657 "title": "ITEM_PROCESSOR",
658 "type": "string",
659 "description": "",
660 "default": "scrapy.pipelines.ItemPipelineManager"
661 },
662 "LOGSTATS_INTERVAL": {
663 "title": "LOGSTATS_INTERVAL",
664 "type": "integer",
665 "description": "",
666 "default": 60
667 },
668 "LOG_DATEFORMAT": {
669 "title": "LOG_DATEFORMAT",
670 "type": "string",
671 "description": "",
672 "default": "%Y-%m-%d %H:%M:%S"
673 },
674 "LOG_ENABLED": {
675 "title": "LOG_ENABLED",
676 "type": "boolean",
677 "description": "",
678 "default": true
679 },
680 "LOG_ENCODING": {
681 "title": "LOG_ENCODING",
682 "type": "string",
683 "description": "",
684 "default": "utf-8"
685 },
686 "LOG_FILE": {
687 "title": "LOG_FILE",
688 "type": "string",
689 "description": ""
690 },
691 "LOG_FILE_APPEND": {
692 "title": "LOG_FILE_APPEND",
693 "type": "boolean",
694 "description": "",
695 "default": true
696 },
697 "LOG_FORMAT": {
698 "title": "LOG_FORMAT",
699 "type": "string",
700 "description": "",
701 "default": "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
702 },
703 "LOG_FORMATTER": {
704 "title": "LOG_FORMATTER",
705 "type": "string",
706 "description": "",
707 "default": "scrapy.logformatter.LogFormatter"
708 },
709 "LOG_LEVEL": {
710 "title": "LOG_LEVEL",
711 "type": "string",
712 "description": "",
713 "default": "DEBUG"
714 },
715 "LOG_SHORT_NAMES": {
716 "title": "LOG_SHORT_NAMES",
717 "type": "boolean",
718 "description": "",
719 "default": false
720 },
721 "LOG_STDOUT": {
722 "title": "LOG_STDOUT",
723 "type": "boolean",
724 "description": "",
725 "default": false
726 },
727 "MAIL_FROM": {
728 "title": "MAIL_FROM",
729 "type": "string",
730 "description": "",
731 "default": "scrapy@localhost"
732 },
733 "MAIL_HOST": {
734 "title": "MAIL_HOST",
735 "type": "string",
736 "description": "",
737 "default": "localhost"
738 },
739 "MAIL_PASS": {
740 "title": "MAIL_PASS",
741 "type": "string",
742 "description": ""
743 },
744 "MAIL_PORT": {
745 "title": "MAIL_PORT",
746 "type": "integer",
747 "description": "",
748 "default": 25
749 },
750 "MAIL_USER": {
751 "title": "MAIL_USER",
752 "type": "string",
753 "description": ""
754 },
755 "MEMDEBUG_ENABLED": {
756 "title": "MEMDEBUG_ENABLED",
757 "type": "boolean",
758 "description": "",
759 "default": false
760 },
761 "MEMDEBUG_NOTIFY": {
762 "title": "MEMDEBUG_NOTIFY",
763 "type": "array",
764 "description": "",
765 "items": {
766 "type": "string"
767 }
768 },
769 "MEMUSAGE_CHECK_INTERVAL_SECONDS": {
770 "title": "MEMUSAGE_CHECK_INTERVAL_SECONDS",
771 "type": "integer",
772 "description": "",
773 "default": 60
774 },
775 "MEMUSAGE_ENABLED": {
776 "title": "MEMUSAGE_ENABLED",
777 "type": "boolean",
778 "description": "",
779 "default": true
780 },
781 "MEMUSAGE_LIMIT_MB": {
782 "title": "MEMUSAGE_LIMIT_MB",
783 "type": "integer",
784 "description": "",
785 "default": 0
786 },
787 "MEMUSAGE_NOTIFY_MAIL": {
788 "title": "MEMUSAGE_NOTIFY_MAIL",
789 "type": "array",
790 "description": "",
791 "items": {
792 "type": "string"
793 }
794 },
795 "MEMUSAGE_WARNING_MB": {
796 "title": "MEMUSAGE_WARNING_MB",
797 "type": "integer",
798 "description": "",
799 "default": 0
800 },
801 "METAREFRESH_ENABLED": {
802 "title": "METAREFRESH_ENABLED",
803 "type": "boolean",
804 "description": "",
805 "default": true
806 },
807 "METAREFRESH_IGNORE_TAGS": {
808 "title": "METAREFRESH_IGNORE_TAGS",
809 "type": "array",
810 "description": "",
811 "items": {
812 "type": "string"
813 }
814 },
815 "METAREFRESH_MAXDELAY": {
816 "title": "METAREFRESH_MAXDELAY",
817 "type": "integer",
818 "description": "",
819 "default": 100
820 },
821 "NEWSPIDER_MODULE": {
822 "title": "NEWSPIDER_MODULE",
823 "type": "string",
824 "description": "",
825 "default": ""
826 },
827 "RANDOMIZE_DOWNLOAD_DELAY": {
828 "title": "RANDOMIZE_DOWNLOAD_DELAY",
829 "type": "boolean",
830 "description": "",
831 "default": true
832 },
833 "REACTOR_THREADPOOL_MAXSIZE": {
834 "title": "REACTOR_THREADPOOL_MAXSIZE",
835 "type": "integer",
836 "description": "",
837 "default": 10
838 },
839 "REDIRECT_ENABLED": {
840 "title": "REDIRECT_ENABLED",
841 "type": "boolean",
842 "description": "",
843 "default": true
844 },
845 "REDIRECT_MAX_TIMES": {
846 "title": "REDIRECT_MAX_TIMES",
847 "type": "integer",
848 "description": "",
849 "default": 20
850 },
851 "REDIRECT_PRIORITY_ADJUST": {
852 "title": "REDIRECT_PRIORITY_ADJUST",
853 "type": "integer",
854 "description": "",
855 "default": 2
856 },
857 "REFERER_ENABLED": {
858 "title": "REFERER_ENABLED",
859 "type": "boolean",
860 "description": "",
861 "default": true
862 },
863 "REFERRER_POLICY": {
864 "title": "REFERRER_POLICY",
865 "type": "string",
866 "description": "",
867 "default": "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
868 },
869 "REQUEST_FINGERPRINTER_CLASS": {
870 "title": "REQUEST_FINGERPRINTER_CLASS",
871 "type": "string",
872 "description": "",
873 "default": "scrapy.utils.request.RequestFingerprinter"
874 },
875 "REQUEST_FINGERPRINTER_IMPLEMENTATION": {
876 "title": "REQUEST_FINGERPRINTER_IMPLEMENTATION",
877 "type": "string",
878 "description": "",
879 "default": "2.6"
880 },
881 "RETRY_ENABLED": {
882 "title": "RETRY_ENABLED",
883 "type": "boolean",
884 "description": "",
885 "default": true
886 },
887 "RETRY_HTTP_CODES": {
888 "title": "RETRY_HTTP_CODES",
889 "type": "array",
890 "description": "",
891 "items": {
892 "type": "string"
893 }
894 },
895 "RETRY_PRIORITY_ADJUST": {
896 "title": "RETRY_PRIORITY_ADJUST",
897 "type": "integer",
898 "description": "",
899 "default": -1
900 },
901 "RETRY_TIMES": {
902 "title": "RETRY_TIMES",
903 "type": "integer",
904 "description": "",
905 "default": 2
906 },
907 "ROBOTSTXT_OBEY": {
908 "title": "ROBOTSTXT_OBEY",
909 "type": "boolean",
910 "description": "",
911 "default": false
912 },
913 "ROBOTSTXT_PARSER": {
914 "title": "ROBOTSTXT_PARSER",
915 "type": "string",
916 "description": "",
917 "default": "scrapy.robotstxt.ProtegoRobotParser"
918 },
919 "ROBOTSTXT_USER_AGENT": {
920 "title": "ROBOTSTXT_USER_AGENT",
921 "type": "string",
922 "description": ""
923 },
924 "SCHEDULER": {
925 "title": "SCHEDULER",
926 "type": "string",
927 "description": "",
928 "default": "scrapy.core.scheduler.Scheduler"
929 },
930 "SCHEDULER_DEBUG": {
931 "title": "SCHEDULER_DEBUG",
932 "type": "boolean",
933 "description": "",
934 "default": false
935 },
936 "SCHEDULER_DISK_QUEUE": {
937 "title": "SCHEDULER_DISK_QUEUE",
938 "type": "string",
939 "description": "",
940 "default": "scrapy.squeues.PickleLifoDiskQueue"
941 },
942 "SCHEDULER_MEMORY_QUEUE": {
943 "title": "SCHEDULER_MEMORY_QUEUE",
944 "type": "string",
945 "description": "",
946 "default": "scrapy.squeues.LifoMemoryQueue"
947 },
948 "SCHEDULER_PRIORITY_QUEUE": {
949 "title": "SCHEDULER_PRIORITY_QUEUE",
950 "type": "string",
951 "description": "",
952 "default": "scrapy.pqueues.ScrapyPriorityQueue"
953 },
954 "SCRAPER_SLOT_MAX_ACTIVE_SIZE": {
955 "title": "SCRAPER_SLOT_MAX_ACTIVE_SIZE",
956 "type": "integer",
957 "description": "",
958 "default": 5000000
959 },
960 "SPIDER_CONTRACTS": {
961 "title": "SPIDER_CONTRACTS",
962 "type": "object",
963 "description": ""
964 },
965 "SPIDER_CONTRACTS_BASE": {
966 "title": "SPIDER_CONTRACTS_BASE",
967 "type": "object",
968 "description": ""
969 },
970 "SPIDER_LOADER_CLASS": {
971 "title": "SPIDER_LOADER_CLASS",
972 "type": "string",
973 "description": "",
974 "default": "scrapy.spiderloader.SpiderLoader"
975 },
976 "SPIDER_LOADER_WARN_ONLY": {
977 "title": "SPIDER_LOADER_WARN_ONLY",
978 "type": "boolean",
979 "description": "",
980 "default": false
981 },
982 "SPIDER_MIDDLEWARES": {
983 "title": "SPIDER_MIDDLEWARES",
984 "type": "object",
985 "description": ""
986 },
987 "SPIDER_MIDDLEWARES_BASE": {
988 "title": "SPIDER_MIDDLEWARES_BASE",
989 "type": "object",
990 "description": ""
991 },
992 "SPIDER_MODULES": {
993 "title": "SPIDER_MODULES",
994 "type": "array",
995 "description": "",
996 "items": {
997 "type": "string"
998 }
999 },
1000 "STATSMAILER_RCPTS": {
1001 "title": "STATSMAILER_RCPTS",
1002 "type": "array",
1003 "description": "",
1004 "items": {
1005 "type": "string"
1006 }
1007 },
1008 "STATS_CLASS": {
1009 "title": "STATS_CLASS",
1010 "type": "string",
1011 "description": "",
1012 "default": "scrapy.statscollectors.MemoryStatsCollector"
1013 },
1014 "STATS_DUMP": {
1015 "title": "STATS_DUMP",
1016 "type": "boolean",
1017 "description": "",
1018 "default": true
1019 },
1020 "TELNETCONSOLE_ENABLED": {
1021 "title": "TELNETCONSOLE_ENABLED",
1022 "type": "integer",
1023 "description": "",
1024 "default": 1
1025 },
1026 "TELNETCONSOLE_HOST": {
1027 "title": "TELNETCONSOLE_HOST",
1028 "type": "string",
1029 "description": "",
1030 "default": "127.0.0.1"
1031 },
1032 "TELNETCONSOLE_PASSWORD": {
1033 "title": "TELNETCONSOLE_PASSWORD",
1034 "type": "string",
1035 "description": ""
1036 },
1037 "TELNETCONSOLE_PORT": {
1038 "title": "TELNETCONSOLE_PORT",
1039 "type": "array",
1040 "description": "",
1041 "items": {
1042 "type": "string"
1043 }
1044 },
1045 "TELNETCONSOLE_USERNAME": {
1046 "title": "TELNETCONSOLE_USERNAME",
1047 "type": "string",
1048 "description": "",
1049 "default": "scrapy"
1050 },
1051 "TEMPLATES_DIR": {
1052 "title": "TEMPLATES_DIR",
1053 "type": "string",
1054 "description": "",
1055 "default": "/usr/local/lib/python3.10/dist-packages/scrapy/templates"
1056 },
1057 "TWISTED_REACTOR": {
1058 "title": "TWISTED_REACTOR",
1059 "type": "string",
1060 "description": ""
1061 },
1062 "URLLENGTH_LIMIT": {
1063 "title": "URLLENGTH_LIMIT",
1064 "type": "integer",
1065 "description": "",
1066 "default": 2083
1067 },
1068 "USER_AGENT": {
1069 "title": "USER_AGENT",
1070 "type": "string",
1071 "description": "",
1072 "default": "Scrapy/2.9.0 (+https://scrapy.org)"
1073 }
1074 }
1075 },
1076 "runsResponseSchema": {
1077 "type": "object",
1078 "properties": {
1079 "data": {
1080 "type": "object",
1081 "properties": {
1082 "id": {
1083 "type": "string"
1084 },
1085 "actId": {
1086 "type": "string"
1087 },
1088 "userId": {
1089 "type": "string"
1090 },
1091 "startedAt": {
1092 "type": "string",
1093 "format": "date-time",
1094 "example": "2025-01-08T00:00:00.000Z"
1095 },
1096 "finishedAt": {
1097 "type": "string",
1098 "format": "date-time",
1099 "example": "2025-01-08T00:00:00.000Z"
1100 },
1101 "status": {
1102 "type": "string",
1103 "example": "READY"
1104 },
1105 "meta": {
1106 "type": "object",
1107 "properties": {
1108 "origin": {
1109 "type": "string",
1110 "example": "API"
1111 },
1112 "userAgent": {
1113 "type": "string"
1114 }
1115 }
1116 },
1117 "stats": {
1118 "type": "object",
1119 "properties": {
1120 "inputBodyLen": {
1121 "type": "integer",
1122 "example": 2000
1123 },
1124 "rebootCount": {
1125 "type": "integer",
1126 "example": 0
1127 },
1128 "restartCount": {
1129 "type": "integer",
1130 "example": 0
1131 },
1132 "resurrectCount": {
1133 "type": "integer",
1134 "example": 0
1135 },
1136 "computeUnits": {
1137 "type": "integer",
1138 "example": 0
1139 }
1140 }
1141 },
1142 "options": {
1143 "type": "object",
1144 "properties": {
1145 "build": {
1146 "type": "string",
1147 "example": "latest"
1148 },
1149 "timeoutSecs": {
1150 "type": "integer",
1151 "example": 300
1152 },
1153 "memoryMbytes": {
1154 "type": "integer",
1155 "example": 1024
1156 },
1157 "diskMbytes": {
1158 "type": "integer",
1159 "example": 2048
1160 }
1161 }
1162 },
1163 "buildId": {
1164 "type": "string"
1165 },
1166 "defaultKeyValueStoreId": {
1167 "type": "string"
1168 },
1169 "defaultDatasetId": {
1170 "type": "string"
1171 },
1172 "defaultRequestQueueId": {
1173 "type": "string"
1174 },
1175 "buildNumber": {
1176 "type": "string",
1177 "example": "1.0.0"
1178 },
1179 "containerUrl": {
1180 "type": "string"
1181 },
1182 "usage": {
1183 "type": "object",
1184 "properties": {
1185 "ACTOR_COMPUTE_UNITS": {
1186 "type": "integer",
1187 "example": 0
1188 },
1189 "DATASET_READS": {
1190 "type": "integer",
1191 "example": 0
1192 },
1193 "DATASET_WRITES": {
1194 "type": "integer",
1195 "example": 0
1196 },
1197 "KEY_VALUE_STORE_READS": {
1198 "type": "integer",
1199 "example": 0
1200 },
1201 "KEY_VALUE_STORE_WRITES": {
1202 "type": "integer",
1203 "example": 1
1204 },
1205 "KEY_VALUE_STORE_LISTS": {
1206 "type": "integer",
1207 "example": 0
1208 },
1209 "REQUEST_QUEUE_READS": {
1210 "type": "integer",
1211 "example": 0
1212 },
1213 "REQUEST_QUEUE_WRITES": {
1214 "type": "integer",
1215 "example": 0
1216 },
1217 "DATA_TRANSFER_INTERNAL_GBYTES": {
1218 "type": "integer",
1219 "example": 0
1220 },
1221 "DATA_TRANSFER_EXTERNAL_GBYTES": {
1222 "type": "integer",
1223 "example": 0
1224 },
1225 "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
1226 "type": "integer",
1227 "example": 0
1228 },
1229 "PROXY_SERPS": {
1230 "type": "integer",
1231 "example": 0
1232 }
1233 }
1234 },
1235 "usageTotalUsd": {
1236 "type": "number",
1237 "example": 0.00005
1238 },
1239 "usageUsd": {
1240 "type": "object",
1241 "properties": {
1242 "ACTOR_COMPUTE_UNITS": {
1243 "type": "integer",
1244 "example": 0
1245 },
1246 "DATASET_READS": {
1247 "type": "integer",
1248 "example": 0
1249 },
1250 "DATASET_WRITES": {
1251 "type": "integer",
1252 "example": 0
1253 },
1254 "KEY_VALUE_STORE_READS": {
1255 "type": "integer",
1256 "example": 0
1257 },
1258 "KEY_VALUE_STORE_WRITES": {
1259 "type": "number",
1260 "example": 0.00005
1261 },
1262 "KEY_VALUE_STORE_LISTS": {
1263 "type": "integer",
1264 "example": 0
1265 },
1266 "REQUEST_QUEUE_READS": {
1267 "type": "integer",
1268 "example": 0
1269 },
1270 "REQUEST_QUEUE_WRITES": {
1271 "type": "integer",
1272 "example": 0
1273 },
1274 "DATA_TRANSFER_INTERNAL_GBYTES": {
1275 "type": "integer",
1276 "example": 0
1277 },
1278 "DATA_TRANSFER_EXTERNAL_GBYTES": {
1279 "type": "integer",
1280 "example": 0
1281 },
1282 "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
1283 "type": "integer",
1284 "example": 0
1285 },
1286 "PROXY_SERPS": {
1287 "type": "integer",
1288 "example": 0
1289 }
1290 }
1291 }
1292 }
1293 }
1294 }
1295 }
1296 }
1297 }
1298}ScrapyFy OpenAPI definition
OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for ScrapyFy from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients:
Developer
catMaintained by Community
Categories
