Instagram Comments Scraper
Pricing
from $2.00 / 1,000 results
Extract Instagram comments including text, likes, and author details from post pages. Powered by Instagram Scraper.
Pricing
from $2.00 / 1,000 results
Rating
0.0
(0)
Developer
Nifty
Maintained by CommunityActor stats
0
Bookmarked
38
Total users
6
Monthly active users
18 days ago
Last modified
Categories
Share
{ "actorSpecification": 1, "name": "instagram-comments-scraper", "title": "Instagram Comments of a Post Scraper", "version": "3.3", "dockerfile": "./Dockerfile", "input": "./INPUT_SCHEMA.json", "storages": { "dataset": { "actorSpecification": 1, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table" } } } } }}FROM apify/actor-node-playwright-chrome:20
COPY package.json ./RUN npm install --omit=dev --omit=optional
COPY . ./
CMD npm start{ "name": "instagram-comments-scraper", "version": "3.3", "type": "module", "main": "src/main.js", "scripts": { "start": "node src/main.js" }, "dependencies": { "crawlee": "^3.12.0", "playwright": "^1.49.0", "apify": "^3.2.0", "user-agents": "^1.1.0", "playwright-extra": "^4.3.0", "puppeteer-extra-plugin-stealth": "^2.11.0" }}{ "app": { "name": "Instagram Scraper", "targetName": "Instagram", "targetURL": "https://instagram.com", "id": "instagram-scraper", "title": "I-Social Scraper: Export Posts, Profiles, Reels & More", "description": "Instagram Scraper: Extract Instagram comments, stories, followers & more. Export to CSV, JSON, XLSX with ease.", "version": "3.3.2", "backend": "https://nifty.codes", "hostnames": [ "instagram.com" ], "matches": [ "*://*.instagram.com/*" ], "status": "Published", "colors": { "background": "#f80261", "elements": "#ffffff" }, "longDescription": "This extension provides a comprehensive toolkit for extracting publicly available data from Instagram profiles, posts, reels, stories, and comments directly within your browser. All data collection and processing happens locally on your device. Built-in monitoring tools let you track page changes and receive alerts over time.\n\nKey Features\n✓ Monitor pages for changes and receive alerts via webhook or ntfy.sh\n✓ All scraping and exports processed locally in your browser\n✓ Export to CSV, Excel (XLSX), JSON, HTML Table, and Raw JSON\n✓ Automatic pagination for feeds, follower lists, and comment threads\n✓ Batch processing for multiple URLs in a single session\n✓ Customizable field selection and conditional formatting\n\nWhat Data You Can Extract\n\nPosts and Reels: Author details, post text, media URLs, like and comment counts, timestamps, and accessibility captions. Works on home feeds, profile grids, reels tabs, saved posts, and tagged posts.\n\nPost Details: Engagement metrics (likes, comments, views), location data, audio information, tagged users, carousel content, duration, and paid partnership status.\n\nComments: Comment text, timestamps, like counts, reply hierarchies, author usernames, and verification status.\n\nStories: Media URLs, expiration times, story type, accessibility captions, and author information.\n\nProfiles: Username, full name, biography, follower and following counts, post count, verification status, privacy status, business category, and external links.\n\nFollowers and Following: Usernames, full names, profile URLs, verification and privacy status, profile pictures, and latest story timestamps.\n\nMonitoring and Alerts\nPage Watcher lets you track changes on any Instagram page and get notified when something updates. Set conditions on specific fields you care about, such as follower count or new posts, and receive alerts via webhook or ntfy.sh when those conditions are met.\n\nBatch Processing\nBatch URL Opener allows you to queue multiple Instagram URLs and scrape them all in one session. Add URLs manually or auto-collect them from a list page, then batch-process every link and export the combined results.\n\nExport Formats\n- CSV\n- Excel (XLSX)\n- JSON\n- HTML Table\n- Raw JSON\n\nPerfect For\nThis tool is ideal for archiving social media content, tracking public engagement metrics, and compiling profile data from Instagram. It suits researchers, marketers, and anyone who needs structured exports of publicly visible information.\n\nSupported Regions\nThis extension works with Instagram globally and is compatible across all regions where the platform is accessible.\n\nPrivacy and Security\nAll scraping and data exports are processed locally on your machine. General usage metrics are collected to help improve the extension, but no scraped data is transmitted to external servers.\n\nUsage Notice\nThis extension provides tools for extracting publicly available data. Users are responsible for ensuring their use complies with applicable terms of service and local regulations. The developer provides the tool and is not liable for how it is used.\n\nTechnical Support\nFor assistance or questions, contact hello@nifty.codes.\n\nTrademark\nInstagram is a registered trademark of Meta Platforms, Inc. This extension is an independent tool and is not affiliated with, endorsed by, authorized, maintained, or sponsored by Meta Platforms, Inc." }, "options": { "posts": { "id": "posts", "title": "Posts in a List", "description": "Works on feeds, profiles, reels, saved posts", "instructions": [ "Go to any page that shows a list of posts (home feed, profile, reels tab, saved, etc.)", "Make sure posts are visible and fully loaded" ], "type": "list", "urlField": "Post URL", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Post Author", "fixes": [], "timeout": 10000, "fields": [ "Post Author", "Post Author Full Name", "Post Author Image", "Post Author URL", "Post Author Is Verified", "Post Type", "Post Text", "Post Image", "Post Video", "Post Likes", "Post Comments Count", "Post URL", "Post Date", "Is Comments Disabled", "Post Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "media_notes" ] }, "condition": [ { "type": "url", "operator": "notContains", "value": "/explore/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "clips_tab_pinned_user_ids" ] }, "condition": [ { "type": "url", "operator": "notContains", "value": "/explore/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "sponsor_tags" }, "parse": true } ], "condition": [ { "type": "url", "operator": "notContains", "value": "/explore/" } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "xdt_api__v1__feed__user_timeline_graphql_connection" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "media_count" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "clips_tab_pinned_user_ids" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "url", "operator": "contains", "value": "/reels/" }, { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 3, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 3, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "isFetching", "posts" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "url", "operator": "contains", "value": "/saved/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "log": false, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "xdt_fbsearch__top_serp_graphql" ], "limit": 1, "ancestorLevel": 0, "maxBreadth": 20 }, "condition": [ { "type": "url", "operator": "contains", "value": "/explore/" } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href*='/p/{{{id2}}}/'], a[href*='/reel/{{{id2}}}/']" }, { "method": "closest", "query": "article" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "preset", "query": "window" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] }, "condition": [ { "type": "url", "operator": "endsWith", "value": "instagram.com/" } ] }, "postDetails": { "id": "postDetails", "title": "Post Details", "description": "Works on individual post or reel pages", "instructions": [ "Open a specific post or reel on Instagram", "Wait for all the details to fully load" ], "type": "detail", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Caption", "fixes": [], "timeout": 10000, "fields": [ "Post Type", "Caption", "Like Count", "Comment Count", "View Count", "Author Username", "Author Full Name", "Author ID", "Author URL", "Author Is Verified", "Author Profile Pic", "Post URL", "Image URL", "Video URL", "Cover Image", "Posted At", "Location", "Duration", "Has Audio", "Audio Title", "Audio Artist", "Tagged Users", "Is Carousel", "Carousel Count", "Original Width", "Original Height", "Is Paid Partnership", "Comments Disabled", "Likes Disabled", "Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1000, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 0, "attempts": 1, "searchConfig": { "data": [ "coauthor_producers" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 0, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "coauthor_producers" ], "limit": 1, "ancestorLevel": 1 } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 0, "attempts": 1, "searchConfig": { "data": [ "clips_metadata" ] }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 0, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "clips_metadata" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "window", "delay": 0, "attempts": 1, "searchConfig": { "nestedKey": "location.href" }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels" } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "article[role='presentation']" } ] }, "paginate": { "delay": 2500, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "preset", "query": "window" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, "comments": { "id": "comments", "title": "Comments of a Post", "description": "Works on post pages with comments", "instructions": [ "Go to a post page on Instagram", "Scroll down to the comments section", "Make sure comments are visible on the page" ], "type": "list", "urlField": "Author URL", "unit": { "singular": "Comment", "plural": "Comments" }, "frontField": "Comment Text", "fixes": [], "timeout": 10000, "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "comments" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "comment_like_count" }, "parse": true } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "time[datetime*='{{{id2}}}']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "article ul" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "article ul" } ] } ] }, { "type": "click", "target": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" }, { "method": "closest", "query": "button" } ], "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" } ] } ] }, { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "svg[aria-label='Comment']" }, { "method": "closest", "query": "section" }, { "method": "parentElement", "query": "" }, { "method": "parentElement", "query": "" }, { "method": "querySelector", "query": ":scope > div:nth-child(3)" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "stories": { "id": "stories", "title": "Stories", "description": "Works on stories viewer pages", "instructions": [ "Go to a user's stories on Instagram", "Make sure the story is playing or visible" ], "type": "list", "urlField": "Story Link", "unit": { "singular": "Story", "plural": "Stories" }, "frontField": "Story Date", "fixes": [], "timeout": 10000, "fields": [ "Story Type", "Story Image", "Story Video", "Story Link", "Story Date", "Story Expiring At", "Story Accessibility Caption", "Author Username", "Author Full Name", "Author Image", "Author URL", "Author Is Verified" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "story_cta" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "story_cta" }, "parse": true } ] } ] } } ], "log": false }, "condition": [ { "type": "url", "operator": "contains", "value": "stories" } ] }, "profile": { "id": "profile", "title": "Profile Details", "description": "Works on user profile pages", "instructions": [ "Navigate to a user's profile page on Instagram", "Make sure their profile information is visible" ], "type": "detail", "unit": { "singular": "Profile", "plural": "Profiles" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Biography", "Profile URL", "External URL", "Follower Count", "Following Count", "Post Count", "Is Verified", "Is Private", "Is Business", "Category", "Profile Picture URL", "All Bio Links" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/graphql/query", "data": [ "follower_count", "following_count", "biography" ] } } ] } }, { "delay": 0, "attempts": 1, "attemptLimit": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "following_count", "biography" ], "limit": 1, "ancestorLevel": 1 } } ] } } ], "log": false } }, "followers": { "id": "followers", "title": "Followers of a User", "description": "Works on the followers popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the followers count to open the followers list", "Make sure the followers list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Follower", "plural": "Followers" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/followers/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "following": { "id": "following", "title": "Following of a User", "description": "Works on the following popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the following count to open the following list", "Make sure the following list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Account", "plural": "Accounts" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/following/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } } }, "tests": [ { "option_id": "posts", "title": "Test Posts on Home Feed", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/", "url_updated_at": "2026-04-13T16:45:46.350Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts on Profile", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T16:49:39.489Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Saved", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/biketonin/saved/all-posts/", "url_updated_at": "2026-05-12T04:23:50.105Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Reels Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/reels/", "https://www.instagram.com/markrober/reels/", "https://www.instagram.com/cristiano/reels/" ], "url_updated_at": "2026-04-13T16:51:58.657Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Tagged Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/tagged/", "https://www.instagram.com/cristiano/tagged/", "https://www.instagram.com/markrober/tagged/" ], "url_updated_at": "2026-04-13T16:53:01.569Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "postDetails", "title": "Test Post Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T16:54:49.214Z" } }, { "action": "select_option", "data": { "optionId": "postDetails" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "postDetails" } }, { "action": "cancel_option", "data": { "optionId": "postDetails" } } ] }, { "option_id": "comments", "title": "Test Post Comments", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T17:15:44.201Z" } }, { "action": "select_option", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "comments" } }, { "action": "cancel_option", "data": { "optionId": "comments" } } ] }, { "option_id": "stories", "title": "Test Stories", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/stories/highlights/17929440964811872/", "https://www.instagram.com/stories/highlights/17942782034297155/" ], "url_updated_at": "2026-04-13T17:18:12.361Z" } }, { "action": "select_option", "data": { "optionId": "stories" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "stories" } }, { "action": "cancel_option", "data": { "optionId": "stories" } } ] }, { "option_id": "profile", "title": "Test Profile Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:25:37.449Z" } }, { "action": "select_option", "data": { "optionId": "profile" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "profile" } }, { "action": "cancel_option", "data": { "optionId": "profile" } } ] }, { "option_id": "followers", "title": "Test Followers of a User", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:28:55.288Z" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "click", "data": { "selector": "a[href*='/followers/']" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "select_option", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "followers" } }, { "action": "cancel_option", "data": { "optionId": "followers" } } ] } ]}1// Browser-side scraping engine — injected into page via page.evaluate()2// All methods run in the real browser context (not Node.js)3
4window.scrapingEngine = {5
6 _shouldSkip: function(obj) {7 try {8 if (obj instanceof Date || obj instanceof RegExp || obj instanceof Error) return true;9 // Check for Window objects (including cross-origin)10 try { if (obj === obj.window) return true; } catch(e) { return true; }11 // Check valueOf — may throw on cross-origin12 try {13 if (typeof obj.valueOf === 'function') {14 var val = obj.valueOf();15 if (val !== obj && (typeof val !== 'object' || val === null)) return true;16 }17 } catch(e) { return true; }18 } catch(e) { return true; }19 return false;20 },21
22 fastDeepSearchField: function(obj, fieldName, opts) {23 opts = opts || {};24 var limit = opts.limit === undefined ? Infinity : opts.limit;25 var maxDepth = opts.maxDepth === undefined ? Infinity : opts.maxDepth;26 var ancestorLevel = opts.ancestorLevel || 0;27 var flexLimit = opts.flexLimit || false;28 var flexLimitThreshold = opts.flexLimitThreshold || 50000;29 var maxBreadth = opts.maxBreadth || Infinity;30 if (!obj || typeof obj !== 'object') return [];31
32 var self = this;33 var matcher = this.buildMatcher(fieldName);34 var results = [];35 var seen = new WeakSet();36 var stack = [[obj, 0, [obj]]];37
38 // flexLimit tracking39 var defaultFlexGap = flexLimitThreshold;40 var flexBuffer = 15000;41 var flexIterations = 0;42 var flexLastMatchAt = 0;43 var flexGaps = [];44
45 while (stack.length > 0 && (flexLimit || results.length < limit)) {46 if (flexLimit) {47 flexIterations++;48 var gap = flexIterations - flexLastMatchAt;49 var sorted = flexGaps.slice().sort(function(a, b) { return a - b; });50 var median = sorted.length > 0 ? sorted[sorted.length >> 1] : 0;51 var threshold = sorted.length > 0 ? (median + flexBuffer) : defaultFlexGap;52 if (gap > threshold) break;53 }54 var item = stack.pop();55 var current = item[0], depth = item[1], path = item[2];56 if (current == null || typeof current !== 'object') continue;57 if (self._shouldSkip(current)) continue;58 try { if (seen.has(current)) continue; } catch(e) { continue; }59 try { seen.add(current); } catch(e) { continue; }60
61 if (matcher(current)) {62 var result = current;63 if (ancestorLevel > 0) {64 var idx = path.length - 1 - ancestorLevel;65 if (idx >= 0) result = path[idx]; else continue;66 }67 results.push(result);68
69 if (flexLimit) {70 var matchGap = flexIterations - flexLastMatchAt;71 if (flexLastMatchAt > 0) flexGaps.push(matchGap);72 flexLastMatchAt = flexIterations;73 }74
75 if (!flexLimit && results.length >= limit) break;76 }77
78 if (depth >= maxDepth) continue;79
80 try {81 if (Array.isArray(current)) {82 var cappedLen = Math.min(current.length, maxBreadth);83 for (var i = cappedLen - 1; i >= 0; i--) {84 var v = current[i];85 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);86 }87 } else {88 var keys;89 try { keys = Object.keys(current); } catch(e) { continue; }90 var cappedKeysLen = Math.min(keys.length, maxBreadth);91 for (var i = cappedKeysLen - 1; i >= 0; i--) {92 var v;93 try { v = current[keys[i]]; } catch(e) { continue; }94 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);95 }96 }97 } catch(e) { }98 }99 return results;100 },101
102 buildMatcher: function(fieldSpec) {103 var self = this;104 if (typeof fieldSpec === 'string') {105 return function(obj) { return fieldSpec in obj; };106 }107 if (Array.isArray(fieldSpec)) {108 if (fieldSpec.length === 0) return function() { return true; };109 if (Array.isArray(fieldSpec[0])) {110 var groups = fieldSpec.map(function(group) { return group.map(function(f) { return self.buildMatcher(f); }); });111 return function(obj) { return groups.some(function(group) { return group.every(function(m) { return m(obj); }); }); };112 }113 var matchers = fieldSpec.map(function(f) { return self.buildMatcher(f); });114 return function(obj) { return matchers.every(function(m) { return m(obj); }); };115 }116 if (typeof fieldSpec === 'object' && fieldSpec.field) {117 var field = fieldSpec.field, value = fieldSpec.value, mode = fieldSpec.mode || 'equal';118 if (value === undefined || value === null) return function(obj) { return field in obj; };119 return function(obj) {120 if (!(field in obj)) return false;121 var v = String(obj[field]).toLowerCase();122 var expected = String(value).toLowerCase();123 return mode === 'includes' ? v.indexOf(expected) !== -1 : v === expected;124 };125 }126 return function() { return false; };127 },128
129 searchReactRoot: function(element, searchConfig) {130 var reactKey = searchConfig.reactKey || '__reactFiber';131 var searchProperty = searchConfig.searchProperty;132 var limit = searchConfig.limit || 1;133 var ancestorLevel = searchConfig.ancestorLevel || 0;134 var maxDepth = searchConfig.maxDepth || Infinity;135 var flexLimit = searchConfig.flexLimit || false;136 var flexLimitThreshold = searchConfig.flexLimitThreshold || 50000;137 var maxBreadth = searchConfig.maxBreadth || Infinity;138 var keys = Object.keys(element);139 for (var i = 0; i < keys.length; i++) {140 if (keys[i].indexOf(reactKey) === 0) {141 return this.fastDeepSearchField(element[keys[i]], searchProperty, { limit: limit, maxDepth: maxDepth, ancestorLevel: ancestorLevel, flexLimit: flexLimit, flexLimitThreshold: flexLimitThreshold, maxBreadth: maxBreadth });142 }143 }144 return null;145 },146
147 DOMSearch: function(searchConfig) {148 if (!Array.isArray(searchConfig)) return [];149 var self = this;150 var results = [];151 for (var q = 0; q < searchConfig.length; q++) {152 var query = searchConfig[q];153 if (!query || !query.key) continue;154 try {155 var elements = self._getElements(query, document);156 if (query.filter) elements = self._filterElements(elements, query.filter);157 if (elements.length === 0) { results.push({ key: query.key, value: null }); continue; }158
159 if (query.nested && query.nested.length > 0) {160 // Nested queries: for each matched element, run sub-queries relative to it161 var nestedResults = self._processNested(elements, query.nested);162 var parsed = query.parse ? nestedResults.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : nestedResults;163 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });164 } else {165 var vals = self._extractValues(elements, query);166 var parsed = query.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;167 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });168 }169 } catch(e) { results.push({ key: query.key, value: null }); }170 }171 return results;172 },173
174 _getElements: function(query, baseEl) {175 var elements = [];176 if (query.type === 'xpath') {177 var xr = document.evaluate(query.query, baseEl, null, query.all ? XPathResult.ORDERED_NODE_ITERATOR_TYPE : XPathResult.FIRST_ORDERED_NODE_TYPE, null);178 if (query.all) { var n = xr.iterateNext(); while (n) { elements.push(n); n = xr.iterateNext(); } }179 else { if (xr.singleNodeValue) elements.push(xr.singleNodeValue); }180 } else if (query.type === 'selector') {181 elements = query.all ? Array.from(baseEl.querySelectorAll(query.query)) : (baseEl.querySelector(query.query) ? [baseEl.querySelector(query.query)] : []);182 } else if (query.type === 'document') {183 elements = [baseEl];184 }185 return elements;186 },187
188 _filterElements: function(elements, filter) {189 if (!filter) return elements;190 return elements.filter(function(el) {191 var text = el.textContent || '';192 if (filter.contains) {193 if (Array.isArray(filter.contains)) { if (!filter.contains.every(function(s) { return text.indexOf(s) !== -1; })) return false; }194 else if (text.indexOf(filter.contains) === -1) return false;195 }196 if (filter.notContains) {197 if (Array.isArray(filter.notContains)) { if (filter.notContains.some(function(s) { return text.indexOf(s) !== -1; })) return false; }198 else if (text.indexOf(filter.notContains) !== -1) return false;199 }200 if (filter.startsWith && !text.trimStart().startsWith(filter.startsWith)) return false;201 if (filter.endsWith && !text.trimEnd().endsWith(filter.endsWith)) return false;202 if (typeof filter.minLength === 'number' && text.length < filter.minLength) return false;203 if (typeof filter.maxLength === 'number' && text.length > filter.maxLength) return false;204 return true;205 });206 },207
208 _extractValues: function(elements, query) {209 var value = query.value || 'text';210 var custom = query.custom || null;211 return elements.map(function(el) {212 if (value === 'text') return el.textContent || '';213 if (value === 'innerText') return el.innerText || '';214 if (value === 'html') return el.innerHTML || '';215 if (value === 'attribute' && custom && custom.attributeKey) return el.getAttribute(custom.attributeKey) || null;216 return el.textContent || '';217 });218 },219
220 _processNested: function(baseElements, nestedQueries) {221 var self = this;222 var results = [];223 for (var i = 0; i < baseElements.length; i++) {224 var baseEl = baseElements[i];225 var elementResult = {};226 for (var n = 0; n < nestedQueries.length; n++) {227 var nq = nestedQueries[n];228 if (!nq || !nq.key) continue;229 try {230 var childElements = self._getElements(nq, baseEl);231 if (nq.filter) childElements = self._filterElements(childElements, nq.filter);232 if (childElements.length === 0) { elementResult[nq.key] = null; continue; }233
234 if (nq.nested && nq.nested.length > 0) {235 var deepNested = self._processNested(childElements, nq.nested);236 var parsed = nq.parse ? deepNested.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : deepNested;237 elementResult[nq.key] = nq.all ? parsed : parsed[0] || null;238 } else {239 var vals = self._extractValues(childElements, nq);240 var parsed = nq.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;241 elementResult[nq.key] = nq.all ? parsed : vals[0] || null;242 }243 } catch(e) { elementResult[nq.key] = null; }244 }245 results.push(elementResult);246 }247 return results;248 }249};1// Anti-detection enhancements — applied in preNavigationHooks and postNavigationHooks2
3import UserAgent from 'user-agents';4
5const TRACKER_DOMAINS = [6 'google-analytics.com', 'googletagmanager.com', 'facebook.net',7 'doubleclick.net', 'hotjar.com', 'segment.com', 'mixpanel.com',8 'amplitude.com', 'fullstory.com', 'clarity.ms', 'newrelic.com',9 'sentry.io', 'bugsnag.com', 'datadoghq.com', 'mouseflow.com',10 'crazyegg.com', 'optimizely.com', 'hubspot.com', 'intercom.io',11 'drift.com', 'tawk.to', 'livechatinc.com'12];13
14const COOKIE_SELECTORS = [15 '[data-testid="cookie-policy-dialog-accept-button"]',16 'button[id*="accept"]', 'button[class*="accept"]',17 'button[class*="consent"]', '#onetrust-accept-btn-handler',18 '.cookie-consent-accept', '[aria-label*="Accept"]',19 '[aria-label*="accept"]', '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',20 '.cc-btn.cc-allow', '#gdpr-cookie-accept', '.js-cookie-consent-agree'21];22
23export async function applyPreNavigation(page, enhancements) {24 if (enhancements.blockTrackers) {25 await page.route('**/*', (route) => {26 const url = route.request().url();27 if (TRACKER_DOMAINS.some(d => url.includes(d))) return route.abort();28 return route.continue();29 });30 }31
32 if (enhancements.rotateUserAgent) {33 const ua = new UserAgent({ deviceCategory: 'desktop' });34 await page.setExtraHTTPHeaders({ 'User-Agent': ua.toString() });35 }36}37
38export async function applyPostNavigation(page, enhancements) {39 if (enhancements.dismissCookies) {40 try {41 for (const sel of COOKIE_SELECTORS) {42 const btn = await page.$(sel);43 if (btn) { await btn.click().catch(() => {}); break; }44 }45 } catch { }46 }47
48 // Post-navigation delay (radio: none / random / custom)49 const postNav = enhancements.postNavigation || 'none';50 if (postNav === 'random') {51 await page.waitForTimeout(1000 + Math.floor(Math.random() * 2000));52 } else if (postNav === 'custom') {53 const delay = typeof enhancements.postNavigationDelay === 'number' ? enhancements.postNavigationDelay : 2000;54 await page.waitForTimeout(delay);55 }56}1import { log } from 'crawlee';2
3/**4 * Execute a custom user script in the browser context.5 * The script should define: async function main(context) { ... }6 * context = { url, hostname, pathname, searchParams, optionKey, timing }7 */8export async function executeCustomScript(page, enhancements, context) {9 const code = enhancements.customScriptCode;10 if (!code) return;11
12 try {13 log.info('Custom script: executing (' + context.timing + ')');14 await page.evaluate(async ({ scriptCode, ctx }) => {15 const fn = new Function('context', `return (async function() { ${scriptCode} }).call(null)`);16 await fn(ctx);17 }, { scriptCode: code, ctx: context });18 log.info('Custom script: completed');19 } catch (err) {20 log.warning('Custom script error: ' + err.message);21 }22}23
24/**25 * Execute a beforeLoad custom script in Node context.26 * Can return a modified URL string.27 */28export function executeBeforeLoadScript(enhancements, url) {29 const code = enhancements.customScriptCode;30 if (!code) return url;31
32 try {33 const parsed = new URL(url);34 const context = {35 url,36 hostname: parsed.hostname,37 pathname: parsed.pathname,38 searchParams: Object.fromEntries(parsed.searchParams),39 timing: 'beforeLoad'40 };41 const fn = new Function('context', code);42 const result = fn(context);43 if (typeof result === 'string') {44 log.info('Custom script (beforeLoad): URL modified to ' + result);45 return result;46 }47 } catch (err) {48 log.warning('Custom script (beforeLoad) error: ' + err.message);49 }50 return url;51}1// Data processing enhancements — raw data, criteria filtering2
3export function formatForDataset(item, enhancements) {4 if (enhancements.rawData) {5 return { ...item.formatted, _raw: item.raw };6 }7 return item.formatted || item;8}9
10// ── Criteria filtering ───────────────────────────────────────11
12export function checkCondition(value, condition) {13 const { operator, value: target, caseSensitive } = condition;14 const str = String(value ?? '');15 const cmp = caseSensitive ? str : str.toLowerCase();16 const tgt = caseSensitive ? target : target.toLowerCase();17 switch (operator) {18 case 'equals': return cmp === tgt;19 case 'notEquals': return cmp !== tgt;20 case 'contains': return cmp.includes(tgt);21 case 'notContains': return !cmp.includes(tgt);22 case 'regex': try { return new RegExp(target, caseSensitive ? '' : 'i').test(str); } catch { return false; }23 case 'greaterThan': return parseFloat(str) > parseFloat(target);24 case 'lessThan': return parseFloat(str) < parseFloat(target);25 case 'dateBefore': return new Date(str) < new Date(target);26 case 'dateOn': return new Date(str).toDateString() === new Date(target).toDateString();27 case 'dateAfter': return new Date(str) > new Date(target);28 default: return true;29 }30}31
32export function applyCriteria(items, criteria) {33 if (!criteria || typeof criteria !== 'object' || Object.keys(criteria).length === 0) return items;34 return items.filter(item => {35 for (const [field, conditions] of Object.entries(criteria)) {36 if (!Array.isArray(conditions)) continue;37 const val = item[field];38 for (const cond of conditions) {39 if (!checkCondition(val, cond)) return false;40 }41 }42 return true;43 });44}1// Pagination enhancement — executes scroll/click pagination from config2// Mirrors extension's AutoPaginationManager: data-driven stopping with retry attempts3
4export async function runPagination(page, option, enhancements, processInBrowser) {5 if (!enhancements.pagination || !option.paginate) return [];6
7 const log = (await import('crawlee')).log;8 const paginateConfig = option.paginate;9 const maxPages = typeof enhancements.maxPages === 'number' ? enhancements.maxPages : 0; // 0 = infinite10 const delay = paginateConfig.delay || 3000;11 const maxAttempts = 2;12 const allResults = [];13
14 let pageNum = 0;15 let attempts = 0;16
17 log.info('Pagination: starting, maxPages=' + (maxPages || 'infinite') + ', delay=' + delay + 'ms, instructions=' + (paginateConfig.instructions || []).length);18
19 while (true) {20 if (maxPages > 0 && pageNum >= maxPages) {21 log.info('Pagination: reached maxPages limit (' + maxPages + ')');22 break;23 }24
25 pageNum++;26 log.info('Pagination: page ' + pageNum + (maxPages > 0 ? '/' + maxPages : '') + ' url=' + page.url());27
28 let acted = await executeInstructions(page, paginateConfig.instructions || [], log);29
30 if (!acted) {31 log.info('Pagination: waiting 8s for page to finish rendering...');32 await page.waitForTimeout(8000);33 acted = await executeInstructions(page, paginateConfig.instructions || [], log);34 }35
36 if (!acted) {37 attempts++;38 log.info('Pagination: nothing executed after wait, attempt ' + attempts + '/' + maxAttempts);39 if (attempts >= maxAttempts) {40 log.info('Pagination: stopping — no pagination action possible');41 break;42 }43 continue;44 }45
46 await page.waitForTimeout(delay);47
48 const newItems = await processInBrowser(page);49
50 log.info('Pagination: page ' + pageNum + ' got ' + newItems.length + ' new items');51
52 if (newItems.length > 0) {53 allResults.push(...newItems);54 attempts = 0;55 log.info('Pagination: total so far ' + allResults.length + ' items, continuing');56 } else {57 attempts++;58 log.info('Pagination: no new data, attempt ' + attempts + '/' + maxAttempts);59 if (attempts >= maxAttempts) {60 log.info('Pagination: stopping after ' + maxAttempts + ' attempts with no new data');61 break;62 }63 }64 }65
66 log.info('Pagination: done, ' + pageNum + ' pages, ' + allResults.length + ' total new items');67 return allResults;68}69
70// Execute pagination instructions (scroll, click, delay) with full target chain support71async function executeInstructions(page, instructions, log) {72 let anyExecuted = false;73
74 for (let i = 0; i < instructions.length; i++) {75 const instruction = instructions[i];76 try {77 if (instruction.condition) {78 const condMet = await checkCondition(page, instruction.condition);79 if (!condMet) {80 log.info('Pagination: instruction[' + i + '] condition not met, skipping');81 continue;82 }83 }84
85 anyExecuted = true;86
87 if (instruction.type === 'delay') {88 await page.waitForTimeout(instruction.amount || 1000);89 } else if (instruction.type === 'scroll') {90 await executeScroll(page, instruction, log);91 } else if (instruction.type === 'click') {92 await executeClick(page, instruction, log);93 }94 } catch (err) {95 log.info('Pagination: instruction[' + i + '] error: ' + err.message);96 }97 }98
99 if (!anyExecuted) {100 log.info('Pagination: all conditions failed, no action taken');101 }102
103 return anyExecuted;104}105
106// Scroll with full support: scrollTo bottom, scrollBy, steps, presets, chained targets107async function executeScroll(page, instruction, log) {108 const { scrollType, amount, steps } = instruction;109 const el = await findElement(page, instruction.target);110
111 const stepCount = steps?.amount || 1;112 const stepDelay = steps?.delay || 100;113
114 if (scrollType === 'to' && amount?.type === 'preset' && amount.value === 'bottom') {115 for (let s = 0; s < stepCount; s++) {116 if (el) {117 await el.evaluate((e, args) => {118 e.scrollTo(0, (e.scrollHeight / args.total) * (args.step + 1));119 }, { step: s, total: stepCount });120 } else {121 await page.evaluate(({ step, total }) => {122 window.scrollTo(0, (document.body.scrollHeight / total) * (step + 1));123 }, { step: s, total: stepCount });124 }125 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);126 }127 log.info('Pagination: scrolled to bottom' + (stepCount > 1 ? ' in ' + stepCount + ' steps' : ''));128 } else if (scrollType === 'by') {129 const scrollAmount = amount?.value || 0;130 const stepSize = scrollAmount / stepCount;131 for (let s = 0; s < stepCount; s++) {132 if (el) {133 await el.evaluate((e, amt) => e.scrollBy(0, amt), stepSize);134 } else {135 await page.evaluate((amt) => window.scrollBy(0, amt), stepSize);136 }137 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);138 }139 log.info('Pagination: scrolled by ' + scrollAmount + 'px');140 } else {141 if (el) {142 await el.evaluate(e => e.scrollTo(0, e.scrollHeight));143 } else {144 await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));145 }146 log.info('Pagination: scrolled');147 }148}149
150// Click with chained target support151async function executeClick(page, instruction, log) {152 const target = instruction.target;153 const step = Array.isArray(target) ? target[0] : target;154 const query = step?.query;155 const method = step?.method;156
157 if (query) {158 const clicked = await page.evaluate(({ query, method }) => {159 let el;160 if (method === 'xpath') {161 el = document.evaluate(query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;162 } else {163 el = document.querySelector(query);164 }165 if (el) { el.click(); return true; }166 return false;167 }, { query, method });168
169 if (clicked) {170 log.info('Pagination: clicked');171 } else {172 log.info('Pagination: click target not found');173 }174 } else {175 log.info('Pagination: no click query');176 }177}178
179// Find element with chained target support (matches extension's unified {method, query} format)180async function findElement(page, target) {181 if (!target) return null;182
183 if (!Array.isArray(target)) target = [target];184
185 let el = null;186 for (const step of target) {187 if (!step.method) continue;188
189 if (step.method === 'querySelector') {190 el = el ? await applyMethod(page, el, step) : await page.$(step.query);191 } else if (step.method === 'xpath') {192 el = el ? await applyMethod(page, el, step) : await page.$('xpath=' + step.query);193 } else if (step.method === 'preset') {194 if (['window', 'document'].includes(step.query?.toLowerCase())) return null;195 if (step.query?.toLowerCase() === 'body') el = await page.$('body');196 } else if (el) {197 el = await applyMethod(page, el, step);198 }199 if (!el) return null;200 }201 return el;202}203
204// Apply chained DOM methods (matches extension's Paginater.applyMethod)205async function applyMethod(page, el, step) {206 const { method, query } = step;207 try {208 switch (method) {209 case 'closest':210 return await el.evaluateHandle((e, q) => e.closest(q), query);211 case 'parentElement':212 return await el.evaluateHandle(e => e.parentElement);213 case 'querySelector':214 return await el.evaluateHandle((e, q) => e.querySelector(q), query);215 case 'nextElementSibling':216 return await el.evaluateHandle(e => e.nextElementSibling);217 case 'previousElementSibling':218 return await el.evaluateHandle(e => e.previousElementSibling);219 case 'firstElementChild':220 return await el.evaluateHandle(e => e.firstElementChild);221 case 'lastElementChild':222 return await el.evaluateHandle(e => e.lastElementChild);223 default:224 return null;225 }226 } catch {227 return null;228 }229}230
231async function checkCondition(page, conditions) {232 if (!Array.isArray(conditions)) return true;233 return page.evaluate((conds) => {234 for (const cond of conds) {235 if (cond.type === 'element') {236 const target = cond.value?.[0];237 if (!target) continue;238 let el = null;239 if (target.method === 'querySelector') {240 el = document.querySelector(target.query);241 } else if (target.method === 'xpath') {242 el = document.evaluate(target.query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;243 }244 if (cond.operator === 'exists' && !el) return false;245 if (cond.operator === 'not_exists' && el) return false;246 } else if (cond.type === 'url') {247 const url = window.location.href;248 if (cond.operator === 'contains' && !url.includes(cond.value)) return false;249 if (cond.operator === 'notContains' && url.includes(cond.value)) return false;250 if (cond.operator === 'startsWith' && !url.startsWith(cond.value)) return false;251 if (cond.operator === 'endsWith' && !url.endsWith(cond.value)) return false;252 if (cond.operator === 'equals' && url !== cond.value) return false;253 if (cond.operator === 'notEquals' && url === cond.value) return false;254 }255 }256 return true;257 }, conditions);258}1// Performance enhancements — resource blocking, crawler config, session pool2
3export async function applyResourceBlocking(page, enhancements) {4 const blockTypes = [];5 if (enhancements.blockImages) blockTypes.push('image');6 if (enhancements.blockMedia) blockTypes.push('media', 'font');7 if (enhancements.blockStylesheets) blockTypes.push('stylesheet');8
9 if (blockTypes.length > 0) {10 await page.route('**/*', (route) => {11 if (blockTypes.includes(route.request().resourceType())) return route.abort();12 return route.continue();13 });14 }15}16
17export async function getCrawlerOptions(enhancements) {18 const { Actor } = await import('apify');19 const opts = {};20
21 // Max timeout (0 = no limit)22 const timeout = typeof enhancements.maxTimeout === 'number' ? enhancements.maxTimeout : 120;23 opts.requestHandlerTimeoutSecs = timeout === 0 ? 999999 : timeout;24
25 // Max concurrent tabs — use autoscaling instead of hard concurrency26 const maxTabs = typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1;27 opts.maxConcurrency = maxTabs;28 opts.minConcurrency = 1;29 if (maxTabs > 1) {30 opts.autoscaledPoolOptions = {31 desiredConcurrencyRatio: 0.85,32 scaleUpStepRatio: 0.5,33 scaleDownStepRatio: 0.2,34 snapshotterOptions: {35 maxBlockedMillis: 2000,36 }37 };38 }39
40 // Session pool — proper Apify SessionPool integration41 if (enhancements.sessionPool) {42 const maxUsage = typeof enhancements.sessionMaxUsage === 'number' ? enhancements.sessionMaxUsage : 5;43 opts.useSessionPool = true;44 opts.persistCookiesPerSession = true;45 opts.sessionPoolOptions = {46 maxPoolSize: 20,47 sessionOptions: {48 maxAgeSecs: 300,49 maxUsageCount: maxUsage,50 }51 };52 }53
54 // Proxy configuration55 if (enhancements.proxyApifyResidential) {56 opts.proxyConfiguration = await Actor.createProxyConfiguration({57 groups: ['RESIDENTIAL']58 });59 } else if (enhancements.proxyApifyDatacenter) {60 opts.proxyConfiguration = await Actor.createProxyConfiguration();61 }62
63 return opts;64}{ "stealth": true, "waitUntil": "domcontentloaded", "postNavigation": "none", "navigationTimeout": 45, "maxTimeout": 0, "retryNavigation": 0, "blockImages": true, "blockMedia": true, "blockStylesheets": true, "blockTrackers": true, "actorMemory": 2048, "maxTabs": 3, "reuseContext": true, "retryIfEmpty": true, "retryMaxAttempts": 2, "abortOnLoss": true, "abortWarmupUrls": 5, "abortWarmupSeconds": 60, "_optionKey": "comments"}1// DOM scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 return window.scrapingEngine.DOMSearch(searchConfig);8 }, method.searchConfig);9
10 return rawData;11}1// React fiber scraping handler2
3export async function scrape(page, option, enhancements) {4 // Extract the single method from the synthetic option5 const method = option.scraping.partials[0].methods.A[0];6 const sc = method.searchConfig;7
8 const rawData = await page.evaluate(({ selector, searchConfig }) => {9 const el = document.querySelector(selector);10 if (!el) return null;11 return window.scrapingEngine.searchReactRoot(el, searchConfig);12 }, { selector: sc.reactSelector, searchConfig: sc });13
14 return rawData;15}1// Network request interception handler2import { log } from 'crawlee';3
4export async function scrape(page, option, enhancements, capturedResponses) {5 if (capturedResponses.length === 0) return null;6
7 // Use per-page processed timestamps stored on the array itself8 const processedTimestamps = capturedResponses._processedTimestamps || (capturedResponses._processedTimestamps = new Set());9
10 // Filter out already-processed responses11 const newResponses = capturedResponses.filter(r => r._timestamp && !processedTimestamps.has(r._timestamp));12 if (newResponses.length === 0) return null;13
14 // Mark as processed15 for (const r of newResponses) {16 processedTimestamps.add(r._timestamp);17 }18
19 return newResponses.map(r => {20 const { _timestamp, ...data } = r;21 return data;22 });23}24
25// Called from main.js preNavigationHooks to set up response listeners before navigation26export function setupInterception(page, option, capturedResponses) {27 const searchConfigs = [];28 for (const partial of option.scraping?.partials || []) {29 for (const [, methods] of Object.entries(partial.methods || {})) {30 for (const method of methods) {31 if (method.type === 'request') searchConfigs.push(method.searchConfig);32 }33 }34 }35
36 page.on('response', async (response) => {37 const url = response.url();38 for (const sc of searchConfigs) {39 const matchUrl = sc.url ? url.includes(sc.url) : true;40 if (!matchUrl) continue;41 try {42 const body = await response.text();43 let json;44 try { json = JSON.parse(body); } catch { continue; }45 const dataMatch = sc.data46 ? (Array.isArray(sc.data) ? sc.data.every(d => body.includes(d)) : body.includes(sc.data))47 : true;48 if (dataMatch) {49 json._timestamp = Date.now();50 capturedResponses.push(json);51 }52 } catch { }53 }54 });55}1// Window object scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 const nestedKey = searchConfig.nestedKey;8 const keys = Array.isArray(nestedKey) ? nestedKey : [nestedKey];9 const results = [];10 for (const key of keys) {11 let value = window;12 for (const part of key.split('.')) {13 if (value == null) break;14 value = value[part];15 }16 if (value != null) {17 if (Array.isArray(value)) results.push(...value);18 else results.push(value);19 }20 }21 return results.length > 0 ? results : null;22 }, method.searchConfig);23
24 return rawData;25}1import { PlaywrightCrawler, Dataset, log, KeyValueStore } from 'crawlee';2import { Actor } from 'apify';3import { fileURLToPath } from 'url';4import { dirname, join } from 'path';5import { readFileSync } from 'fs';6
7import { applyPreNavigation, applyPostNavigation } from './enhancements/anti-detection.js';8import { applyResourceBlocking, getCrawlerOptions } from './enhancements/performance.js';9import { runPagination } from './enhancements/pagination.js';10import { formatForDataset } from './enhancements/data.js';11import { executeCustomScript, executeBeforeLoadScript } from './enhancements/custom-script.js';12
13// Import ALL handlers14import * as reactHandler from './handlers/react.js';15import * as requestHandler from './handlers/request.js';16import * as domHandler from './handlers/dom.js';17import * as windowHandler from './handlers/window.js';18
19const handlerMap = {20 react: reactHandler,21 request: requestHandler,22 dom: domHandler,23 window: windowHandler24};25
26const __dirname = dirname(fileURLToPath(import.meta.url));27
28// ── Load config files ────────────────────────────────────────────29const config = JSON.parse(readFileSync(join(__dirname, 'config.json'), 'utf-8'));30const enhancements = JSON.parse(readFileSync(join(__dirname, 'enhancements.json'), 'utf-8'));31
32// ── Debug flags (set via test tab debug toggles) ─────────────────33const dbg = enhancements._debug || {};34const D = {35 config: dbg.config || false,36 navigation: dbg.navigation || false,37 injection: dbg.injection || false,38 partials: dbg.partials || false,39 rawData: dbg.rawData || false,40 processing: dbg.processing || false,41 pagination: dbg.pagination || false,42 requests: dbg.requests || false,43 browser: dbg.browser || false,44 performance: dbg.performance || false,45 screenshot: dbg.screenshot || false,46};47
48// Read script.js and engine.js as strings — they run in the BROWSER, not Node.js49const scriptCode = readFileSync(join(__dirname, 'script.js'), 'utf-8');50const engineCode = readFileSync(join(__dirname, 'engine.js'), 'utf-8');51
52const optionKey = enhancements._optionKey;53const option = config.options[optionKey];54if (!option) {55 log.error('Option "' + optionKey + '" not found in config');56 process.exit(1);57}58
59if (D.config) {60 log.info('DEBUG[config] optionKey=' + optionKey);61 log.info('DEBUG[config] enhancements=' + JSON.stringify(Object.keys(enhancements).filter(k => !k.startsWith('_') && enhancements[k])));62 log.info('DEBUG[config] hostnames=' + JSON.stringify(config.app.hostnames));63 log.info('DEBUG[config] fields=' + JSON.stringify(option.fields?.length || 0));64}65
66const hostnames = config.app.hostnames || [];67
68function validateUrl(url) {69 try {70 const parsed = new URL(url);71 return hostnames.some(h => parsed.hostname === h || parsed.hostname.endsWith('.' + h));72 } catch {73 return false;74 }75}76
77// Check if any partial uses request interception78function hasRequestMethods(opt) {79 for (const partial of opt.scraping?.partials || []) {80 for (const methods of Object.values(partial.methods || {})) {81 for (const method of methods) {82 if (method.type === 'request') return true;83 }84 }85 }86 return false;87}88
89const needsRequestInterception = hasRequestMethods(option);90
91function getPartialMethodType(partial) {92 if (!partial?.methods) return 'unknown';93 for (const methods of Object.values(partial.methods)) {94 for (const method of methods) {95 if (method.type) return method.type;96 }97 }98 return 'unknown';99}100
101// ── Signature tracking (mirrors extension's processedSignatures) ──────102
103let processedSignatures = new Set();104
105function hashString(str) {106 let hash = 0;107 for (let i = 0; i < str.length; i++) {108 const char = str.charCodeAt(i);109 hash = ((hash << 5) - hash) + char;110 hash = hash & hash;111 }112 return Math.abs(hash).toString(16).padStart(8, '0');113}114
115function generateSignature(data) {116 const seen = new WeakSet();117 return hashString(JSON.stringify(data, function(key, value) {118 if (typeof value === 'object' && value !== null) {119 if (seen.has(value)) return '[Circular]';120 seen.add(value);121 }122 return value;123 }));124}125
126// ── Build enhancement overrides for a retry attempt ──────────────127
128function getRetryEnhancements(attempt) {129 if (!enhancements.retryIfEmpty || !enhancements.retryEnhancements) return enhancements;130 const overrides = enhancements.retryEnhancements[attempt];131 if (!overrides) return enhancements;132 return { ...enhancements, ...overrides };133}134
135// ── Inject code into browser ─────────────────────────────────────136
137async function injectIntoPage(page) {138 if (D.injection) log.info('DEBUG[injection] Injecting engine.js and script.js into page');139 await page.evaluate((code) => {140 if (!window.scrapingEngine) new Function(code)();141 }, engineCode);142
143 await page.evaluate((code) => {144 if (!window.__scriptInjected) {145 new Function(code)();146 window.__scriptInjected = true;147 }148 }, scriptCode);149
150 if (D.browser) {151 const info = await page.evaluate((optKey) => {152 const fns = Object.keys(window).filter(k => { try { return typeof window[k] === 'function'; } catch { return false; } });153 const customFns = fns.filter(k => !['addEventListener','removeEventListener','dispatchEvent','eval','isFinite','isNaN','parseFloat','parseInt','decodeURI','decodeURIComponent','encodeURI','encodeURIComponent','escape','unescape','toString','valueOf'].includes(k) && !k.startsWith('on') && !k.startsWith('webkit') && k.length > 2);154 return {155 processFnExists: typeof window[optKey] === 'function',156 searchField: typeof window.searchField,157 scrapingEngine: typeof window.scrapingEngine,158 scriptInjected: !!window.__scriptInjected,159 customFnCount: customFns.length,160 sampleFns: customFns.slice(0, 15).join(',')161 };162 }, optionKey);163 log.info('DEBUG[browser] ' + JSON.stringify(info));164 }165
166 if (D.injection) log.info('DEBUG[injection] Done');167}168
169// ── Execute a single method (mirrors controller.js executeMethod) ──170
171async function executeMethod(page, method, capturedResponses) {172 const type = method.type;173 const handler = handlerMap[type];174 if (!handler) return { success: false, error: 'Unknown method type: ' + type };175
176 if (D.partials) log.info('DEBUG[partials] Executing method type=' + type + ' delay=' + (method.delay || 0) + ' attempts=' + (method.attempts || 1));177
178 try {179 let rawData;180 if (type === 'request') {181 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements, capturedResponses);182 } else {183 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements);184 }185
186 if (rawData && (Array.isArray(rawData) ? rawData.length > 0 : true)) {187 if (D.rawData) {188 log.info('DEBUG[rawData] type=' + type + ' isArray=' + Array.isArray(rawData) + ' length=' + (Array.isArray(rawData) ? rawData.length : 'N/A'));189 if (Array.isArray(rawData) && rawData.length > 0) {190 const sample = rawData[0];191 log.info('DEBUG[rawData] firstItem type=' + typeof sample + ' keys=' + (typeof sample === 'object' && sample ? Object.keys(sample).slice(0, 8).join(',') : 'N/A'));192 }193 }194 return { success: true, data: rawData };195 }196 if (D.partials) log.info('DEBUG[partials] method type=' + type + ' returned no data');197 return { success: false, error: 'No data returned' };198 } catch (err) {199 if (D.partials) log.error('DEBUG[partials] method type=' + type + ' error: ' + err.message);200 return { success: false, error: err.message };201 }202}203
204// ── Evaluate method conditions in browser context ────────────────205
206async function evaluateConditions(page, conditions) {207 if (!conditions || !Array.isArray(conditions) || conditions.length === 0) return true;208 return page.evaluate((conds) => {209 function evalSingle(cond) {210 const { type, operator, value } = cond;211 if (!type || !operator || value === undefined) return false;212 if (type === 'url') {213 const url = window.location.href;214 switch (operator) {215 case 'equals': return url === value;216 case 'contains': return url.includes(value);217 case 'startsWith': return url.startsWith(value);218 case 'endsWith': return url.endsWith(value);219 case 'regex': try { return new RegExp(value).test(url); } catch { return false; }220 case 'notEquals': return url !== value;221 case 'notContains': return !url.includes(value);222 default: return false;223 }224 }225 if (type === 'element') {226 if (!Array.isArray(value)) return false;227 // Check element existence/visibility using selectors from value array228 for (const v of value) {229 const query = v.query || v;230 if (typeof query !== 'string') continue;231 const el = document.querySelector(query);232 if (operator === 'exists' && !el) return false;233 if (operator === 'not_exists' && el) return false;234 if (operator === 'visible') {235 if (!el) return false;236 const rect = el.getBoundingClientRect();237 const style = window.getComputedStyle(el);238 if (rect.width === 0 || rect.height === 0 || style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;239 }240 }241 return true;242 }243 return false;244 }245
246 function evalGroup(arr) {247 if (!Array.isArray(arr) || arr.length === 0) return true;248 // If first element is array → OR logic249 if (Array.isArray(arr[0])) {250 return arr.some(sub => evalGroup(sub));251 }252 // AND logic253 return arr.every(item => {254 if (Array.isArray(item)) return evalGroup(item);255 if (typeof item === 'object') return evalSingle(item);256 return false;257 });258 }259 return evalGroup(conds);260 }, conditions);261}262
263// ── Process partials (mirrors controller.js processInstructions) ──264
265async function processAllPartials(page, capturedResponses) {266 const scraping = option.scraping;267 if (!scraping?.partials) return [];268
269 const partials = scraping.partials;270 const qualityOrder = ['A', 'B', 'C'];271
272 // Run all partials in parallel273 const partialPromises = partials.map(async (partial, pi) => {274 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] delay=' + (partial.delay || 0) + ' attempts=' + (partial.attempts || 1) + ' group=' + (partial.group || 'none') + ' methods=' + Object.keys(partial.methods || {}).join(','));275 if (partial.delay) await page.waitForTimeout(partial.delay);276
277 const partialAttempts = partial.attempts || 1;278
279 for (let attempt = 0; attempt < partialAttempts; attempt++) {280 for (const quality of qualityOrder) {281 const methods = partial.methods?.[quality];282 if (!methods || !Array.isArray(methods)) continue;283
284 for (const method of methods) {285 if (!method?.type) continue;286
287 // Check method conditions288 if (method.condition && Array.isArray(method.condition)) {289 const condMet = await evaluateConditions(page, method.condition);290 if (!condMet) {291 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] method condition not met, skipping');292 continue;293 }294 }295
296 if (method.delay) await page.waitForTimeout(method.delay);297
298 const methodAttempts = method.attempts || 1;299 for (let mAttempt = 0; mAttempt < methodAttempts; mAttempt++) {300 const result = await executeMethod(page, method, capturedResponses);301 if (result.success) {302 const signature = generateSignature(result.data);303 const alreadyProcessed = processedSignatures.has(signature);304 if (!alreadyProcessed) processedSignatures.add(signature);305 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] signature=' + signature + ' alreadyProcessed=' + alreadyProcessed);306 return { success: true, data: result.data, partialIndex: pi, alreadyProcessed };307 }308 if (mAttempt < methodAttempts - 1) await page.waitForTimeout(500);309 }310 }311 }312
313 if (attempt < partialAttempts - 1) await page.waitForTimeout(500);314 }315
316 return { success: false, partialIndex: pi };317 });318
319 const results = await Promise.all(partialPromises);320
321 // Group logic — mirrors extension's processInstructions322 const groupedPartials = new Map();323 const independentIndices = [];324
325 partials.forEach((partial, index) => {326 const group = partial.group;327 if (group) {328 const groups = Array.isArray(group) ? group : (typeof group === 'string' ? [group] : []);329 if (groups.length > 0) {330 groups.forEach(g => {331 if (!groupedPartials.has(g)) groupedPartials.set(g, []);332 groupedPartials.get(g).push(index);333 });334 } else {335 independentIndices.push(index);336 }337 } else {338 independentIndices.push(index);339 }340 });341
342 // Check group signatures (mirrors extension's processScrapingResults)343 for (const [group, indices] of groupedPartials.entries()) {344 const groupResults = results.filter(r => indices.includes(r?.partialIndex));345 const allSuccess = groupResults.every(r => r?.success);346 if (allSuccess) {347 const groupData = groupResults.reduce((acc, r) => [...acc, ...(Array.isArray(r.data) ? r.data : [r.data])], []);348 const groupSig = generateSignature(groupData);349 if (processedSignatures.has(groupSig)) {350 groupResults.forEach(r => { r.alreadyProcessed = true; });351 if (D.partials) log.info('DEBUG[partials] group "' + group + '" signature already processed');352 } else {353 processedSignatures.add(groupSig);354 }355 }356 }357
358 // Check if at least one group is fully satisfied359 let groupSatisfied = groupedPartials.size === 0;360 for (const [group, indices] of groupedPartials.entries()) {361 const allSuccess = indices.every(i => results[i]?.success);362 if (allSuccess) {363 groupSatisfied = true;364 if (D.partials) log.info('DEBUG[partials] group "' + group + '" satisfied');365 break;366 }367 }368
369 // Check independent partials (at least one must succeed)370 const independentSuccess = independentIndices.length === 0 || independentIndices.some(i => results[i]?.success);371
372 if (!groupSatisfied && !independentSuccess) {373 if (D.partials) log.info('DEBUG[partials] no groups satisfied and no independent successes');374 return [];375 }376
377 // Filter out already-processed results, collect only new data (mirrors extension line 631)378 const newResults = results.filter(r => r?.success && r.data && !r.alreadyProcessed);379 if (D.partials) log.info('DEBUG[partials] ' + results.filter(r => r?.success).length + ' successful partials, ' + newResults.length + ' new (not already processed)');380
381 if (newResults.length === 0) {382 if (D.partials) log.info('DEBUG[partials] all partials already processed, skipping');383 return [];384 }385
386 const allRawData = [];387 for (const r of newResults) {388 const methodType = getPartialMethodType(partials[r.partialIndex]);389 allRawData.push({390 data: r.data,391 metadata: {392 method: methodType,393 location: r.partialIndex + '0A',394 timestamp: Date.now()395 }396 });397 }398 return allRawData;399}400
401// ── Process raw data in the browser using script.js ──────────────402
403async function processRawDataInBrowser(page, allRawData) {404 if (allRawData.length === 0) {405 if (D.processing) log.info('DEBUG[processing] No raw data to process');406 return [];407 }408
409 if (D.processing) log.info('DEBUG[processing] partials=' + allRawData.length);410
411 const combinedRawData = allRawData;412
413 const results = await page.evaluate(({ rawData, optionKey, debug }) => {414 const processFn = window[optionKey];415 if (!processFn) {416 if (debug) console.log('DEBUG[processing] processFn not found for: ' + optionKey);417 return [];418 }419 try {420 const input = Array.isArray(rawData) ? rawData : [rawData];421 if (debug) console.log('DEBUG[processing] calling ' + optionKey + '() with ' + input.length + ' items');422 const items = processFn(input);423 if (!Array.isArray(items)) return items ? [{ id: null, formatted: items }] : [];424 if (debug) console.log('DEBUG[processing] ' + optionKey + '() returned ' + items.length + ' items');425 return items.map(item => ({426 id: item.id || null,427 raw: item.raw || null,428 formatted: item.formatted || item429 }));430 } catch (err) {431 console.error('Processing error:', err.message, err.stack);432 return [];433 }434 }, { rawData: combinedRawData, optionKey, debug: D.processing });435
436 if (D.processing) log.info('DEBUG[processing] browser returned ' + results.length + ' items');437
438 return results;439}440
441// ── Scrape a page with given enhancements (used for initial + retry) ──442
443async function scrapePage(page, enh, capturedResponses, seenIds) {444 // Apply wait-until strategy445 const waitUntil = enh.waitUntil || 'networkidle';446 if (waitUntil !== 'none' && waitUntil !== 'commit') {447 await page.waitForLoadState(waitUntil).catch(() => {});448 if (D.navigation) log.info('DEBUG[navigation] ' + waitUntil + ' reached');449 }450
451 if (D.screenshot) {452 try {453 const buf = await page.screenshot({ fullPage: false });454 const store = await KeyValueStore.open();455 await store.setValue('debug-before-scrape-' + Date.now(), buf, { contentType: 'image/png' });456 log.info('DEBUG[screenshot] saved pre-scrape screenshot');457 } catch { }458 }459
460 if (D.requests) {461 log.info('DEBUG[requests] capturedResponses=' + capturedResponses.length);462 capturedResponses.forEach(function(r, i) { log.info('DEBUG[requests] [' + i + '] keys=' + Object.keys(r).slice(0, 5).join(',')); });463 }464
465 await injectIntoPage(page);466
467 // Polling loop — mirrors extension's scraping.attempts + scraping.delay468 const scrapingAttempts = option.scraping?.attempts ?? 1;469 const scrapingDelay = option.scraping?.delay || 0;470 const maxScrapingAttempts = Math.min(scrapingAttempts === 0 ? 5 : scrapingAttempts, 5);471
472 let items = [];473 for (let attempt = 1; attempt <= maxScrapingAttempts; attempt++) {474 const rawData = await processAllPartials(page, capturedResponses);475 items = await processRawDataInBrowser(page, rawData);476
477 if (items.length > 0) {478 if (attempt > 1 && D.partials) log.info('DEBUG[partials] scraping succeeded on attempt ' + attempt);479 break;480 }481
482 if (attempt < maxScrapingAttempts && scrapingDelay > 0) {483 if (D.partials) log.info('DEBUG[partials] attempt ' + attempt + ' yielded 0 items, waiting ' + scrapingDelay + 'ms');484 await page.waitForTimeout(scrapingDelay);485 await injectIntoPage(page); // Re-inject in case page changed486 }487 }488
489 // Deduplicate against seenIds490 const newItems = [];491 for (const item of items) {492 const id = item.id || JSON.stringify(item.formatted || item).slice(0, 100);493 if (D.processing) log.info('DEBUG[dedup] id=' + (item.id ? item.id : 'hash:' + id.slice(0, 60)) + ' seen=' + seenIds.has(id));494 if (seenIds.has(id)) continue;495 seenIds.add(id);496 newItems.push(item);497 }498 if (D.processing) log.info('DEBUG[dedup] ' + items.length + ' items -> ' + newItems.length + ' new (seenIds total=' + seenIds.size + ')');499 return newItems;500}501
502// ── Main ─────────────────────────────────────────────────────────503
504await Actor.init();505const input = await Actor.getInput();506
507if (!input?.urls?.length) {508 log.error('No URLs provided in input');509 await Actor.exit();510 process.exit(1);511}512
513const urls = input.urls514 .map(u => typeof u === 'string' ? u : u.url)515 .filter(u => {516 if (!validateUrl(u)) {517 log.warning('Skipping invalid URL: ' + u);518 return false;519 }520 return true;521 });522
523if (urls.length === 0) {524 log.error('No valid URLs after filtering');525 await Actor.exit();526 process.exit(1);527}528
529const maxItems = input.maxItems || 100;530const enablePagination = input.enablePagination !== undefined ? input.enablePagination : !!enhancements.pagination;531const maxPages = typeof input.maxPages === 'number' ? input.maxPages : (enhancements.maxPages || 1);532
533// ── Watchdog & abort tracking ────────────────────────────────534let lastRowTime = Date.now();535const WATCHDOG_TIMEOUT = 100000;536const watchdog = setInterval(() => {537 if (Date.now() - lastRowTime > WATCHDOG_TIMEOUT) {538 log.error('Watchdog: no new rows for ' + (WATCHDOG_TIMEOUT / 1000) + 's, aborting');539 process.exit(1);540 }541}, 15000);542
543const runState = { urlsProcessed: 0, totalResults: 0, startTime: Date.now(), aborted: false };544const navTimeout = typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout * 1000 : 30000;545const navRetries = typeof enhancements.retryNavigation === 'number' ? enhancements.retryNavigation : 0;546
547function shouldAbort() {548 if (!enhancements.abortOnLoss || runState.totalResults > 0) return false;549 const warmupUrls = typeof enhancements.abortWarmupUrls === 'number' ? enhancements.abortWarmupUrls : 5;550 const warmupSecs = typeof enhancements.abortWarmupSeconds === 'number' ? enhancements.abortWarmupSeconds : 60;551 const elapsedSecs = (Date.now() - runState.startTime) / 1000;552 return runState.urlsProcessed >= warmupUrls && elapsedSecs >= warmupSecs;553}554
555// ── Process a single URL on a given page ─────────────────────556
557async function processUrl(page, url, capturedResponses) {558 if (runState.aborted) { log.info('Skipping (aborted): ' + url); return; }559 if (shouldAbort()) {560 runState.aborted = true;561 log.warning('Aborting: no results after ' + runState.urlsProcessed + ' URLs and ' + Math.round((Date.now() - runState.startTime) / 1000) + 's');562 return;563 }564
565 log.info('Processing: ' + url);566 const _t0 = Date.now();567
568 // Reset per-URL state569 processedSignatures = new Set();570
571 await applyPostNavigation(page, enhancements);572
573 if (D.navigation) {574 const pageUrl = page.url();575 const title = await page.title();576 log.info('DEBUG[navigation] landed on url=' + pageUrl.slice(0, 150) + ' title=' + title);577 }578
579 // Custom script — afterLoad, afterDelay, concurrent timing580 let concurrentScriptPromise = null;581 if (enhancements.customScript && enhancements.customScriptCode && enhancements.customScriptTiming !== 'beforeLoad') {582 const timing = enhancements.customScriptTiming || 'afterLoad';583 const parsedUrl = new URL(url);584 const ctx = {585 url,586 hostname: parsedUrl.hostname,587 pathname: parsedUrl.pathname,588 searchParams: Object.fromEntries(parsedUrl.searchParams),589 optionKey,590 timing591 };592
593 if (timing === 'afterLoad') {594 await executeCustomScript(page, enhancements, ctx);595 } else if (timing === 'afterDelay') {596 const delay = typeof enhancements.customScriptDelay === 'number' ? enhancements.customScriptDelay : 2000;597 await page.waitForTimeout(delay);598 await executeCustomScript(page, enhancements, ctx);599 } else if (timing === 'concurrent') {600 concurrentScriptPromise = executeCustomScript(page, enhancements, ctx);601 }602 }603
604 const seenIds = new Set();605 let totalExtracted = 0;606
607 const initialResults = await scrapePage(page, enhancements, capturedResponses, seenIds);608
609 // Retry If Empty610 let results = initialResults;611 if (results.length === 0 && enhancements.retryIfEmpty) {612 const maxRetries = typeof enhancements.retryMaxAttempts === 'number' ? enhancements.retryMaxAttempts : 3;613 log.info('Retry: 0 results, attempting up to ' + maxRetries + ' retries');614
615 for (let attempt = 0; attempt < maxRetries; attempt++) {616 const retryEnh = getRetryEnhancements(attempt);617 log.info('Retry: attempt ' + (attempt + 1) + '/' + maxRetries + (retryEnh.newContext ? ' (new context)' : ''));618
619 let retryPage = page;620 let retryContext = null;621 let retryCaptured = capturedResponses;622
623 if (retryEnh.newContext) {624 // Fresh browser context — new cookies, storage, fingerprint625 retryContext = await page.context().browser().newContext();626 retryPage = await retryContext.newPage();627 retryCaptured = [];628 await applyResourceBlocking(retryPage, retryEnh);629 await applyPreNavigation(retryPage, retryEnh);630 if (needsRequestInterception) {631 requestHandler.setupInterception(retryPage, option, retryCaptured);632 }633 }634
635 await retryPage.goto(url, { timeout: navTimeout, waitUntil: 'commit' });636 await applyPostNavigation(retryPage, retryEnh);637
638 const retryWait = retryEnh.waitUntil || 'networkidle';639 if (retryWait !== 'none' && retryWait !== 'commit') {640 await retryPage.waitForLoadState(retryWait).catch(() => {});641 }642
643 results = await scrapePage(retryPage, retryEnh, retryCaptured, seenIds);644
645 // Clean up new context if we created one646 if (retryContext) {647 await retryContext.close().catch(() => {});648 }649
650 if (results.length > 0) {651 log.info('Retry: attempt ' + (attempt + 1) + ' succeeded with ' + results.length + ' items');652 break;653 }654 }655 }656
657 const initialFormatted = results.map(item => formatForDataset(item, enhancements));658 for (const item of initialFormatted) {659 if (totalExtracted >= maxItems) break;660 await Dataset.pushData(item); lastRowTime = Date.now();661 totalExtracted++;662 }663
664 if (option.paginate && enablePagination) {665 await runPagination(666 page, option, { ...enhancements, pagination: true, maxPages },667 async (pg) => {668 const newItems = await scrapePage(pg, enhancements, capturedResponses, seenIds);669 const formatted = newItems.map(item => formatForDataset(item, enhancements));670 for (const item of formatted) {671 if (totalExtracted >= maxItems) break;672 await Dataset.pushData(item); lastRowTime = Date.now();673 totalExtracted++;674 }675 return formatted;676 }677 );678 }679
680 runState.urlsProcessed++;681 runState.totalResults += totalExtracted;682
683 if (D.performance) log.info('DEBUG[performance] total time=' + ((Date.now() - _t0) / 1000).toFixed(1) + 's items=' + totalExtracted);684 log.info('Extracted ' + totalExtracted + ' items from ' + url);685}686
687// ── Run ──────────────────────────────────────────────────────688
689const crawlerOptions = await getCrawlerOptions(enhancements);690
691if (enhancements.reuseContext) {692 // Reuse context mode: split URLs across N tabs, each tab navigates its URLs sequentially via goto693 const tabCount = Math.min(typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1, urls.length);694
695 // Distribute URLs round-robin across tabs696 const tabUrls = Array.from({ length: tabCount }, () => []);697 urls.forEach((url, i) => tabUrls[i % tabCount].push(url));698
699 // Each tab gets its first URL as the crawler request, remaining URLs navigated via goto700 const crawler = new PlaywrightCrawler({701 headless: true,702 maxRequestsPerCrawl: tabCount,703 ...crawlerOptions,704 maxConcurrency: tabCount,705 minConcurrency: tabCount,706 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,707
708 preNavigationHooks: [709 async ({ page, request }) => {710 await applyResourceBlocking(page, enhancements);711 await applyPreNavigation(page, enhancements);712 // Custom script — beforeLoad timing (Node context, can modify URL)713 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {714 request.url = executeBeforeLoadScript(enhancements, request.url);715 }716 // Create per-page capturedResponses, stored on the page object717 if (needsRequestInterception) {718 page._capturedResponses = [];719 requestHandler.setupInterception(page, option, page._capturedResponses);720 }721 }722 ],723
724 async requestHandler({ page, request }) {725 const capturedResponses = page._capturedResponses || [];726 // Find which tab's URL list this request belongs to727 const tabIndex = tabUrls.findIndex(t => t[0] === request.url);728 const myUrls = tabIndex >= 0 ? tabUrls[tabIndex] : [request.url];729
730 // Process first URL (already navigated by crawler)731 await processUrl(page, request.url, capturedResponses);732
733 // Navigate remaining URLs on the same page734 for (let i = 1; i < myUrls.length; i++) {735 if (runState.aborted) break;736 try {737 const gotoWait = enhancements.waitUntil || 'networkidle';738 await page.goto(myUrls[i], { timeout: navTimeout, waitUntil: gotoWait === 'none' ? 'commit' : gotoWait });739 await processUrl(page, myUrls[i], capturedResponses);740 } catch (err) {741 log.error('Navigation failed for ' + myUrls[i] + ': ' + err.message);742 runState.urlsProcessed++;743 }744 }745 },746
747 maxRequestRetries: navRetries,748 async failedRequestHandler({ request }) {749 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);750 runState.urlsProcessed++;751 }752 });753
754 await crawler.run(tabUrls.map(t => ({ url: t[0] })));755} else {756 // Normal mode: one request per URL, crawler manages contexts757 const crawler = new PlaywrightCrawler({758 headless: true,759 maxRequestsPerCrawl: urls.length,760 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,761 ...crawlerOptions,762
763 preNavigationHooks: [764 async ({ page, request }) => {765 await applyResourceBlocking(page, enhancements);766 await applyPreNavigation(page, enhancements);767 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {768 request.url = executeBeforeLoadScript(enhancements, request.url);769 }770 if (needsRequestInterception) {771 page._capturedResponses = [];772 requestHandler.setupInterception(page, option, page._capturedResponses);773 }774 }775 ],776
777 async requestHandler({ page, request }) {778 const capturedResponses = page._capturedResponses || [];779 await processUrl(page, request.url, capturedResponses);780 },781
782 maxRequestRetries: navRetries,783 async failedRequestHandler({ request }) {784 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);785 runState.urlsProcessed++;786 }787 });788
789 await crawler.run(urls.map(url => ({ url })));790}791
792clearInterval(watchdog);793await Actor.exit();1window.searchField = function(obj, fieldName, parentDepth = 0, results = [], parents = []) {2 if (typeof obj !== "object" || obj === null) {3 return results;4 }5
6 for (const key in obj) {7 if (key === fieldName) {8 let ancestorIndex = parents.length - parentDepth;9 if (ancestorIndex >= 0) {10 results.push(parents[ancestorIndex] || obj);11 } else {12 results.push(obj);13 }14 }15 if (typeof obj[key] === "object" && obj[key] !== null) {16 searchField(obj[key], fieldName, parentDepth, results, [...parents, obj]);17 }18 }19 return results;20}21
22window.searchFieldMap = function(data, fieldName, parentDepth = 0) {23 const results = [];24
25 if (!(data instanceof Map)) {26 return searchField(data, fieldName, parentDepth, results);27 }28
29 for (const [, value] of data) {30 if (typeof value === "object" && value !== null) {31 if (fieldName in value) {32 results.push(value);33 }34 const nested = searchField(value, fieldName, parentDepth, []);35 results.push(...nested);36 }37 }38
39 return results;40}41
42window.getValue = function(value, fallback = "Not Available") {43 if (value === null || value === undefined || value === "") return fallback;44 return String(value);45}46
47window.formatBoolean = function(value) {48 if (typeof value === "boolean") return value ? "Yes" : "No";49 return "Not Available";50}51
52window.joinList = function(items, separator = " || ") {53 if (!Array.isArray(items)) return "Not Available";54 const filtered = items.filter(Boolean);55 return filtered.length > 0 ? filtered.join(separator) : "Not Available";56}57
58window.moduleExport = function(exports) {59 if (typeof module !== "undefined" && module.exports) {60 module.exports = exports;61 } else {62 Object.assign(window, exports);63 }64}65
66window.userCache = new Map();67window.postDetailsCache = new Map();68
69window.getCachedUser = function(user) {70 if (!user) return null;71
72 const userId = user.pk?.toString() || user.id?.toString();73 if (!userId) return user;74
75 if (user.username) {76 userCache.set(userId, user);77 return user;78 }79
80 if (userCache.has(userId)) {81 return { ...user, ...userCache.get(userId) };82 }83
84 return user;85}86
87window.resolveRefs = function(obj, map, visited = new Set()) {88 if (typeof obj !== "object" || obj === null) {89 return obj;90 }91
92 if (!(map instanceof Map)) {93 return obj;94 }95
96 if (obj.__ref) {97 const refKey = obj.__ref;98 if (visited.has(refKey)) return obj;99 visited.add(refKey);100
101 if (map.has(refKey)) {102 const resolved = map.get(refKey);103 return resolveRefs(resolved, map, visited);104 }105 return obj;106 }107
108 if (obj.__refs && Array.isArray(obj.__refs)) {109 return obj.__refs.map(refKey => {110 if (visited.has(refKey)) return { __ref: refKey };111 visited.add(refKey);112
113 if (map.has(refKey)) {114 const resolved = map.get(refKey);115 return resolveRefs(resolved, map, visited);116 }117 return { __ref: refKey };118 });119 }120
121 if (obj.__id) {122 const refKey = obj.__id;123 if (!visited.has(refKey) && map.has(refKey)) {124 visited.add(refKey);125 const lookedUp = map.get(refKey);126 const merged = { ...obj, ...lookedUp };127 const resolved = {};128 for (const key in merged) {129 resolved[key] = resolveRefs(merged[key], map, visited);130 }131 return resolved;132 }133 }134
135 if (Array.isArray(obj)) {136 return obj.map(item => resolveRefs(item, map, visited));137 }138
139 const resolved = {};140 for (const key in obj) {141 resolved[key] = resolveRefs(obj[key], map, visited);142 }143 return resolved;144}145
146window.getPostType = function(mediaType) {147 switch (mediaType) {148 case 1: return "Photo";149 case 2: return "Video";150 case 8: return "Carousel";151 default: return null;152 }153}154
155window.getSavedPostType = function(item) {156 if (item.isSidecar) return "Carousel";157 if (item.isVideo) return "Video";158 return "Photo";159}160
161window.getImageUrl = function(item) {162 const candidates = item.image_versions2?.candidates;163 if (Array.isArray(candidates) && candidates[0]?.url) {164 return candidates[0].url;165 }166
167 const resolverData = item["$r:client__srcSet"]?.__resolverSnapshot?.data;168 if (resolverData?.image_versions2?.candidates?.[0]?.url) {169 return resolverData.image_versions2.candidates[0].url;170 }171
172 return null;173}174
175window.getCurrentInstagramUrl = function() {176 if (typeof location === "undefined") return "";177 return location.href || "";178}179
180window.isInstagramReelsUrl = function() {181 const currentUrl = getCurrentInstagramUrl();182 return currentUrl.includes("instagram.com/reels/") || currentUrl.includes("instagram.com/reel/");183}184
185window.getCurrentInstagramReelsCode = function() {186 if (!isInstagramReelsUrl()) return null;187 return getInstagramCodeFromUrl(getCurrentInstagramUrl());188}189
190window.getInstagramCodeFromUrl = function(url) {191 if (!url) return null;192 try {193 const parsedUrl = new URL(url);194 const parts = parsedUrl.pathname.split("/").filter(Boolean);195 return ["p", "reel", "reels"].includes(parts[0]) && parts[1] ? parts[1] : null;196 } catch (error) {197 const match = url.match(/instagram\.com\/(?:p|reel|reels)\/([^/?#]+)/);198 return match ? match[1] : null;199 }200}201
202window.cacheUsersFromData = function(data) {203 if (!data || typeof data !== "object") return;204 let users = [];205 if (data instanceof Map) {206 users = searchFieldMap(data, "full_name");207 users = users.map(item => resolveRefs(item, data));208 } else {209 users = searchField(data, "full_name");210 }211 for (const user of users) {212 if (user && user.username) {213 getCachedUser(user);214 }215 }216}217
218window.getPostDetailValueCount = function(item) {219 return [220 item.caption,221 item.likeCount,222 item.commentCount,223 item.viewCount,224 item.user?.username,225 item.user?.full_name,226 item.user?.profile_pic_url,227 item.imageUrl,228 item.videoUrl,229 item.postedAt,230 item.duration,231 item.hasAudio,232 item.audioTitle,233 item.audioArtist,234 item.isPaidPartnership,235 item.commentsDisabled,236 item.accessibilityCaption237 ].filter(value => value !== null && value !== undefined && value !== "").length;238}239
240window.mergePostDetails = function(existing, incoming) {241 if (!existing) return incoming;242 const preferred = getPostDetailValueCount(incoming) > getPostDetailValueCount(existing) ? incoming : existing;243 const fallback = preferred === incoming ? existing : incoming;244 return {245 ...fallback,246 ...preferred,247 user: {248 ...(fallback.user || {}),249 ...(preferred.user || {})250 },251 raw: preferred.raw || fallback.raw252 };253}254
255window.getPostDetailCandidates = function(data) {256 if (!data || typeof data !== "object") return [];257 let candidates = [];258 if (data instanceof Map) {259 candidates = [260 ...searchFieldMap(data, "coauthor_producers"),261 ...searchFieldMap(data, "code")262 ].map(item => resolveRefs(item, data));263 } else {264 candidates = [265 ...searchField(data, "coauthor_producers"),266 ...searchField(data, "code")267 ];268 }269
270 return candidates.filter(item => {271 if (!item || typeof item !== "object") return false;272 const key = item.code || item.pk?.toString() || item.id?.toString();273 return Boolean(key);274 });275}276
277// ── POSTS ──278
279window.isReelPostDetailItem = function(item) {280 return item?.code && (item.product_type === "clips" || item.media_type === 2 || item.clips_metadata);281}282
283window.preparePostDetailItem = function(item) {284 const postId = item.pk?.toString() || item.id?.toString();285 if (!postId) return null;286
287 const user = getCachedUser(item.user || item.owner || item.caption?.user);288 const caption = item.caption;289 const captionText = typeof caption === "string" ? caption : caption?.text;290 const isReel = item.product_type === "clips" || item.media_type === 2;291 const isCarousel = item.media_type === 8 || item.carousel_media_count > 0;292
293 let imageUrl = item.image_versions2?.candidates?.[0]?.url294 || item.carousel_media?.[0]?.image_versions2?.candidates?.[0]?.url;295
296 let videoUrl = item.video_versions?.[0]?.url297 || item.carousel_media?.[0]?.video_versions?.[0]?.url;298
299 let taggedUsers = null;300 if (item.usertags?.in?.length > 0) {301 taggedUsers = item.usertags.in.map(tag => tag.user?.username).filter(Boolean).join(", ");302 }303
304 return {305 id: postId,306 code: item.code,307 type: getPostType(item.media_type),308 caption: captionText,309 likeCount: item.like_count?.toString(),310 commentCount: item.comment_count?.toString(),311 viewCount: item.view_count?.toString() || item.play_count?.toString(),312 user,313 url: item.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${item.code}/` : null,314 imageUrl,315 videoUrl,316 coverImage: item.image_versions2?.candidates?.[0]?.url,317 postedAt: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,318 location: item.location?.name,319 duration: item.video_duration?.toString(),320 hasAudio: item.has_audio,321 audioTitle: item.clips_metadata?.original_sound_info?.original_audio_title,322 audioArtist: item.clips_metadata?.original_sound_info?.ig_artist?.username,323 taggedUsers,324 isCarousel,325 carouselCount: item.carousel_media_count?.toString() || item.carousel_media?.length?.toString(),326 originalWidth: item.original_width?.toString(),327 originalHeight: item.original_height?.toString(),328 isPaidPartnership: item.is_paid_partnership,329 commentsDisabled: item.comments_disabled,330 likesDisabled: item.like_and_view_counts_disabled,331 accessibilityCaption: item.accessibility_caption,332 raw: item333 };334}335
336window.cachePostDetailItem = function(item) {337 if (!isReelPostDetailItem(item)) return;338 const prepared = preparePostDetailItem(item);339 if (!prepared) return;340 const cached = postDetailsCache.get(item.code);341 if (!cached) {342 postDetailsCache.set(item.code, item);343 return;344 }345 const cachedPrepared = preparePostDetailItem(cached);346 const merged = mergePostDetails(cachedPrepared, prepared);347 postDetailsCache.set(item.code, merged.raw);348}349
350window.preparePostsFromRequest = function(data) {351 const results = [];352 const seen = new Set();353
354 for (const item of data) {355 let posts = searchField(item, "media_notes");356 if (posts.length === 0) {357 posts = searchField(item, "clips_tab_pinned_user_ids");358 }359
360 for (const post of posts) {361 const postId = post.id?.toString() || post.pk?.toString();362 if (!postId || seen.has(postId)) continue;363
364 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {365 if (post.is_paid_partnership !== false) continue;366 }367
368 seen.add(postId);369 const user = getCachedUser(post.user);370 const isReel = post.product_type === "clips" || post.media_type === 2;371
372 results.push({373 id: postId,374 code: post.code,375 user,376 type: getPostType(post.media_type),377 text: post.caption?.text,378 image: getImageUrl(post),379 video: post.video_versions?.[0]?.url,380 likes: post.like_count?.toString(),381 commentsCount: post.comment_count?.toString(),382 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,383 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,384 commentsDisabled: post.comments_disabled,385 accessibilityCaption: post.accessibility_caption,386 raw: post387 });388 }389 }390
391 return results;392}393
394window.preparePostsFromDom = function(data) {395 const results = [];396 const seen = new Set();397
398 for (const kv of data) {399 const parsed = typeof kv === "string" ? JSON.parse(kv) : kv;400 let posts = searchField(parsed, "media_notes");401 if (posts.length === 0) {402 posts = searchField(parsed, "clips_tab_pinned_user_ids");403 }404
405 for (const post of posts) {406 const postId = post.id?.toString() || post.pk?.toString();407 if (!postId || seen.has(postId)) continue;408
409 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {410 if (post.is_paid_partnership !== false) continue;411 }412
413 seen.add(postId);414 const user = getCachedUser(post.user);415 const isReel = post.product_type === "clips" || post.media_type === 2;416
417 results.push({418 id: postId,419 code: post.code,420 user,421 type: getPostType(post.media_type),422 text: post.caption?.text,423 image: getImageUrl(post),424 video: post.video_versions?.[0]?.url,425 likes: post.like_count?.toString(),426 commentsCount: post.comment_count?.toString(),427 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,428 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,429 commentsDisabled: post.comments_disabled,430 accessibilityCaption: post.accessibility_caption,431 raw: post432 });433 }434 }435
436 return results;437}438
439window.preparePostsFromReact = function(data) {440 const results = [];441 const seen = new Set();442
443 for (const item of data) {444 const connection = item?.xdt_fbsearch__top_serp_graphql;445 if (connection?.edges?.length > 0) {446 for (const edge of (connection.edges || [])) {447 const items = edge.node?.items || [];448 for (const mediaItem of items) {449 const fragKey = "usePolarisKeywordSearchRelayReduxSync_media";450 const frag = mediaItem?.[fragKey]?.__fragments?.[fragKey];451 const post = frag && Object.keys(frag).length > 2 ? frag : mediaItem;452 const postId = post.pk?.toString() || post.id?.toString();453 if (!postId || !post.code || seen.has(postId)) continue;454 seen.add(postId);455 const isReel = post.media_type === 2;456 results.push({457 id: postId,458 code: post.code,459 user: post.user || mediaItem.user,460 type: getPostType(post.media_type),461 text: post.caption?.text,462 image: post.image_versions2?.candidates?.[0]?.url,463 video: post.video_versions?.[0]?.url,464 likes: post.like_count?.toString(),465 commentsCount: post.comment_count?.toString(),466 url: `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/`,467 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,468 commentsDisabled: post.comments_disabled,469 accessibilityCaption: post.accessibility_caption,470 raw: mediaItem471 });472 }473 }474 continue;475 }476
477 if (item && "isFetching" in item && Array.isArray(item.posts)) {478 for (const post of item.posts) {479 const postId = post.id?.toString();480 if (!postId || seen.has(postId)) continue;481 if (post.isSponsored) continue;482
483 seen.add(postId);484 const owner = post.owner;485 const isReel = post.productType === "clips" || post.isVideo;486
487 results.push({488 id: postId,489 code: post.code,490 user: owner ? {491 username: owner.username,492 full_name: owner.fullName,493 profile_pic_url: owner.profilePictureUrl,494 is_verified: owner.isVerified495 } : null,496 type: getSavedPostType(post),497 text: post.caption,498 image: post.src || post.displayResources?.[0]?.src,499 video: post.videoResources?.[0]?.src,500 likes: post.numLikes?.toString(),501 commentsCount: null,502 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,503 date: post.postedAt ? new Date(post.postedAt * 1000).toISOString() : null,504 commentsDisabled: post.commentsDisabled,505 accessibilityCaption: post.accessibilityCaption,506 raw: post507 });508 }509 continue;510 }511
512 if (item instanceof Map) {513 cacheUsersFromData(item);514 let posts = searchFieldMap(item, "sponsor_tags");515 if (posts.length === 0) {516 posts = searchFieldMap(item, "clips_tab_pinned_user_ids");517 }518 posts = posts.map(p => resolveRefs(p, item));519
520 for (const post of posts) {521 const postId = post.id?.toString() || post.pk?.toString();522 if (!postId || seen.has(postId)) continue;523
524 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {525 if (post.is_paid_partnership !== false) continue;526 }527
528 seen.add(postId);529 const user = getCachedUser(post.user);530 const isReel = post.product_type === "clips" || post.media_type === 2;531
532 results.push({533 id: postId,534 code: post.code,535 user,536 type: getPostType(post.media_type),537 text: post.caption?.text,538 image: getImageUrl(post),539 video: post.video_versions?.[0]?.url,540 likes: post.like_count?.toString(),541 commentsCount: post.comment_count?.toString(),542 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,543 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,544 commentsDisabled: post.comments_disabled,545 accessibilityCaption: post.accessibility_caption,546 raw: post547 });548 }549 continue;550 }551
552 const edges = item?.data?.xdt_api__v1__feed__user_timeline_graphql_connection?.edges;553 if (edges) {554 for (const edge of edges) {555 const post = edge.node;556 const postId = post.id?.toString() || post.pk?.toString();557 if (!postId || seen.has(postId)) continue;558
559 seen.add(postId);560 const isReel = post.product_type === "clips" || post.media_type === 2;561
562 results.push({563 id: postId,564 code: post.code,565 user: post.user,566 type: getPostType(post.media_type),567 text: post.caption?.text,568 image: post.image_versions2?.candidates?.[0]?.url,569 video: post.video_versions?.[0]?.url,570 likes: post.like_count?.toString(),571 commentsCount: post.comment_count?.toString(),572 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,573 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,574 commentsDisabled: post.comments_disabled,575 accessibilityCaption: post.accessibility_caption,576 raw: edge577 });578 }579 continue;580 }581
582 cacheUsersFromData(item);583 let posts = searchField(item, "media_notes");584 if (posts.length === 0) {585 posts = searchField(item, "clips_tab_pinned_user_ids");586 }587
588 for (const post of posts) {589 const postId = post.id?.toString() || post.pk?.toString();590 if (!postId || seen.has(postId)) continue;591
592 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {593 if (post.is_paid_partnership !== false) continue;594 }595
596 seen.add(postId);597 const user = getCachedUser(post.user);598 const isReel = post.product_type === "clips" || post.media_type === 2;599
600 results.push({601 id: postId,602 code: post.code,603 user,604 type: getPostType(post.media_type),605 text: post.caption?.text,606 image: getImageUrl(post),607 video: post.video_versions?.[0]?.url,608 likes: post.like_count?.toString(),609 commentsCount: post.comment_count?.toString(),610 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,611 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,612 commentsDisabled: post.comments_disabled,613 accessibilityCaption: post.accessibility_caption,614 raw: post615 });616 }617 }618
619 return results;620}621
622window.formatPosts = function(prepared) {623 return prepared.map(item => ({624 id: item.id,625 id2: item.code,626 highlight: { id2: item.code },627 raw: item.raw,628 formatted: {629 "id": item.id,630 "Post Author": item.user?.username || "Not Available",631 "Post Author Full Name": item.user?.full_name || item.user?.fullName || "Not Available",632 "Post Author Image": item.user?.profile_pic_url || item.user?.profilePictureUrl || "Not Available",633 "Post Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",634 "Post Author Is Verified": formatBoolean(item.user?.is_verified ?? item.user?.isVerified),635 "Post Type": item.type || "Not Available",636 "Post Text": item.text || "Not Available",637 "Post Image": item.image || "Not Available",638 "Post Video": item.video || "Not Available",639 "Post Likes": item.likes || "Not Available",640 "Post Comments Count": item.commentsCount || "Not Available",641 "Post URL": item.url || "Not Available",642 "Post Date": item.date || "Not Available",643 "Is Comments Disabled": formatBoolean(item.commentsDisabled),644 "Post Accessibility Caption": item.accessibilityCaption || "Not Available"645 }646 }));647}648
649window.posts = function(rawData) {650 try {651 if (isInstagramReelsUrl()) return [];652
653 let prepared = [];654 const seen = new Set();655
656 for (const entry of rawData) {657 let batch = [];658 if (entry.metadata.method === "request") {659 batch = preparePostsFromRequest(entry.data);660 } else if (entry.metadata.method === "dom") {661 batch = preparePostsFromDom(entry.data);662 } else if (entry.metadata.method === "react") {663 batch = preparePostsFromReact(entry.data);664 }665 for (const item of batch) {666 if (!seen.has(item.id)) {667 seen.add(item.id);668 prepared.push(item);669 }670 }671 }672
673 return formatPosts(prepared);674 } catch (error) {675 return [];676 }677}678
679// ── STORIES ──680
681window.prepareStories = function(rawData) {682 const results = [];683 const seen = new Set();684 const userStore = new Map();685
686 for (const entry of rawData) {687 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];688 for (const data of dataItems) {689 const stories = searchField(data, "story_cta");690 searchField(data, "profile_pic_url").forEach(user => {691 userStore.set(user.id, user);692 });693
694 for (const item of stories) {695 const storyId = item.pk?.toString() || item.id?.toString();696 if (!storyId || seen.has(storyId)) continue;697 seen.add(storyId);698
699 let user = item.user;700 if (user && !user.profile_pic_url && userStore.get(user.id)) {701 user = userStore.get(user.id);702 }703
704 results.push({705 id: storyId,706 type: getPostType(item.media_type),707 image: item.image_versions2?.candidates?.[0]?.url,708 video: item.video_versions?.[0]?.url,709 link: item.pk && user?.username ? `https://www.instagram.com/stories/${user.username}/${item.pk}/` : null,710 date: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,711 expiringAt: item.expiring_at ? new Date(item.expiring_at * 1000).toISOString() : null,712 accessibilityCaption: item.accessibility_caption,713 user,714 bloksFullName: item.story_bloks_stickers?.[0]?.bloks_sticker?.sticker_data?.ig_mention?.full_name,715 raw: item716 });717 }718 }719 }720
721 return results;722}723
724window.formatStories = function(prepared) {725 return prepared.map(item => ({726 id: item.id,727 highlight: { id: item.id },728 raw: item.raw,729 formatted: {730 "id": item.id,731 "Story Type": item.type || "Not Available",732 "Story Image": item.image || "Not Available",733 "Story Video": item.video || "Not Available",734 "Story Link": item.link || "Not Available",735 "Story Date": item.date || "Not Available",736 "Story Expiring At": item.expiringAt || "Not Available",737 "Story Accessibility Caption": item.accessibilityCaption || "Not Available",738 "Author Username": item.user?.username || "Not Available",739 "Author Full Name": item.user?.full_name || item.bloksFullName || "Not Available",740 "Author Image": item.user?.profile_pic_url || "Not Available",741 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",742 "Author Is Verified": formatBoolean(item.user?.is_verified)743 }744 }));745}746
747window.stories = function(rawData) {748 try {749 const prepared = prepareStories(rawData);750 return formatStories(prepared);751 } catch (error) {752 return [];753 }754}755
756// ── POST DETAILS ──757
758window.preparePostDetails = function(rawData) {759 const byPostKey = new Map();760 const reelsCode = getCurrentInstagramReelsCode();761 const currentUrl = getCurrentInstagramUrl();762
763 for (const entry of rawData) {764 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];765 const entryUrl = entry.metadata?.url || entry.metadata?.href || entry.url || entry.sourceUrl;766 const entryCode = getInstagramCodeFromUrl(entryUrl);767
768 for (const data of dataItems) {769 cacheUsersFromData(data);770 }771
772 for (const data of dataItems) {773 const items = getPostDetailCandidates(data);774
775 for (const item of items) {776 cachePostDetailItem(item);777 const prepared = preparePostDetailItem(item);778 if (!prepared) continue;779
780 const targetCode = entryCode || reelsCode;781 if (targetCode) {782 if (item.code !== targetCode) continue;783 } else if (!item.code || !currentUrl.includes(item.code)) {784 continue;785 }786 const postKey = item.code || prepared.id;787 byPostKey.set(postKey, mergePostDetails(byPostKey.get(postKey), prepared));788 }789 }790 }791
792 if (reelsCode && postDetailsCache.has(reelsCode)) {793 console.log("Instagram postDetails cache check", {794 currentUrl,795 reelsCode,796 cacheSize: postDetailsCache.size,797 cacheKeys: Array.from(postDetailsCache.keys()),798 cacheHit: true799 });800 const cachedPrepared = preparePostDetailItem(postDetailsCache.get(reelsCode));801 if (cachedPrepared) {802 byPostKey.set(reelsCode, mergePostDetails(byPostKey.get(reelsCode), cachedPrepared));803 }804 } else if (reelsCode) {805 console.log("Instagram postDetails cache check", {806 currentUrl,807 reelsCode,808 cacheSize: postDetailsCache.size,809 cacheKeys: Array.from(postDetailsCache.keys()),810 cacheHit: false811 });812 }813
814 return Array.from(byPostKey.values());815}816
817window.formatPostDetails = function(prepared) {818 return prepared.map(item => ({819 id: item.id,820 id2: item.code,821 highlight: { id: item.id },822 raw: item.raw,823 formatted: {824 "id": item.id,825 "Post Type": item.type || "Not Available",826 "Caption": item.caption || "Not Available",827 "Like Count": item.likeCount || "Not Available",828 "Comment Count": item.commentCount || "Not Available",829 "View Count": item.viewCount || "Not Available",830 "Author Username": item.user?.username || "Not Available",831 "Author Full Name": item.user?.full_name || "Not Available",832 "Author ID": item.user?.pk?.toString() || item.user?.id?.toString() || "Not Available",833 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",834 "Author Is Verified": formatBoolean(item.user?.is_verified),835 "Author Profile Pic": item.user?.profile_pic_url || item.user?.hd_profile_pic_url_info?.url || "Not Available",836 "Post URL": item.url || "Not Available",837 "Image URL": item.imageUrl || "Not Available",838 "Video URL": item.videoUrl || "Not Available",839 "Cover Image": item.coverImage || "Not Available",840 "Posted At": item.postedAt || "Not Available",841 "Location": item.location || "Not Available",842 "Duration": item.duration || "Not Available",843 "Has Audio": formatBoolean(item.hasAudio),844 "Audio Title": item.audioTitle || "Not Available",845 "Audio Artist": item.audioArtist || "Not Available",846 "Tagged Users": item.taggedUsers || "Not Available",847 "Is Carousel": formatBoolean(item.isCarousel),848 "Carousel Count": item.carouselCount || "Not Available",849 "Original Width": item.originalWidth || "Not Available",850 "Original Height": item.originalHeight || "Not Available",851 "Is Paid Partnership": formatBoolean(item.isPaidPartnership),852 "Comments Disabled": formatBoolean(item.commentsDisabled),853 "Likes Disabled": formatBoolean(item.likesDisabled),854 "Accessibility Caption": item.accessibilityCaption || "Not Available"855 }856 }));857}858
859window.postDetails = function(rawData) {860 console.log(rawData)861 try {862 const prepared = preparePostDetails(rawData);863 return formatPostDetails(prepared);864 } catch (error) {865 return [];866 }867}868
869// ── COMMENTS ──870
871window.prepareComments = function(rawData) {872 const results = [];873 const seen = new Set();874
875 for (const entry of rawData) {876 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];877 for (const data of dataItems) {878 const comments = searchField(data, "comment_like_count");879
880 for (const item of comments) {881 const commentId = item.pk?.toString() || item.id?.toString();882 if (!commentId || seen.has(commentId)) continue;883 seen.add(commentId);884
885 const commentDate = item.created_at ? new Date(item.created_at * 1000).toISOString() : null;886
887 results.push({888 id: commentId,889 date: commentDate,890 text: item.text,891 media: item.giphy_media_info?.first_party_cdn_proxied_images?.fixed_height?.url,892 likes: item.comment_like_count?.toString(),893 user: item.user,894 isReply: !!item.parent_comment_id,895 parentCommentId: item.parent_comment_id?.toString(),896 childCommentCount: item.child_comment_count,897 raw: item898 });899 }900 }901 }902
903 return results;904}905
906window.formatComments = function(prepared) {907 return prepared.map(item => ({908 id: item.id,909 id2: item.date,910 highlight: { id2: item.date },911 raw: item.raw,912 formatted: {913 "id": item.id,914 "Comment Text": item.text || "Not Available",915 "Comment Media": item.media || "Not Available",916 "Comment Date": item.date || "Not Available",917 "Comment Likes": item.likes || "Not Available",918 "Author Username": item.user?.username || "Not Available",919 "Author ID": item.user?.pk || item.user?.id || "Not Available",920 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",921 "Author Is Verified": formatBoolean(item.user?.is_verified),922 "Is Reply": formatBoolean(item.isReply),923 "Parent Comment ID": item.parentCommentId || "Not Available",924 "Has Replies": formatBoolean(item.childCommentCount > 0),925 "Reply Count": item.childCommentCount?.toString() || "Not Available"926 }927 }));928}929
930window.comments = function(rawData) {931 try {932 const prepared = prepareComments(rawData);933 return formatComments(prepared);934 } catch (error) {935 return [];936 }937}938
939// ── PROFILE ──940
941window.prepareProfile = function(rawData) {942 const results = [];943 const seen = new Set();944
945 for (const entry of rawData) {946 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];947
948 for (const data of dataItems) {949 let users = [];950
951 if (data instanceof Map) {952 users = searchFieldMap(data, "biography");953 users = users.map(user => resolveRefs(user, data));954 } else {955 users = searchField(data, "biography");956 }957
958 for (const item of users) {959 const userId = item.pk?.toString() || item.id?.toString();960 if (!userId || seen.has(userId)) continue;961 seen.add(userId);962
963 results.push({964 id: userId,965 username: item.username,966 fullName: item.full_name,967 biography: item.biography,968 externalUrl: item.external_url,969 followerCount: item.follower_count?.toString(),970 followingCount: item.following_count?.toString(),971 mediaCount: item.media_count?.toString(),972 isVerified: item.is_verified,973 isPrivate: item.is_private,974 isBusiness: item.is_business,975 category: item.category,976 profilePicUrl: item.hd_profile_pic_url_info?.url || item.profile_pic_url,977 bioLinks: item.bio_links?.map(link => link.url),978 raw: item979 });980 }981 }982 }983
984 return results;985}986
987window.formatProfile = function(prepared) {988 return prepared.map(item => ({989 id: item.id,990 id2: item.username,991 highlight: { id: item.id },992 raw: item.raw,993 formatted: {994 "id": item.id,995 "Username": item.username || "Not Available",996 "Full Name": item.fullName || "Not Available",997 "Biography": item.biography || "Not Available",998 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",999 "External URL": item.externalUrl || "Not Available",1000 "Follower Count": item.followerCount || "Not Available",1001 "Following Count": item.followingCount || "Not Available",1002 "Post Count": item.mediaCount || "Not Available",1003 "Is Verified": formatBoolean(item.isVerified),1004 "Is Private": formatBoolean(item.isPrivate),1005 "Is Business": formatBoolean(item.isBusiness),1006 "Category": item.category || "Not Available",1007 "Profile Picture URL": item.profilePicUrl || "Not Available",1008 "All Bio Links": joinList(item.bioLinks)1009 }1010 }));1011}1012
1013window.profile = function(rawData) {1014 try {1015 const prepared = prepareProfile(rawData);1016 return formatProfile(prepared);1017 } catch (error) {1018 return [];1019 }1020}1021
1022// ── FOLLOWERS / FOLLOWING ──1023
1024window.prepareUsers = function(rawData) {1025 const results = [];1026 const seen = new Set();1027
1028 for (const entry of rawData) {1029 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];1030 for (const data of dataItems) {1031 const users = searchField(data, "profile_pic_url");1032
1033 for (const item of users) {1034 const userId = item.pk?.toString() || item.id?.toString();1035 if (!userId || seen.has(userId)) continue;1036 seen.add(userId);1037
1038 results.push({1039 id: userId,1040 username: item.username,1041 fullName: item.full_name,1042 isPrivate: item.is_private,1043 isVerified: item.is_verified,1044 profilePicUrl: item.profile_pic_url,1045 hasAnonPic: item.has_anonymous_profile_picture,1046 latestStoryTime: item.latest_reel_media ? new Date(item.latest_reel_media * 1000).toISOString() : null,1047 raw: item1048 });1049 }1050 }1051 }1052
1053 return results;1054}1055
1056window.formatUsers = function(prepared) {1057 return prepared.map(item => ({1058 id: item.id,1059 id2: item.username,1060 highlight: { id2: item.username },1061 raw: item.raw,1062 formatted: {1063 "id": item.id,1064 "Username": item.username || "Not Available",1065 "Full Name": item.fullName || "Not Available",1066 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",1067 "Is Private": formatBoolean(item.isPrivate),1068 "Is Verified": formatBoolean(item.isVerified),1069 "Profile Picture URL": item.profilePicUrl || "Not Available",1070 "Has Anonymous Profile Picture": formatBoolean(item.hasAnonPic),1071 "Latest Story Time": item.latestStoryTime || "Not Available"1072 }1073 }));1074}1075
1076window.followers = function(rawData) {1077 try {1078 const prepared = prepareUsers(rawData);1079 return formatUsers(prepared);1080 } catch (error) {1081 return [];1082 }1083}1084
1085window.following = function(rawData) {1086 try {1087 const prepared = prepareUsers(rawData);1088 return formatUsers(prepared);1089 } catch (error) {1090 return [];1091 }1092}{ "title": "Instagram Comments of a Post Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "urls": { "title": "URLs", "type": "array", "items": { "type": "string" }, "description": "URLs to scrape. Must match: instagram.com", "editor": "stringList", "default": [ "https://www.instagram.com/p/DWZitMjgHp1/" ] }, "maxItems": { "title": "Max items", "type": "integer", "description": "Maximum number of items to extract per URL", "default": 100, "minimum": 1 }, "enablePagination": { "title": "Paginate", "type": "boolean", "description": "Load more results beyond the first page.", "default": true }, "maxPages": { "title": "Max pagination steps", "type": "integer", "description": "Maximum number of pagination actions (pages, scrolls, or loads)", "default": 1, "minimum": 1 } }, "required": [ "urls" ]}{ "actorSpecification": 1, "fields": { "Comment Text": { "type": "string", "title": "Comment Text" }, "Comment Media": { "type": "string", "title": "Comment Media" }, "Comment Date": { "type": "string", "title": "Comment Date" }, "Comment Likes": { "type": "string", "title": "Comment Likes" }, "Author Username": { "type": "string", "title": "Author Username" }, "Author ID": { "type": "string", "title": "Author ID" }, "Author URL": { "type": "string", "title": "Author URL" }, "Author Is Verified": { "type": "string", "title": "Author Is Verified" }, "Is Reply": { "type": "string", "title": "Is Reply" }, "Parent Comment ID": { "type": "string", "title": "Parent Comment ID" }, "Has Replies": { "type": "string", "title": "Has Replies" }, "Reply Count": { "type": "string", "title": "Reply Count" } }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table", "properties": {} } } }}