Instagram Post Comments Scraper
Pricing
from $2.00 / 1,000 results
Extract Instagram comment text, dates, likes, and author details from post pages. Powered by Instagram Scraper.
Pricing
from $2.00 / 1,000 results
Rating
0.0
(0)
Developer
Nifty
Actor stats
0
Bookmarked
3
Total users
1
Monthly active users
3 days ago
Last modified
Categories
Share
{ "actorSpecification": 1, "name": "instagram-comments-scraper", "title": "Instagram Comments of a Post Scraper", "version": "3.2", "dockerfile": "./Dockerfile", "input": "./INPUT_SCHEMA.json", "storages": { "dataset": { "actorSpecification": 1, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table" } } } } }}FROM apify/actor-node-playwright-chrome:20
COPY package.json ./RUN npm install --omit=dev --omit=optional
COPY . ./
CMD npm start{ "name": "instagram-comments-scraper", "version": "3.2", "type": "module", "main": "src/main.js", "scripts": { "start": "node src/main.js" }, "dependencies": { "crawlee": "^3.12.0", "playwright": "^1.49.0", "apify": "^3.2.0", "user-agents": "^1.1.0", "playwright-extra": "^4.3.0", "puppeteer-extra-plugin-stealth": "^2.11.0" }}{ "app": { "name": "Instagram Scraper", "targetName": "Instagram", "targetURL": "https://instagram.com", "id": "instagram-scraper", "title": "I-Social Scraper: Export Posts, Profiles, Reels & More", "description": "Instagram Scraper: Extract Instagram comments, stories, followers & more. Export to CSV, JSON, XLSX with ease.", "version": "3.2.0", "backend": "https://nifty.codes", "hostnames": [ "instagram.com" ], "matches": [ "*://*.instagram.com/*" ], "status": "Published", "colors": { "background": "#f80261", "elements": "#ffffff" }, "longDescription": "This extension provides a comprehensive toolkit for extracting publicly available data from Instagram profiles, posts, reels, stories, and comments directly within your browser. All data collection and processing happens locally on your device. Built-in monitoring tools let you track page changes and receive alerts over time.\n\nKey Features\n✓ Monitor pages for changes and receive alerts via webhook or ntfy.sh\n✓ All scraping and exports processed locally in your browser\n✓ Export to CSV, Excel (XLSX), JSON, HTML Table, and Raw JSON\n✓ Automatic pagination for feeds, follower lists, and comment threads\n✓ Batch processing for multiple URLs in a single session\n✓ Customizable field selection and conditional formatting\n\nWhat Data You Can Extract\n\nPosts and Reels: Author details, post text, media URLs, like and comment counts, timestamps, and accessibility captions. Works on home feeds, profile grids, reels tabs, saved posts, and tagged posts.\n\nPost Details: Engagement metrics (likes, comments, views), location data, audio information, tagged users, carousel content, duration, and paid partnership status.\n\nComments: Comment text, timestamps, like counts, reply hierarchies, author usernames, and verification status.\n\nStories: Media URLs, expiration times, story type, accessibility captions, and author information.\n\nProfiles: Username, full name, biography, follower and following counts, post count, verification status, privacy status, business category, and external links.\n\nFollowers and Following: Usernames, full names, profile URLs, verification and privacy status, profile pictures, and latest story timestamps.\n\nMonitoring and Alerts\nPage Watcher lets you track changes on any Instagram page and get notified when something updates. Set conditions on specific fields you care about, such as follower count or new posts, and receive alerts via webhook or ntfy.sh when those conditions are met.\n\nBatch Processing\nBatch URL Opener allows you to queue multiple Instagram URLs and scrape them all in one session. Add URLs manually or auto-collect them from a list page, then batch-process every link and export the combined results.\n\nExport Formats\n- CSV\n- Excel (XLSX)\n- JSON\n- HTML Table\n- Raw JSON\n\nPerfect For\nThis tool is ideal for archiving social media content, tracking public engagement metrics, and compiling profile data from Instagram. It suits researchers, marketers, and anyone who needs structured exports of publicly visible information.\n\nSupported Regions\nThis extension works with Instagram globally and is compatible across all regions where the platform is accessible.\n\nPrivacy and Security\nAll scraping and data exports are processed locally on your machine. General usage metrics are collected to help improve the extension, but no scraped data is transmitted to external servers.\n\nUsage Notice\nThis extension provides tools for extracting publicly available data. Users are responsible for ensuring their use complies with applicable terms of service and local regulations. The developer provides the tool and is not liable for how it is used.\n\nTechnical Support\nFor assistance or questions, contact hello@nifty.codes.\n\nTrademark\nInstagram is a registered trademark of Meta Platforms, Inc. This extension is an independent tool and is not affiliated with, endorsed by, authorized, maintained, or sponsored by Meta Platforms, Inc." }, "options": { "posts": { "id": "posts", "title": "Posts in a List", "description": "Works on feeds, profiles, reels, saved posts", "instructions": [ "Go to any page that shows a list of posts (home feed, profile, reels tab, saved, etc.)", "Make sure posts are visible and fully loaded" ], "type": "list", "urlField": "Post URL", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Post Author", "fixes": [], "timeout": 10000, "fields": [ "Post Author", "Post Author Full Name", "Post Author Image", "Post Author URL", "Post Author Is Verified", "Post Type", "Post Text", "Post Image", "Post Video", "Post Likes", "Post Comments Count", "Post URL", "Post Date", "Is Comments Disabled", "Post Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "media_notes" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "clips_tab_pinned_user_ids" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "sponsor_tags" }, "parse": true } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "xdt_api__v1__feed__user_timeline_graphql_connection" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "media_count" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "clips_tab_pinned_user_ids" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "url", "operator": "contains", "value": "/reels/" }, { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "isFetching", "posts" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "url", "operator": "contains", "value": "/saved/" } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href*='/p/{{{id2}}}/'], a[href*='/reel/{{{id2}}}/']" }, { "method": "closest", "query": "article" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "preset", "query": "window" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] }, "condition": [ { "type": "url", "operator": "endsWith", "value": "instagram.com/" } ] }, "postDetails": { "id": "postDetails", "title": "Post Details", "description": "Works on individual post or reel pages", "instructions": [ "Open a specific post or reel on Instagram", "Wait for all the details to fully load" ], "type": "detail", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Caption", "fixes": [], "timeout": 10000, "fields": [ "Post Type", "Caption", "Like Count", "Comment Count", "View Count", "Author Username", "Author Full Name", "Author ID", "Author URL", "Author Is Verified", "Author Profile Pic", "Post URL", "Image URL", "Video URL", "Cover Image", "Posted At", "Location", "Duration", "Has Audio", "Audio Title", "Audio Artist", "Tagged Users", "Is Carousel", "Carousel Count", "Original Width", "Original Height", "Is Paid Partnership", "Comments Disabled", "Likes Disabled", "Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "coauthor_producers" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "coauthor_producers" ], "limit": 1, "ancestorLevel": 1 } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "article[role='presentation']" } ] } }, "comments": { "id": "comments", "title": "Comments of a Post", "description": "Works on post pages with comments", "instructions": [ "Go to a post page on Instagram", "Scroll down to the comments section", "Make sure comments are visible on the page" ], "type": "list", "urlField": "Author URL", "unit": { "singular": "Comment", "plural": "Comments" }, "frontField": "Comment Text", "fixes": [], "timeout": 10000, "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "comments" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "comment_like_count" }, "parse": true } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "time[datetime*='{{{id2}}}']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "article ul" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "article ul" } ] } ] }, { "type": "click", "target": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" }, { "method": "closest", "query": "button" } ], "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" } ] } ] }, { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "svg[aria-label='Comment']" }, { "method": "closest", "query": "section" }, { "method": "parentElement", "query": "" }, { "method": "parentElement", "query": "" }, { "method": "querySelector", "query": ":scope > div:nth-child(3)" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "stories": { "id": "stories", "title": "Stories", "description": "Works on stories viewer pages", "instructions": [ "Go to a user's stories on Instagram", "Make sure the story is playing or visible" ], "type": "list", "urlField": "Story Link", "unit": { "singular": "Story", "plural": "Stories" }, "frontField": "Story Date", "fixes": [], "timeout": 10000, "fields": [ "Story Type", "Story Image", "Story Video", "Story Link", "Story Date", "Story Expiring At", "Story Accessibility Caption", "Author Username", "Author Full Name", "Author Image", "Author URL", "Author Is Verified" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "story_cta" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "story_cta" }, "parse": true } ] } ] } } ], "log": false }, "condition": [ { "type": "url", "operator": "contains", "value": "stories" } ] }, "profile": { "id": "profile", "title": "Profile Details", "description": "Works on user profile pages", "instructions": [ "Navigate to a user's profile page on Instagram", "Make sure their profile information is visible" ], "type": "detail", "unit": { "singular": "Profile", "plural": "Profiles" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Biography", "Profile URL", "External URL", "Follower Count", "Following Count", "Post Count", "Is Verified", "Is Private", "Is Business", "Category", "Profile Picture URL", "All Bio Links" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/graphql/query", "data": [ "follower_count", "following_count", "biography" ] } } ] } }, { "delay": 0, "attempts": 1, "attemptLimit": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "following_count", "biography" ], "limit": 1, "ancestorLevel": 1 } } ] } } ], "log": false } }, "followers": { "id": "followers", "title": "Followers of a User", "description": "Works on the followers popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the followers count to open the followers list", "Make sure the followers list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Follower", "plural": "Followers" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/followers/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "following": { "id": "following", "title": "Following of a User", "description": "Works on the following popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the following count to open the following list", "Make sure the following list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Account", "plural": "Accounts" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/following/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } } }, "tests": [ { "option_id": "posts", "title": "Test Posts on Home Feed", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/", "url_updated_at": "2026-04-13T16:45:46.350Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts on Profile", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T16:49:39.489Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Saved", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/_7eight8/saved/all-posts/", "url_updated_at": "2026-04-13T16:51:08.551Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Reels Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/reels/", "https://www.instagram.com/markrober/reels/", "https://www.instagram.com/cristiano/reels/" ], "url_updated_at": "2026-04-13T16:51:58.657Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Tagged Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/tagged/", "https://www.instagram.com/cristiano/tagged/", "https://www.instagram.com/markrober/tagged/" ], "url_updated_at": "2026-04-13T16:53:01.569Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "postDetails", "title": "Test Post Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T16:54:49.214Z" } }, { "action": "select_option", "data": { "optionId": "postDetails" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "postDetails" } }, { "action": "cancel_option", "data": { "optionId": "postDetails" } } ] }, { "option_id": "comments", "title": "Test Post Comments", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T17:15:44.201Z" } }, { "action": "select_option", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "comments" } }, { "action": "cancel_option", "data": { "optionId": "comments" } } ] }, { "option_id": "stories", "title": "Test Stories", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/stories/highlights/17929440964811872/", "https://www.instagram.com/stories/highlights/17942782034297155/" ], "url_updated_at": "2026-04-13T17:18:12.361Z" } }, { "action": "select_option", "data": { "optionId": "stories" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "stories" } }, { "action": "cancel_option", "data": { "optionId": "stories" } } ] }, { "option_id": "profile", "title": "Test Profile Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:25:37.449Z" } }, { "action": "select_option", "data": { "optionId": "profile" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "profile" } }, { "action": "cancel_option", "data": { "optionId": "profile" } } ] }, { "option_id": "followers", "title": "Test Followers of a User", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:28:55.288Z" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "click", "data": { "selector": "a[href*='/followers/']" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "select_option", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "followers" } }, { "action": "cancel_option", "data": { "optionId": "followers" } } ] } ]}1// Browser-side scraping engine — injected into page via page.evaluate()2// All methods run in the real browser context (not Node.js)3
4window.scrapingEngine = {5
6 _shouldSkip: function(obj) {7 try {8 if (obj instanceof Date || obj instanceof RegExp || obj instanceof Error) return true;9 // Check for Window objects (including cross-origin)10 try { if (obj === obj.window) return true; } catch(e) { return true; }11 // Check valueOf — may throw on cross-origin12 try {13 if (typeof obj.valueOf === 'function') {14 var val = obj.valueOf();15 if (val !== obj && (typeof val !== 'object' || val === null)) return true;16 }17 } catch(e) { return true; }18 } catch(e) { return true; }19 return false;20 },21
22 fastDeepSearchField: function(obj, fieldName, opts) {23 opts = opts || {};24 var limit = opts.limit === undefined ? Infinity : opts.limit;25 var maxDepth = opts.maxDepth === undefined ? Infinity : opts.maxDepth;26 var ancestorLevel = opts.ancestorLevel || 0;27 var flexLimit = opts.flexLimit || false;28 var flexLimitThreshold = opts.flexLimitThreshold || 50000;29 var maxBreadth = opts.maxBreadth || Infinity;30 if (!obj || typeof obj !== 'object') return [];31
32 var self = this;33 var matcher = this.buildMatcher(fieldName);34 var results = [];35 var seen = new WeakSet();36 var stack = [[obj, 0, [obj]]];37
38 // flexLimit tracking39 var defaultFlexGap = flexLimitThreshold;40 var flexBuffer = 15000;41 var flexIterations = 0;42 var flexLastMatchAt = 0;43 var flexGaps = [];44
45 while (stack.length > 0 && (flexLimit || results.length < limit)) {46 if (flexLimit) {47 flexIterations++;48 var gap = flexIterations - flexLastMatchAt;49 var sorted = flexGaps.slice().sort(function(a, b) { return a - b; });50 var median = sorted.length > 0 ? sorted[sorted.length >> 1] : 0;51 var threshold = sorted.length > 0 ? (median + flexBuffer) : defaultFlexGap;52 if (gap > threshold) break;53 }54 var item = stack.pop();55 var current = item[0], depth = item[1], path = item[2];56 if (current == null || typeof current !== 'object') continue;57 if (self._shouldSkip(current)) continue;58 try { if (seen.has(current)) continue; } catch(e) { continue; }59 try { seen.add(current); } catch(e) { continue; }60
61 if (matcher(current)) {62 var result = current;63 if (ancestorLevel > 0) {64 var idx = path.length - 1 - ancestorLevel;65 if (idx >= 0) result = path[idx]; else continue;66 }67 results.push(result);68
69 if (flexLimit) {70 var matchGap = flexIterations - flexLastMatchAt;71 if (flexLastMatchAt > 0) flexGaps.push(matchGap);72 flexLastMatchAt = flexIterations;73 }74
75 if (!flexLimit && results.length >= limit) break;76 }77
78 if (depth >= maxDepth) continue;79
80 try {81 if (Array.isArray(current)) {82 var cappedLen = Math.min(current.length, maxBreadth);83 for (var i = cappedLen - 1; i >= 0; i--) {84 var v = current[i];85 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);86 }87 } else {88 var keys;89 try { keys = Object.keys(current); } catch(e) { continue; }90 var cappedKeysLen = Math.min(keys.length, maxBreadth);91 for (var i = cappedKeysLen - 1; i >= 0; i--) {92 var v;93 try { v = current[keys[i]]; } catch(e) { continue; }94 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);95 }96 }97 } catch(e) { }98 }99 return results;100 },101
102 buildMatcher: function(fieldSpec) {103 var self = this;104 if (typeof fieldSpec === 'string') {105 return function(obj) { return fieldSpec in obj; };106 }107 if (Array.isArray(fieldSpec)) {108 if (fieldSpec.length === 0) return function() { return true; };109 if (Array.isArray(fieldSpec[0])) {110 var groups = fieldSpec.map(function(group) { return group.map(function(f) { return self.buildMatcher(f); }); });111 return function(obj) { return groups.some(function(group) { return group.every(function(m) { return m(obj); }); }); };112 }113 var matchers = fieldSpec.map(function(f) { return self.buildMatcher(f); });114 return function(obj) { return matchers.every(function(m) { return m(obj); }); };115 }116 if (typeof fieldSpec === 'object' && fieldSpec.field) {117 var field = fieldSpec.field, value = fieldSpec.value, mode = fieldSpec.mode || 'equal';118 if (value === undefined || value === null) return function(obj) { return field in obj; };119 return function(obj) {120 if (!(field in obj)) return false;121 var v = String(obj[field]).toLowerCase();122 var expected = String(value).toLowerCase();123 return mode === 'includes' ? v.indexOf(expected) !== -1 : v === expected;124 };125 }126 return function() { return false; };127 },128
129 searchReactRoot: function(element, searchConfig) {130 var reactKey = searchConfig.reactKey || '__reactFiber';131 var searchProperty = searchConfig.searchProperty;132 var limit = searchConfig.limit || 1;133 var ancestorLevel = searchConfig.ancestorLevel || 0;134 var maxDepth = searchConfig.maxDepth || Infinity;135 var flexLimit = searchConfig.flexLimit || false;136 var flexLimitThreshold = searchConfig.flexLimitThreshold || 50000;137 var maxBreadth = searchConfig.maxBreadth || Infinity;138 var keys = Object.keys(element);139 for (var i = 0; i < keys.length; i++) {140 if (keys[i].indexOf(reactKey) === 0) {141 return this.fastDeepSearchField(element[keys[i]], searchProperty, { limit: limit, maxDepth: maxDepth, ancestorLevel: ancestorLevel, flexLimit: flexLimit, flexLimitThreshold: flexLimitThreshold, maxBreadth: maxBreadth });142 }143 }144 return null;145 },146
147 DOMSearch: function(searchConfig) {148 if (!Array.isArray(searchConfig)) return [];149 var self = this;150 var results = [];151 for (var q = 0; q < searchConfig.length; q++) {152 var query = searchConfig[q];153 if (!query || !query.key) continue;154 try {155 var elements = self._getElements(query, document);156 if (query.filter) elements = self._filterElements(elements, query.filter);157 if (elements.length === 0) { results.push({ key: query.key, value: null }); continue; }158
159 if (query.nested && query.nested.length > 0) {160 // Nested queries: for each matched element, run sub-queries relative to it161 var nestedResults = self._processNested(elements, query.nested);162 var parsed = query.parse ? nestedResults.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : nestedResults;163 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });164 } else {165 var vals = self._extractValues(elements, query);166 var parsed = query.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;167 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });168 }169 } catch(e) { results.push({ key: query.key, value: null }); }170 }171 return results;172 },173
174 _getElements: function(query, baseEl) {175 var elements = [];176 if (query.type === 'xpath') {177 var xr = document.evaluate(query.query, baseEl, null, query.all ? XPathResult.ORDERED_NODE_ITERATOR_TYPE : XPathResult.FIRST_ORDERED_NODE_TYPE, null);178 if (query.all) { var n = xr.iterateNext(); while (n) { elements.push(n); n = xr.iterateNext(); } }179 else { if (xr.singleNodeValue) elements.push(xr.singleNodeValue); }180 } else if (query.type === 'selector') {181 elements = query.all ? Array.from(baseEl.querySelectorAll(query.query)) : (baseEl.querySelector(query.query) ? [baseEl.querySelector(query.query)] : []);182 } else if (query.type === 'document') {183 elements = [baseEl];184 }185 return elements;186 },187
188 _filterElements: function(elements, filter) {189 if (!filter) return elements;190 return elements.filter(function(el) {191 var text = el.textContent || '';192 if (filter.contains) {193 if (Array.isArray(filter.contains)) { if (!filter.contains.every(function(s) { return text.indexOf(s) !== -1; })) return false; }194 else if (text.indexOf(filter.contains) === -1) return false;195 }196 if (filter.notContains) {197 if (Array.isArray(filter.notContains)) { if (filter.notContains.some(function(s) { return text.indexOf(s) !== -1; })) return false; }198 else if (text.indexOf(filter.notContains) !== -1) return false;199 }200 if (filter.startsWith && !text.trimStart().startsWith(filter.startsWith)) return false;201 if (filter.endsWith && !text.trimEnd().endsWith(filter.endsWith)) return false;202 if (typeof filter.minLength === 'number' && text.length < filter.minLength) return false;203 if (typeof filter.maxLength === 'number' && text.length > filter.maxLength) return false;204 return true;205 });206 },207
208 _extractValues: function(elements, query) {209 var value = query.value || 'text';210 var custom = query.custom || null;211 return elements.map(function(el) {212 if (value === 'text') return el.textContent || '';213 if (value === 'innerText') return el.innerText || '';214 if (value === 'html') return el.innerHTML || '';215 if (value === 'attribute' && custom && custom.attributeKey) return el.getAttribute(custom.attributeKey) || null;216 return el.textContent || '';217 });218 },219
220 _processNested: function(baseElements, nestedQueries) {221 var self = this;222 var results = [];223 for (var i = 0; i < baseElements.length; i++) {224 var baseEl = baseElements[i];225 var elementResult = {};226 for (var n = 0; n < nestedQueries.length; n++) {227 var nq = nestedQueries[n];228 if (!nq || !nq.key) continue;229 try {230 var childElements = self._getElements(nq, baseEl);231 if (nq.filter) childElements = self._filterElements(childElements, nq.filter);232 if (childElements.length === 0) { elementResult[nq.key] = null; continue; }233
234 if (nq.nested && nq.nested.length > 0) {235 var deepNested = self._processNested(childElements, nq.nested);236 var parsed = nq.parse ? deepNested.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : deepNested;237 elementResult[nq.key] = nq.all ? parsed : parsed[0] || null;238 } else {239 var vals = self._extractValues(childElements, nq);240 var parsed = nq.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;241 elementResult[nq.key] = nq.all ? parsed : vals[0] || null;242 }243 } catch(e) { elementResult[nq.key] = null; }244 }245 results.push(elementResult);246 }247 return results;248 }249};1// Anti-detection enhancements — applied in preNavigationHooks and postNavigationHooks2
3import UserAgent from 'user-agents';4
5const TRACKER_DOMAINS = [6 'google-analytics.com', 'googletagmanager.com', 'facebook.net',7 'doubleclick.net', 'hotjar.com', 'segment.com', 'mixpanel.com',8 'amplitude.com', 'fullstory.com', 'clarity.ms', 'newrelic.com',9 'sentry.io', 'bugsnag.com', 'datadoghq.com', 'mouseflow.com',10 'crazyegg.com', 'optimizely.com', 'hubspot.com', 'intercom.io',11 'drift.com', 'tawk.to', 'livechatinc.com'12];13
14const COOKIE_SELECTORS = [15 '[data-testid="cookie-policy-dialog-accept-button"]',16 'button[id*="accept"]', 'button[class*="accept"]',17 'button[class*="consent"]', '#onetrust-accept-btn-handler',18 '.cookie-consent-accept', '[aria-label*="Accept"]',19 '[aria-label*="accept"]', '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',20 '.cc-btn.cc-allow', '#gdpr-cookie-accept', '.js-cookie-consent-agree'21];22
23export async function applyPreNavigation(page, enhancements) {24 if (enhancements.blockTrackers) {25 await page.route('**/*', (route) => {26 const url = route.request().url();27 if (TRACKER_DOMAINS.some(d => url.includes(d))) return route.abort();28 return route.continue();29 });30 }31
32 if (enhancements.rotateUserAgent) {33 const ua = new UserAgent({ deviceCategory: 'desktop' });34 await page.setExtraHTTPHeaders({ 'User-Agent': ua.toString() });35 }36}37
38export async function applyPostNavigation(page, enhancements) {39 if (enhancements.dismissCookies) {40 try {41 for (const sel of COOKIE_SELECTORS) {42 const btn = await page.$(sel);43 if (btn) { await btn.click().catch(() => {}); break; }44 }45 } catch { }46 }47
48 // Post-navigation delay (radio: none / random / custom)49 const postNav = enhancements.postNavigation || 'none';50 if (postNav === 'random') {51 await page.waitForTimeout(1000 + Math.floor(Math.random() * 2000));52 } else if (postNav === 'custom') {53 const delay = typeof enhancements.postNavigationDelay === 'number' ? enhancements.postNavigationDelay : 2000;54 await page.waitForTimeout(delay);55 }56}1import { log } from 'crawlee';2
3/**4 * Execute a custom user script in the browser context.5 * The script should define: async function main(context) { ... }6 * context = { url, hostname, pathname, searchParams, optionKey, timing }7 */8export async function executeCustomScript(page, enhancements, context) {9 const code = enhancements.customScriptCode;10 if (!code) return;11
12 try {13 log.info('Custom script: executing (' + context.timing + ')');14 await page.evaluate(async ({ scriptCode, ctx }) => {15 const fn = new Function('context', `return (async function() { ${scriptCode} }).call(null)`);16 await fn(ctx);17 }, { scriptCode: code, ctx: context });18 log.info('Custom script: completed');19 } catch (err) {20 log.warning('Custom script error: ' + err.message);21 }22}23
24/**25 * Execute a beforeLoad custom script in Node context.26 * Can return a modified URL string.27 */28export function executeBeforeLoadScript(enhancements, url) {29 const code = enhancements.customScriptCode;30 if (!code) return url;31
32 try {33 const parsed = new URL(url);34 const context = {35 url,36 hostname: parsed.hostname,37 pathname: parsed.pathname,38 searchParams: Object.fromEntries(parsed.searchParams),39 timing: 'beforeLoad'40 };41 const fn = new Function('context', code);42 const result = fn(context);43 if (typeof result === 'string') {44 log.info('Custom script (beforeLoad): URL modified to ' + result);45 return result;46 }47 } catch (err) {48 log.warning('Custom script (beforeLoad) error: ' + err.message);49 }50 return url;51}1// Data processing enhancements — raw data, criteria filtering2
3export function formatForDataset(item, enhancements) {4 if (enhancements.rawData) {5 return { ...item.formatted, _raw: item.raw };6 }7 return item.formatted || item;8}9
10// ── Criteria filtering ───────────────────────────────────────11
12export function checkCondition(value, condition) {13 const { operator, value: target, caseSensitive } = condition;14 const str = String(value ?? '');15 const cmp = caseSensitive ? str : str.toLowerCase();16 const tgt = caseSensitive ? target : target.toLowerCase();17 switch (operator) {18 case 'equals': return cmp === tgt;19 case 'notEquals': return cmp !== tgt;20 case 'contains': return cmp.includes(tgt);21 case 'notContains': return !cmp.includes(tgt);22 case 'regex': try { return new RegExp(target, caseSensitive ? '' : 'i').test(str); } catch { return false; }23 case 'greaterThan': return parseFloat(str) > parseFloat(target);24 case 'lessThan': return parseFloat(str) < parseFloat(target);25 case 'dateBefore': return new Date(str) < new Date(target);26 case 'dateOn': return new Date(str).toDateString() === new Date(target).toDateString();27 case 'dateAfter': return new Date(str) > new Date(target);28 default: return true;29 }30}31
32export function applyCriteria(items, criteria) {33 if (!criteria || typeof criteria !== 'object' || Object.keys(criteria).length === 0) return items;34 return items.filter(item => {35 for (const [field, conditions] of Object.entries(criteria)) {36 if (!Array.isArray(conditions)) continue;37 const val = item[field];38 for (const cond of conditions) {39 if (!checkCondition(val, cond)) return false;40 }41 }42 return true;43 });44}1// Pagination enhancement — executes scroll/click pagination from config2// Mirrors extension's AutoPaginationManager: data-driven stopping with retry attempts3
4export async function runPagination(page, option, enhancements, processInBrowser) {5 if (!enhancements.pagination || !option.paginate) return [];6
7 const log = (await import('crawlee')).log;8 const paginateConfig = option.paginate;9 const maxPages = typeof enhancements.maxPages === 'number' ? enhancements.maxPages : 0; // 0 = infinite10 const delay = paginateConfig.delay || 3000;11 const maxAttempts = 2;12 const allResults = [];13
14 let pageNum = 0;15 let attempts = 0;16
17 log.info('Pagination: starting, maxPages=' + (maxPages || 'infinite') + ', delay=' + delay + 'ms, instructions=' + (paginateConfig.instructions || []).length);18
19 while (true) {20 if (maxPages > 0 && pageNum >= maxPages) {21 log.info('Pagination: reached maxPages limit (' + maxPages + ')');22 break;23 }24
25 pageNum++;26 log.info('Pagination: page ' + pageNum + (maxPages > 0 ? '/' + maxPages : '') + ' url=' + page.url());27
28 let acted = await executeInstructions(page, paginateConfig.instructions || [], log);29
30 if (!acted) {31 log.info('Pagination: waiting 8s for page to finish rendering...');32 await page.waitForTimeout(8000);33 acted = await executeInstructions(page, paginateConfig.instructions || [], log);34 }35
36 if (!acted) {37 attempts++;38 log.info('Pagination: nothing executed after wait, attempt ' + attempts + '/' + maxAttempts);39 if (attempts >= maxAttempts) {40 log.info('Pagination: stopping — no pagination action possible');41 break;42 }43 continue;44 }45
46 await page.waitForTimeout(delay);47
48 const newItems = await processInBrowser(page);49
50 log.info('Pagination: page ' + pageNum + ' got ' + newItems.length + ' new items');51
52 if (newItems.length > 0) {53 allResults.push(...newItems);54 attempts = 0;55 log.info('Pagination: total so far ' + allResults.length + ' items, continuing');56 } else {57 attempts++;58 log.info('Pagination: no new data, attempt ' + attempts + '/' + maxAttempts);59 if (attempts >= maxAttempts) {60 log.info('Pagination: stopping after ' + maxAttempts + ' attempts with no new data');61 break;62 }63 }64 }65
66 log.info('Pagination: done, ' + pageNum + ' pages, ' + allResults.length + ' total new items');67 return allResults;68}69
70// Execute pagination instructions (scroll, click, delay) with full target chain support71async function executeInstructions(page, instructions, log) {72 let anyExecuted = false;73
74 for (let i = 0; i < instructions.length; i++) {75 const instruction = instructions[i];76 try {77 if (instruction.condition) {78 const condMet = await checkCondition(page, instruction.condition);79 if (!condMet) {80 log.info('Pagination: instruction[' + i + '] condition not met, skipping');81 continue;82 }83 }84
85 anyExecuted = true;86
87 if (instruction.type === 'delay') {88 await page.waitForTimeout(instruction.amount || 1000);89 } else if (instruction.type === 'scroll') {90 await executeScroll(page, instruction, log);91 } else if (instruction.type === 'click') {92 await executeClick(page, instruction, log);93 }94 } catch (err) {95 log.info('Pagination: instruction[' + i + '] error: ' + err.message);96 }97 }98
99 if (!anyExecuted) {100 log.info('Pagination: all conditions failed, no action taken');101 }102
103 return anyExecuted;104}105
106// Scroll with full support: scrollTo bottom, scrollBy, steps, presets, chained targets107async function executeScroll(page, instruction, log) {108 const { scrollType, amount, steps } = instruction;109 const el = await findElement(page, instruction.target);110
111 const stepCount = steps?.amount || 1;112 const stepDelay = steps?.delay || 100;113
114 if (scrollType === 'to' && amount?.type === 'preset' && amount.value === 'bottom') {115 for (let s = 0; s < stepCount; s++) {116 if (el) {117 await el.evaluate((e, args) => {118 e.scrollTo(0, (e.scrollHeight / args.total) * (args.step + 1));119 }, { step: s, total: stepCount });120 } else {121 await page.evaluate(({ step, total }) => {122 window.scrollTo(0, (document.body.scrollHeight / total) * (step + 1));123 }, { step: s, total: stepCount });124 }125 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);126 }127 log.info('Pagination: scrolled to bottom' + (stepCount > 1 ? ' in ' + stepCount + ' steps' : ''));128 } else if (scrollType === 'by') {129 const scrollAmount = amount?.value || 0;130 const stepSize = scrollAmount / stepCount;131 for (let s = 0; s < stepCount; s++) {132 if (el) {133 await el.evaluate((e, amt) => e.scrollBy(0, amt), stepSize);134 } else {135 await page.evaluate((amt) => window.scrollBy(0, amt), stepSize);136 }137 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);138 }139 log.info('Pagination: scrolled by ' + scrollAmount + 'px');140 } else {141 if (el) {142 await el.evaluate(e => e.scrollTo(0, e.scrollHeight));143 } else {144 await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));145 }146 log.info('Pagination: scrolled');147 }148}149
150// Click with chained target support151async function executeClick(page, instruction, log) {152 const target = instruction.target;153 const step = Array.isArray(target) ? target[0] : target;154 const query = step?.query;155 const method = step?.method;156
157 if (query) {158 const clicked = await page.evaluate(({ query, method }) => {159 let el;160 if (method === 'xpath') {161 el = document.evaluate(query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;162 } else {163 el = document.querySelector(query);164 }165 if (el) { el.click(); return true; }166 return false;167 }, { query, method });168
169 if (clicked) {170 log.info('Pagination: clicked');171 } else {172 log.info('Pagination: click target not found');173 }174 } else {175 log.info('Pagination: no click query');176 }177}178
179// Find element with chained target support (matches extension's unified {method, query} format)180async function findElement(page, target) {181 if (!target) return null;182
183 if (!Array.isArray(target)) target = [target];184
185 let el = null;186 for (const step of target) {187 if (!step.method) continue;188
189 if (step.method === 'querySelector') {190 el = el ? await applyMethod(page, el, step) : await page.$(step.query);191 } else if (step.method === 'xpath') {192 el = el ? await applyMethod(page, el, step) : await page.$('xpath=' + step.query);193 } else if (step.method === 'preset') {194 if (['window', 'document'].includes(step.query?.toLowerCase())) return null;195 if (step.query?.toLowerCase() === 'body') el = await page.$('body');196 } else if (el) {197 el = await applyMethod(page, el, step);198 }199 if (!el) return null;200 }201 return el;202}203
204// Apply chained DOM methods (matches extension's Paginater.applyMethod)205async function applyMethod(page, el, step) {206 const { method, query } = step;207 try {208 switch (method) {209 case 'closest':210 return await el.evaluateHandle((e, q) => e.closest(q), query);211 case 'parentElement':212 return await el.evaluateHandle(e => e.parentElement);213 case 'querySelector':214 return await el.evaluateHandle((e, q) => e.querySelector(q), query);215 case 'nextElementSibling':216 return await el.evaluateHandle(e => e.nextElementSibling);217 case 'previousElementSibling':218 return await el.evaluateHandle(e => e.previousElementSibling);219 case 'firstElementChild':220 return await el.evaluateHandle(e => e.firstElementChild);221 case 'lastElementChild':222 return await el.evaluateHandle(e => e.lastElementChild);223 default:224 return null;225 }226 } catch {227 return null;228 }229}230
231async function checkCondition(page, conditions) {232 if (!Array.isArray(conditions)) return true;233 return page.evaluate((conds) => {234 for (const cond of conds) {235 if (cond.type === 'element') {236 const target = cond.value?.[0];237 if (!target) continue;238 let el = null;239 if (target.method === 'querySelector') {240 el = document.querySelector(target.query);241 } else if (target.method === 'xpath') {242 el = document.evaluate(target.query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;243 }244 if (cond.operator === 'exists' && !el) return false;245 if (cond.operator === 'not_exists' && el) return false;246 } else if (cond.type === 'url') {247 const url = window.location.href;248 if (cond.operator === 'contains' && !url.includes(cond.value)) return false;249 if (cond.operator === 'notContains' && url.includes(cond.value)) return false;250 if (cond.operator === 'startsWith' && !url.startsWith(cond.value)) return false;251 if (cond.operator === 'endsWith' && !url.endsWith(cond.value)) return false;252 if (cond.operator === 'equals' && url !== cond.value) return false;253 if (cond.operator === 'notEquals' && url === cond.value) return false;254 }255 }256 return true;257 }, conditions);258}1// Performance enhancements — resource blocking, crawler config, session pool2
3export async function applyResourceBlocking(page, enhancements) {4 const blockTypes = [];5 if (enhancements.blockImages) blockTypes.push('image');6 if (enhancements.blockMedia) blockTypes.push('media', 'font');7 if (enhancements.blockStylesheets) blockTypes.push('stylesheet');8
9 if (blockTypes.length > 0) {10 await page.route('**/*', (route) => {11 if (blockTypes.includes(route.request().resourceType())) return route.abort();12 return route.continue();13 });14 }15}16
17export async function getCrawlerOptions(enhancements) {18 const { Actor } = await import('apify');19 const opts = {};20
21 // Max timeout (0 = no limit)22 const timeout = typeof enhancements.maxTimeout === 'number' ? enhancements.maxTimeout : 120;23 opts.requestHandlerTimeoutSecs = timeout === 0 ? 999999 : timeout;24
25 // Max concurrent tabs — use autoscaling instead of hard concurrency26 const maxTabs = typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1;27 opts.maxConcurrency = maxTabs;28 opts.minConcurrency = 1;29 if (maxTabs > 1) {30 opts.autoscaledPoolOptions = {31 desiredConcurrencyRatio: 0.85,32 scaleUpStepRatio: 0.5,33 scaleDownStepRatio: 0.2,34 snapshotterOptions: {35 maxBlockedMillis: 2000,36 }37 };38 }39
40 // Session pool — proper Apify SessionPool integration41 if (enhancements.sessionPool) {42 const maxUsage = typeof enhancements.sessionMaxUsage === 'number' ? enhancements.sessionMaxUsage : 5;43 opts.useSessionPool = true;44 opts.persistCookiesPerSession = true;45 opts.sessionPoolOptions = {46 maxPoolSize: 20,47 sessionOptions: {48 maxAgeSecs: 300,49 maxUsageCount: maxUsage,50 }51 };52 }53
54 // Proxy configuration55 if (enhancements.proxyApifyResidential) {56 opts.proxyConfiguration = await Actor.createProxyConfiguration({57 groups: ['RESIDENTIAL']58 });59 } else if (enhancements.proxyApifyDatacenter) {60 opts.proxyConfiguration = await Actor.createProxyConfiguration();61 }62
63 return opts;64}{ "stealth": true, "waitUntil": "domcontentloaded", "postNavigation": "none", "navigationTimeout": 45, "maxTimeout": 0, "retryNavigation": 0, "blockImages": true, "blockMedia": true, "blockStylesheets": true, "blockTrackers": true, "actorMemory": 2048, "maxTabs": 3, "reuseContext": true, "retryIfEmpty": true, "retryMaxAttempts": 2, "abortOnLoss": true, "abortWarmupUrls": 5, "abortWarmupSeconds": 60, "_optionKey": "comments"}1// DOM scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 return window.scrapingEngine.DOMSearch(searchConfig);8 }, method.searchConfig);9
10 return rawData;11}1// React fiber scraping handler2
3export async function scrape(page, option, enhancements) {4 // Extract the single method from the synthetic option5 const method = option.scraping.partials[0].methods.A[0];6 const sc = method.searchConfig;7
8 const rawData = await page.evaluate(({ selector, searchConfig }) => {9 const el = document.querySelector(selector);10 if (!el) return null;11 return window.scrapingEngine.searchReactRoot(el, searchConfig);12 }, { selector: sc.reactSelector, searchConfig: sc });13
14 return rawData;15}1// Network request interception handler2import { log } from 'crawlee';3
4export async function scrape(page, option, enhancements, capturedResponses) {5 if (capturedResponses.length === 0) return null;6
7 // Use per-page processed timestamps stored on the array itself8 const processedTimestamps = capturedResponses._processedTimestamps || (capturedResponses._processedTimestamps = new Set());9
10 // Filter out already-processed responses11 const newResponses = capturedResponses.filter(r => r._timestamp && !processedTimestamps.has(r._timestamp));12 if (newResponses.length === 0) return null;13
14 // Mark as processed15 for (const r of newResponses) {16 processedTimestamps.add(r._timestamp);17 }18
19 return newResponses.map(r => {20 const { _timestamp, ...data } = r;21 return data;22 });23}24
25// Called from main.js preNavigationHooks to set up response listeners before navigation26export function setupInterception(page, option, capturedResponses) {27 const searchConfigs = [];28 for (const partial of option.scraping?.partials || []) {29 for (const [, methods] of Object.entries(partial.methods || {})) {30 for (const method of methods) {31 if (method.type === 'request') searchConfigs.push(method.searchConfig);32 }33 }34 }35
36 page.on('response', async (response) => {37 const url = response.url();38 for (const sc of searchConfigs) {39 const matchUrl = sc.url ? url.includes(sc.url) : true;40 if (!matchUrl) continue;41 try {42 const body = await response.text();43 let json;44 try { json = JSON.parse(body); } catch { continue; }45 const dataMatch = sc.data46 ? (Array.isArray(sc.data) ? sc.data.every(d => body.includes(d)) : body.includes(sc.data))47 : true;48 if (dataMatch) {49 json._timestamp = Date.now();50 capturedResponses.push(json);51 }52 } catch { }53 }54 });55}1// Window object scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 const nestedKey = searchConfig.nestedKey;8 const keys = Array.isArray(nestedKey) ? nestedKey : [nestedKey];9 const results = [];10 for (const key of keys) {11 let value = window;12 for (const part of key.split('.')) {13 if (value == null) break;14 value = value[part];15 }16 if (value != null) {17 if (Array.isArray(value)) results.push(...value);18 else results.push(value);19 }20 }21 return results.length > 0 ? results : null;22 }, method.searchConfig);23
24 return rawData;25}1import { PlaywrightCrawler, Dataset, log, KeyValueStore } from 'crawlee';2import { Actor } from 'apify';3import { fileURLToPath } from 'url';4import { dirname, join } from 'path';5import { readFileSync } from 'fs';6
7import { applyPreNavigation, applyPostNavigation } from './enhancements/anti-detection.js';8import { applyResourceBlocking, getCrawlerOptions } from './enhancements/performance.js';9import { runPagination } from './enhancements/pagination.js';10import { formatForDataset } from './enhancements/data.js';11import { executeCustomScript, executeBeforeLoadScript } from './enhancements/custom-script.js';12
13// Import ALL handlers14import * as reactHandler from './handlers/react.js';15import * as requestHandler from './handlers/request.js';16import * as domHandler from './handlers/dom.js';17import * as windowHandler from './handlers/window.js';18
19const handlerMap = {20 react: reactHandler,21 request: requestHandler,22 dom: domHandler,23 window: windowHandler24};25
26const __dirname = dirname(fileURLToPath(import.meta.url));27
28// ── Load config files ────────────────────────────────────────────29const config = JSON.parse(readFileSync(join(__dirname, 'config.json'), 'utf-8'));30const enhancements = JSON.parse(readFileSync(join(__dirname, 'enhancements.json'), 'utf-8'));31
32// ── Debug flags (set via test tab debug toggles) ─────────────────33const dbg = enhancements._debug || {};34const D = {35 config: dbg.config || false,36 navigation: dbg.navigation || false,37 injection: dbg.injection || false,38 partials: dbg.partials || false,39 rawData: dbg.rawData || false,40 processing: dbg.processing || false,41 pagination: dbg.pagination || false,42 requests: dbg.requests || false,43 browser: dbg.browser || false,44 performance: dbg.performance || false,45 screenshot: dbg.screenshot || false,46};47
48// Read script.js and engine.js as strings — they run in the BROWSER, not Node.js49const scriptCode = readFileSync(join(__dirname, 'script.js'), 'utf-8');50const engineCode = readFileSync(join(__dirname, 'engine.js'), 'utf-8');51
52const optionKey = enhancements._optionKey;53const option = config.options[optionKey];54if (!option) {55 log.error('Option "' + optionKey + '" not found in config');56 process.exit(1);57}58
59if (D.config) {60 log.info('DEBUG[config] optionKey=' + optionKey);61 log.info('DEBUG[config] enhancements=' + JSON.stringify(Object.keys(enhancements).filter(k => !k.startsWith('_') && enhancements[k])));62 log.info('DEBUG[config] hostnames=' + JSON.stringify(config.app.hostnames));63 log.info('DEBUG[config] fields=' + JSON.stringify(option.fields?.length || 0));64}65
66const hostnames = config.app.hostnames || [];67
68function validateUrl(url) {69 try {70 const parsed = new URL(url);71 return hostnames.some(h => parsed.hostname === h || parsed.hostname.endsWith('.' + h));72 } catch {73 return false;74 }75}76
77// Check if any partial uses request interception78function hasRequestMethods(opt) {79 for (const partial of opt.scraping?.partials || []) {80 for (const methods of Object.values(partial.methods || {})) {81 for (const method of methods) {82 if (method.type === 'request') return true;83 }84 }85 }86 return false;87}88
89const needsRequestInterception = hasRequestMethods(option);90
91function getPartialMethodType(partial) {92 if (!partial?.methods) return 'unknown';93 for (const methods of Object.values(partial.methods)) {94 for (const method of methods) {95 if (method.type) return method.type;96 }97 }98 return 'unknown';99}100
101// ── Signature tracking (mirrors extension's processedSignatures) ──────102
103let processedSignatures = new Set();104
105function hashString(str) {106 let hash = 0;107 for (let i = 0; i < str.length; i++) {108 const char = str.charCodeAt(i);109 hash = ((hash << 5) - hash) + char;110 hash = hash & hash;111 }112 return Math.abs(hash).toString(16).padStart(8, '0');113}114
115function generateSignature(data) {116 const seen = new WeakSet();117 return hashString(JSON.stringify(data, function(key, value) {118 if (typeof value === 'object' && value !== null) {119 if (seen.has(value)) return '[Circular]';120 seen.add(value);121 }122 return value;123 }));124}125
126// ── Build enhancement overrides for a retry attempt ──────────────127
128function getRetryEnhancements(attempt) {129 if (!enhancements.retryIfEmpty || !enhancements.retryEnhancements) return enhancements;130 const overrides = enhancements.retryEnhancements[attempt];131 if (!overrides) return enhancements;132 return { ...enhancements, ...overrides };133}134
135// ── Inject code into browser ─────────────────────────────────────136
137async function injectIntoPage(page) {138 if (D.injection) log.info('DEBUG[injection] Injecting engine.js and script.js into page');139 await page.evaluate((code) => {140 if (!window.scrapingEngine) new Function(code)();141 }, engineCode);142
143 await page.evaluate((code) => {144 if (!window.__scriptInjected) {145 new Function(code)();146 window.__scriptInjected = true;147 }148 }, scriptCode);149
150 if (D.browser) {151 const info = await page.evaluate((optKey) => {152 const fns = Object.keys(window).filter(k => { try { return typeof window[k] === 'function'; } catch { return false; } });153 const customFns = fns.filter(k => !['addEventListener','removeEventListener','dispatchEvent','eval','isFinite','isNaN','parseFloat','parseInt','decodeURI','decodeURIComponent','encodeURI','encodeURIComponent','escape','unescape','toString','valueOf'].includes(k) && !k.startsWith('on') && !k.startsWith('webkit') && k.length > 2);154 return {155 processFnExists: typeof window[optKey] === 'function',156 searchField: typeof window.searchField,157 scrapingEngine: typeof window.scrapingEngine,158 scriptInjected: !!window.__scriptInjected,159 customFnCount: customFns.length,160 sampleFns: customFns.slice(0, 15).join(',')161 };162 }, optionKey);163 log.info('DEBUG[browser] ' + JSON.stringify(info));164 }165
166 if (D.injection) log.info('DEBUG[injection] Done');167}168
169// ── Execute a single method (mirrors controller.js executeMethod) ──170
171async function executeMethod(page, method, capturedResponses) {172 const type = method.type;173 const handler = handlerMap[type];174 if (!handler) return { success: false, error: 'Unknown method type: ' + type };175
176 if (D.partials) log.info('DEBUG[partials] Executing method type=' + type + ' delay=' + (method.delay || 0) + ' attempts=' + (method.attempts || 1));177
178 try {179 let rawData;180 if (type === 'request') {181 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements, capturedResponses);182 } else {183 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements);184 }185
186 if (rawData && (Array.isArray(rawData) ? rawData.length > 0 : true)) {187 if (D.rawData) {188 log.info('DEBUG[rawData] type=' + type + ' isArray=' + Array.isArray(rawData) + ' length=' + (Array.isArray(rawData) ? rawData.length : 'N/A'));189 if (Array.isArray(rawData) && rawData.length > 0) {190 const sample = rawData[0];191 log.info('DEBUG[rawData] firstItem type=' + typeof sample + ' keys=' + (typeof sample === 'object' && sample ? Object.keys(sample).slice(0, 8).join(',') : 'N/A'));192 }193 }194 return { success: true, data: rawData };195 }196 if (D.partials) log.info('DEBUG[partials] method type=' + type + ' returned no data');197 return { success: false, error: 'No data returned' };198 } catch (err) {199 if (D.partials) log.error('DEBUG[partials] method type=' + type + ' error: ' + err.message);200 return { success: false, error: err.message };201 }202}203
204// ── Evaluate method conditions in browser context ────────────────205
206async function evaluateConditions(page, conditions) {207 if (!conditions || !Array.isArray(conditions) || conditions.length === 0) return true;208 return page.evaluate((conds) => {209 function evalSingle(cond) {210 const { type, operator, value } = cond;211 if (!type || !operator || value === undefined) return false;212 if (type === 'url') {213 const url = window.location.href;214 switch (operator) {215 case 'equals': return url === value;216 case 'contains': return url.includes(value);217 case 'startsWith': return url.startsWith(value);218 case 'endsWith': return url.endsWith(value);219 case 'regex': try { return new RegExp(value).test(url); } catch { return false; }220 case 'notEquals': return url !== value;221 case 'notContains': return !url.includes(value);222 default: return false;223 }224 }225 if (type === 'element') {226 if (!Array.isArray(value)) return false;227 // Check element existence/visibility using selectors from value array228 for (const v of value) {229 const query = v.query || v;230 if (typeof query !== 'string') continue;231 const el = document.querySelector(query);232 if (operator === 'exists' && !el) return false;233 if (operator === 'not_exists' && el) return false;234 if (operator === 'visible') {235 if (!el) return false;236 const rect = el.getBoundingClientRect();237 const style = window.getComputedStyle(el);238 if (rect.width === 0 || rect.height === 0 || style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;239 }240 }241 return true;242 }243 return false;244 }245
246 function evalGroup(arr) {247 if (!Array.isArray(arr) || arr.length === 0) return true;248 // If first element is array → OR logic249 if (Array.isArray(arr[0])) {250 return arr.some(sub => evalGroup(sub));251 }252 // AND logic253 return arr.every(item => {254 if (Array.isArray(item)) return evalGroup(item);255 if (typeof item === 'object') return evalSingle(item);256 return false;257 });258 }259 return evalGroup(conds);260 }, conditions);261}262
263// ── Process partials (mirrors controller.js processInstructions) ──264
265async function processAllPartials(page, capturedResponses) {266 const scraping = option.scraping;267 if (!scraping?.partials) return [];268
269 const partials = scraping.partials;270 const qualityOrder = ['A', 'B', 'C'];271
272 // Run all partials in parallel273 const partialPromises = partials.map(async (partial, pi) => {274 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] delay=' + (partial.delay || 0) + ' attempts=' + (partial.attempts || 1) + ' group=' + (partial.group || 'none') + ' methods=' + Object.keys(partial.methods || {}).join(','));275 if (partial.delay) await page.waitForTimeout(partial.delay);276
277 const partialAttempts = partial.attempts || 1;278
279 for (let attempt = 0; attempt < partialAttempts; attempt++) {280 for (const quality of qualityOrder) {281 const methods = partial.methods?.[quality];282 if (!methods || !Array.isArray(methods)) continue;283
284 for (const method of methods) {285 if (!method?.type) continue;286
287 // Check method conditions288 if (method.condition && Array.isArray(method.condition)) {289 const condMet = await evaluateConditions(page, method.condition);290 if (!condMet) {291 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] method condition not met, skipping');292 continue;293 }294 }295
296 if (method.delay) await page.waitForTimeout(method.delay);297
298 const methodAttempts = method.attempts || 1;299 for (let mAttempt = 0; mAttempt < methodAttempts; mAttempt++) {300 const result = await executeMethod(page, method, capturedResponses);301 if (result.success) {302 const signature = generateSignature(result.data);303 const alreadyProcessed = processedSignatures.has(signature);304 if (!alreadyProcessed) processedSignatures.add(signature);305 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] signature=' + signature + ' alreadyProcessed=' + alreadyProcessed);306 return { success: true, data: result.data, partialIndex: pi, alreadyProcessed };307 }308 if (mAttempt < methodAttempts - 1) await page.waitForTimeout(500);309 }310 }311 }312
313 if (attempt < partialAttempts - 1) await page.waitForTimeout(500);314 }315
316 return { success: false, partialIndex: pi };317 });318
319 const results = await Promise.all(partialPromises);320
321 // Group logic — mirrors extension's processInstructions322 const groupedPartials = new Map();323 const independentIndices = [];324
325 partials.forEach((partial, index) => {326 const group = partial.group;327 if (group) {328 const groups = Array.isArray(group) ? group : (typeof group === 'string' ? [group] : []);329 if (groups.length > 0) {330 groups.forEach(g => {331 if (!groupedPartials.has(g)) groupedPartials.set(g, []);332 groupedPartials.get(g).push(index);333 });334 } else {335 independentIndices.push(index);336 }337 } else {338 independentIndices.push(index);339 }340 });341
342 // Check group signatures (mirrors extension's processScrapingResults)343 for (const [group, indices] of groupedPartials.entries()) {344 const groupResults = results.filter(r => indices.includes(r?.partialIndex));345 const allSuccess = groupResults.every(r => r?.success);346 if (allSuccess) {347 const groupData = groupResults.reduce((acc, r) => [...acc, ...(Array.isArray(r.data) ? r.data : [r.data])], []);348 const groupSig = generateSignature(groupData);349 if (processedSignatures.has(groupSig)) {350 groupResults.forEach(r => { r.alreadyProcessed = true; });351 if (D.partials) log.info('DEBUG[partials] group "' + group + '" signature already processed');352 } else {353 processedSignatures.add(groupSig);354 }355 }356 }357
358 // Check if at least one group is fully satisfied359 let groupSatisfied = groupedPartials.size === 0;360 for (const [group, indices] of groupedPartials.entries()) {361 const allSuccess = indices.every(i => results[i]?.success);362 if (allSuccess) {363 groupSatisfied = true;364 if (D.partials) log.info('DEBUG[partials] group "' + group + '" satisfied');365 break;366 }367 }368
369 // Check independent partials (at least one must succeed)370 const independentSuccess = independentIndices.length === 0 || independentIndices.some(i => results[i]?.success);371
372 if (!groupSatisfied && !independentSuccess) {373 if (D.partials) log.info('DEBUG[partials] no groups satisfied and no independent successes');374 return [];375 }376
377 // Filter out already-processed results, collect only new data (mirrors extension line 631)378 const newResults = results.filter(r => r?.success && r.data && !r.alreadyProcessed);379 if (D.partials) log.info('DEBUG[partials] ' + results.filter(r => r?.success).length + ' successful partials, ' + newResults.length + ' new (not already processed)');380
381 if (newResults.length === 0) {382 if (D.partials) log.info('DEBUG[partials] all partials already processed, skipping');383 return [];384 }385
386 const allRawData = [];387 for (const r of newResults) {388 const methodType = getPartialMethodType(partials[r.partialIndex]);389 allRawData.push({390 data: r.data,391 metadata: {392 method: methodType,393 location: r.partialIndex + '0A',394 timestamp: Date.now()395 }396 });397 }398 return allRawData;399}400
401// ── Process raw data in the browser using script.js ──────────────402
403async function processRawDataInBrowser(page, allRawData) {404 if (allRawData.length === 0) {405 if (D.processing) log.info('DEBUG[processing] No raw data to process');406 return [];407 }408
409 if (D.processing) log.info('DEBUG[processing] partials=' + allRawData.length);410
411 const combinedRawData = allRawData;412
413 const results = await page.evaluate(({ rawData, optionKey, debug }) => {414 const processFn = window[optionKey];415 if (!processFn) {416 if (debug) console.log('DEBUG[processing] processFn not found for: ' + optionKey);417 return [];418 }419 try {420 const input = Array.isArray(rawData) ? rawData : [rawData];421 if (debug) console.log('DEBUG[processing] calling ' + optionKey + '() with ' + input.length + ' items');422 const items = processFn(input);423 if (!Array.isArray(items)) return items ? [{ id: null, formatted: items }] : [];424 if (debug) console.log('DEBUG[processing] ' + optionKey + '() returned ' + items.length + ' items');425 return items.map(item => ({426 id: item.id || null,427 raw: item.raw || null,428 formatted: item.formatted || item429 }));430 } catch (err) {431 console.error('Processing error:', err.message, err.stack);432 return [];433 }434 }, { rawData: combinedRawData, optionKey, debug: D.processing });435
436 if (D.processing) log.info('DEBUG[processing] browser returned ' + results.length + ' items');437
438 return results;439}440
441// ── Scrape a page with given enhancements (used for initial + retry) ──442
443async function scrapePage(page, enh, capturedResponses, seenIds) {444 // Apply wait-until strategy445 const waitUntil = enh.waitUntil || 'networkidle';446 if (waitUntil !== 'none' && waitUntil !== 'commit') {447 await page.waitForLoadState(waitUntil).catch(() => {});448 if (D.navigation) log.info('DEBUG[navigation] ' + waitUntil + ' reached');449 }450
451 if (D.screenshot) {452 try {453 const buf = await page.screenshot({ fullPage: false });454 const store = await KeyValueStore.open();455 await store.setValue('debug-before-scrape-' + Date.now(), buf, { contentType: 'image/png' });456 log.info('DEBUG[screenshot] saved pre-scrape screenshot');457 } catch { }458 }459
460 if (D.requests) {461 log.info('DEBUG[requests] capturedResponses=' + capturedResponses.length);462 capturedResponses.forEach(function(r, i) { log.info('DEBUG[requests] [' + i + '] keys=' + Object.keys(r).slice(0, 5).join(',')); });463 }464
465 await injectIntoPage(page);466
467 // Polling loop — mirrors extension's scraping.attempts + scraping.delay468 const scrapingAttempts = option.scraping?.attempts ?? 1;469 const scrapingDelay = option.scraping?.delay || 0;470 const maxScrapingAttempts = Math.min(scrapingAttempts === 0 ? 5 : scrapingAttempts, 5);471
472 let items = [];473 for (let attempt = 1; attempt <= maxScrapingAttempts; attempt++) {474 const rawData = await processAllPartials(page, capturedResponses);475 items = await processRawDataInBrowser(page, rawData);476
477 if (items.length > 0) {478 if (attempt > 1 && D.partials) log.info('DEBUG[partials] scraping succeeded on attempt ' + attempt);479 break;480 }481
482 if (attempt < maxScrapingAttempts && scrapingDelay > 0) {483 if (D.partials) log.info('DEBUG[partials] attempt ' + attempt + ' yielded 0 items, waiting ' + scrapingDelay + 'ms');484 await page.waitForTimeout(scrapingDelay);485 await injectIntoPage(page); // Re-inject in case page changed486 }487 }488
489 // Deduplicate against seenIds490 const newItems = [];491 for (const item of items) {492 const id = item.id || JSON.stringify(item.formatted || item).slice(0, 100);493 if (D.processing) log.info('DEBUG[dedup] id=' + (item.id ? item.id : 'hash:' + id.slice(0, 60)) + ' seen=' + seenIds.has(id));494 if (seenIds.has(id)) continue;495 seenIds.add(id);496 newItems.push(item);497 }498 if (D.processing) log.info('DEBUG[dedup] ' + items.length + ' items -> ' + newItems.length + ' new (seenIds total=' + seenIds.size + ')');499 return newItems;500}501
502// ── Main ─────────────────────────────────────────────────────────503
504await Actor.init();505const input = await Actor.getInput();506
507if (!input?.urls?.length) {508 log.error('No URLs provided in input');509 await Actor.exit();510 process.exit(1);511}512
513const urls = input.urls514 .map(u => typeof u === 'string' ? u : u.url)515 .filter(u => {516 if (!validateUrl(u)) {517 log.warning('Skipping invalid URL: ' + u);518 return false;519 }520 return true;521 });522
523if (urls.length === 0) {524 log.error('No valid URLs after filtering');525 await Actor.exit();526 process.exit(1);527}528
529const maxItems = input.maxItems || 100;530const enablePagination = input.enablePagination !== undefined ? input.enablePagination : !!enhancements.pagination;531const maxPages = typeof input.maxPages === 'number' ? input.maxPages : (enhancements.maxPages || 1);532
533// ── Watchdog & abort tracking ────────────────────────────────534let lastRowTime = Date.now();535const WATCHDOG_TIMEOUT = 100000;536const watchdog = setInterval(() => {537 if (Date.now() - lastRowTime > WATCHDOG_TIMEOUT) {538 log.error('Watchdog: no new rows for ' + (WATCHDOG_TIMEOUT / 1000) + 's, aborting');539 process.exit(1);540 }541}, 15000);542
543const runState = { urlsProcessed: 0, totalResults: 0, startTime: Date.now(), aborted: false };544const navTimeout = typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout * 1000 : 30000;545const navRetries = typeof enhancements.retryNavigation === 'number' ? enhancements.retryNavigation : 0;546
547function shouldAbort() {548 if (!enhancements.abortOnLoss || runState.totalResults > 0) return false;549 const warmupUrls = typeof enhancements.abortWarmupUrls === 'number' ? enhancements.abortWarmupUrls : 5;550 const warmupSecs = typeof enhancements.abortWarmupSeconds === 'number' ? enhancements.abortWarmupSeconds : 60;551 const elapsedSecs = (Date.now() - runState.startTime) / 1000;552 return runState.urlsProcessed >= warmupUrls && elapsedSecs >= warmupSecs;553}554
555// ── Process a single URL on a given page ─────────────────────556
557async function processUrl(page, url, capturedResponses) {558 if (runState.aborted) { log.info('Skipping (aborted): ' + url); return; }559 if (shouldAbort()) {560 runState.aborted = true;561 log.warning('Aborting: no results after ' + runState.urlsProcessed + ' URLs and ' + Math.round((Date.now() - runState.startTime) / 1000) + 's');562 return;563 }564
565 log.info('Processing: ' + url);566 const _t0 = Date.now();567
568 // Reset per-URL state569 processedSignatures = new Set();570
571 await applyPostNavigation(page, enhancements);572
573 if (D.navigation) {574 const pageUrl = page.url();575 const title = await page.title();576 log.info('DEBUG[navigation] landed on url=' + pageUrl.slice(0, 150) + ' title=' + title);577 }578
579 // Custom script — afterLoad, afterDelay, concurrent timing580 let concurrentScriptPromise = null;581 if (enhancements.customScript && enhancements.customScriptCode && enhancements.customScriptTiming !== 'beforeLoad') {582 const timing = enhancements.customScriptTiming || 'afterLoad';583 const parsedUrl = new URL(url);584 const ctx = {585 url,586 hostname: parsedUrl.hostname,587 pathname: parsedUrl.pathname,588 searchParams: Object.fromEntries(parsedUrl.searchParams),589 optionKey,590 timing591 };592
593 if (timing === 'afterLoad') {594 await executeCustomScript(page, enhancements, ctx);595 } else if (timing === 'afterDelay') {596 const delay = typeof enhancements.customScriptDelay === 'number' ? enhancements.customScriptDelay : 2000;597 await page.waitForTimeout(delay);598 await executeCustomScript(page, enhancements, ctx);599 } else if (timing === 'concurrent') {600 concurrentScriptPromise = executeCustomScript(page, enhancements, ctx);601 }602 }603
604 const seenIds = new Set();605 let totalExtracted = 0;606
607 const initialResults = await scrapePage(page, enhancements, capturedResponses, seenIds);608
609 // Retry If Empty610 let results = initialResults;611 if (results.length === 0 && enhancements.retryIfEmpty) {612 const maxRetries = typeof enhancements.retryMaxAttempts === 'number' ? enhancements.retryMaxAttempts : 3;613 log.info('Retry: 0 results, attempting up to ' + maxRetries + ' retries');614
615 for (let attempt = 0; attempt < maxRetries; attempt++) {616 const retryEnh = getRetryEnhancements(attempt);617 log.info('Retry: attempt ' + (attempt + 1) + '/' + maxRetries + (retryEnh.newContext ? ' (new context)' : ''));618
619 let retryPage = page;620 let retryContext = null;621 let retryCaptured = capturedResponses;622
623 if (retryEnh.newContext) {624 // Fresh browser context — new cookies, storage, fingerprint625 retryContext = await page.context().browser().newContext();626 retryPage = await retryContext.newPage();627 retryCaptured = [];628 await applyResourceBlocking(retryPage, retryEnh);629 await applyPreNavigation(retryPage, retryEnh);630 if (needsRequestInterception) {631 requestHandler.setupInterception(retryPage, option, retryCaptured);632 }633 }634
635 await retryPage.goto(url, { timeout: navTimeout, waitUntil: 'commit' });636 await applyPostNavigation(retryPage, retryEnh);637
638 const retryWait = retryEnh.waitUntil || 'networkidle';639 if (retryWait !== 'none' && retryWait !== 'commit') {640 await retryPage.waitForLoadState(retryWait).catch(() => {});641 }642
643 results = await scrapePage(retryPage, retryEnh, retryCaptured, seenIds);644
645 // Clean up new context if we created one646 if (retryContext) {647 await retryContext.close().catch(() => {});648 }649
650 if (results.length > 0) {651 log.info('Retry: attempt ' + (attempt + 1) + ' succeeded with ' + results.length + ' items');652 break;653 }654 }655 }656
657 const initialFormatted = results.map(item => formatForDataset(item, enhancements));658 for (const item of initialFormatted) {659 if (totalExtracted >= maxItems) break;660 await Dataset.pushData(item); lastRowTime = Date.now();661 totalExtracted++;662 }663
664 if (option.paginate && enablePagination) {665 await runPagination(666 page, option, { ...enhancements, pagination: true, maxPages },667 async (pg) => {668 const newItems = await scrapePage(pg, enhancements, capturedResponses, seenIds);669 const formatted = newItems.map(item => formatForDataset(item, enhancements));670 for (const item of formatted) {671 if (totalExtracted >= maxItems) break;672 await Dataset.pushData(item); lastRowTime = Date.now();673 totalExtracted++;674 }675 return formatted;676 }677 );678 }679
680 runState.urlsProcessed++;681 runState.totalResults += totalExtracted;682
683 if (D.performance) log.info('DEBUG[performance] total time=' + ((Date.now() - _t0) / 1000).toFixed(1) + 's items=' + totalExtracted);684 log.info('Extracted ' + totalExtracted + ' items from ' + url);685}686
687// ── Run ──────────────────────────────────────────────────────688
689const crawlerOptions = await getCrawlerOptions(enhancements);690
691if (enhancements.reuseContext) {692 // Reuse context mode: split URLs across N tabs, each tab navigates its URLs sequentially via goto693 const tabCount = Math.min(typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1, urls.length);694
695 // Distribute URLs round-robin across tabs696 const tabUrls = Array.from({ length: tabCount }, () => []);697 urls.forEach((url, i) => tabUrls[i % tabCount].push(url));698
699 // Each tab gets its first URL as the crawler request, remaining URLs navigated via goto700 const crawler = new PlaywrightCrawler({701 headless: true,702 maxRequestsPerCrawl: tabCount,703 ...crawlerOptions,704 maxConcurrency: tabCount,705 minConcurrency: tabCount,706 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,707
708 preNavigationHooks: [709 async ({ page, request }) => {710 await applyResourceBlocking(page, enhancements);711 await applyPreNavigation(page, enhancements);712 // Custom script — beforeLoad timing (Node context, can modify URL)713 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {714 request.url = executeBeforeLoadScript(enhancements, request.url);715 }716 // Create per-page capturedResponses, stored on the page object717 if (needsRequestInterception) {718 page._capturedResponses = [];719 requestHandler.setupInterception(page, option, page._capturedResponses);720 }721 }722 ],723
724 async requestHandler({ page, request }) {725 const capturedResponses = page._capturedResponses || [];726 // Find which tab's URL list this request belongs to727 const tabIndex = tabUrls.findIndex(t => t[0] === request.url);728 const myUrls = tabIndex >= 0 ? tabUrls[tabIndex] : [request.url];729
730 // Process first URL (already navigated by crawler)731 await processUrl(page, request.url, capturedResponses);732
733 // Navigate remaining URLs on the same page734 for (let i = 1; i < myUrls.length; i++) {735 if (runState.aborted) break;736 try {737 const gotoWait = enhancements.waitUntil || 'networkidle';738 await page.goto(myUrls[i], { timeout: navTimeout, waitUntil: gotoWait === 'none' ? 'commit' : gotoWait });739 await processUrl(page, myUrls[i], capturedResponses);740 } catch (err) {741 log.error('Navigation failed for ' + myUrls[i] + ': ' + err.message);742 runState.urlsProcessed++;743 }744 }745 },746
747 maxRequestRetries: navRetries,748 async failedRequestHandler({ request }) {749 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);750 runState.urlsProcessed++;751 }752 });753
754 await crawler.run(tabUrls.map(t => ({ url: t[0] })));755} else {756 // Normal mode: one request per URL, crawler manages contexts757 const crawler = new PlaywrightCrawler({758 headless: true,759 maxRequestsPerCrawl: urls.length,760 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,761 ...crawlerOptions,762
763 preNavigationHooks: [764 async ({ page, request }) => {765 await applyResourceBlocking(page, enhancements);766 await applyPreNavigation(page, enhancements);767 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {768 request.url = executeBeforeLoadScript(enhancements, request.url);769 }770 if (needsRequestInterception) {771 page._capturedResponses = [];772 requestHandler.setupInterception(page, option, page._capturedResponses);773 }774 }775 ],776
777 async requestHandler({ page, request }) {778 const capturedResponses = page._capturedResponses || [];779 await processUrl(page, request.url, capturedResponses);780 },781
782 maxRequestRetries: navRetries,783 async failedRequestHandler({ request }) {784 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);785 runState.urlsProcessed++;786 }787 });788
789 await crawler.run(urls.map(url => ({ url })));790}791
792clearInterval(watchdog);793await Actor.exit();1window.searchField = function(obj, fieldName, parentDepth = 0, results = [], parents = []) {2 if (typeof obj !== "object" || obj === null) {3 return results;4 }5
6 for (const key in obj) {7 if (key === fieldName) {8 let ancestorIndex = parents.length - parentDepth;9 if (ancestorIndex >= 0) {10 results.push(parents[ancestorIndex] || obj);11 } else {12 results.push(obj);13 }14 }15 if (typeof obj[key] === "object" && obj[key] !== null) {16 searchField(obj[key], fieldName, parentDepth, results, [...parents, obj]);17 }18 }19 return results;20}21
22window.searchFieldMap = function(data, fieldName, parentDepth = 0) {23 const results = [];24
25 if (!(data instanceof Map)) {26 return searchField(data, fieldName, parentDepth, results);27 }28
29 for (const [, value] of data) {30 if (typeof value === "object" && value !== null) {31 if (fieldName in value) {32 results.push(value);33 }34 const nested = searchField(value, fieldName, parentDepth, []);35 results.push(...nested);36 }37 }38
39 return results;40}41
42window.getValue = function(value, fallback = "Not Available") {43 if (value === null || value === undefined || value === "") return fallback;44 return String(value);45}46
47window.formatBoolean = function(value) {48 if (typeof value === "boolean") return value ? "Yes" : "No";49 return "Not Available";50}51
52window.joinList = function(items, separator = " || ") {53 if (!Array.isArray(items)) return "Not Available";54 const filtered = items.filter(Boolean);55 return filtered.length > 0 ? filtered.join(separator) : "Not Available";56}57
58window.moduleExport = function(exports) {59 if (typeof module !== "undefined" && module.exports) {60 module.exports = exports;61 } else {62 Object.assign(window, exports);63 }64}65
66window.userCache = new Map();67
68window.getCachedUser = function(user) {69 if (!user) return null;70
71 const userId = user.pk?.toString() || user.id?.toString();72 if (!userId) return user;73
74 if (user.username) {75 userCache.set(userId, user);76 return user;77 }78
79 if (userCache.has(userId)) {80 return { ...user, ...userCache.get(userId) };81 }82
83 return user;84}85
86window.resolveRefs = function(obj, map, visited = new Set()) {87 if (typeof obj !== "object" || obj === null) {88 return obj;89 }90
91 if (!(map instanceof Map)) {92 return obj;93 }94
95 if (obj.__ref) {96 const refKey = obj.__ref;97 if (visited.has(refKey)) return obj;98 visited.add(refKey);99
100 if (map.has(refKey)) {101 const resolved = map.get(refKey);102 return resolveRefs(resolved, map, visited);103 }104 return obj;105 }106
107 if (obj.__refs && Array.isArray(obj.__refs)) {108 return obj.__refs.map(refKey => {109 if (visited.has(refKey)) return { __ref: refKey };110 visited.add(refKey);111
112 if (map.has(refKey)) {113 const resolved = map.get(refKey);114 return resolveRefs(resolved, map, visited);115 }116 return { __ref: refKey };117 });118 }119
120 if (obj.__id) {121 const refKey = obj.__id;122 if (!visited.has(refKey) && map.has(refKey)) {123 visited.add(refKey);124 const lookedUp = map.get(refKey);125 const merged = { ...obj, ...lookedUp };126 const resolved = {};127 for (const key in merged) {128 resolved[key] = resolveRefs(merged[key], map, visited);129 }130 return resolved;131 }132 }133
134 if (Array.isArray(obj)) {135 return obj.map(item => resolveRefs(item, map, visited));136 }137
138 const resolved = {};139 for (const key in obj) {140 resolved[key] = resolveRefs(obj[key], map, visited);141 }142 return resolved;143}144
145window.getPostType = function(mediaType) {146 switch (mediaType) {147 case 1: return "Photo";148 case 2: return "Video";149 case 8: return "Carousel";150 default: return null;151 }152}153
154window.getSavedPostType = function(item) {155 if (item.isSidecar) return "Carousel";156 if (item.isVideo) return "Video";157 return "Photo";158}159
160window.getImageUrl = function(item) {161 const candidates = item.image_versions2?.candidates;162 if (Array.isArray(candidates) && candidates[0]?.url) {163 return candidates[0].url;164 }165
166 const resolverData = item["$r:client__srcSet"]?.__resolverSnapshot?.data;167 if (resolverData?.image_versions2?.candidates?.[0]?.url) {168 return resolverData.image_versions2.candidates[0].url;169 }170
171 return null;172}173
174window.cacheUsersFromData = function(data) {175 let users = [];176 if (data instanceof Map) {177 users = searchFieldMap(data, "full_name");178 users = users.map(item => resolveRefs(item, data));179 } else {180 users = searchField(data, "full_name");181 }182 for (const user of users) {183 if (user && user.username) {184 getCachedUser(user);185 }186 }187}188
189// ── POSTS ──190
191window.preparePostsFromRequest = function(data) {192 const results = [];193 const seen = new Set();194
195 for (const item of data) {196 let posts = searchField(item, "media_notes");197 if (posts.length === 0) {198 posts = searchField(item, "clips_tab_pinned_user_ids");199 }200
201 for (const post of posts) {202 const postId = post.id?.toString() || post.pk?.toString();203 if (!postId || seen.has(postId)) continue;204
205 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {206 if (post.is_paid_partnership !== false) continue;207 }208
209 seen.add(postId);210 const user = getCachedUser(post.user);211 const isReel = post.product_type === "clips" || post.media_type === 2;212
213 results.push({214 id: postId,215 code: post.code,216 user,217 type: getPostType(post.media_type),218 text: post.caption?.text,219 image: getImageUrl(post),220 video: post.video_versions?.[0]?.url,221 likes: post.like_count?.toString(),222 commentsCount: post.comment_count?.toString(),223 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,224 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,225 commentsDisabled: post.comments_disabled,226 accessibilityCaption: post.accessibility_caption,227 raw: post228 });229 }230 }231
232 return results;233}234
235window.preparePostsFromDom = function(data) {236 const results = [];237 const seen = new Set();238
239 for (const kv of data) {240 const parsed = typeof kv === "string" ? JSON.parse(kv) : kv;241 let posts = searchField(parsed, "media_notes");242 if (posts.length === 0) {243 posts = searchField(parsed, "clips_tab_pinned_user_ids");244 }245
246 for (const post of posts) {247 const postId = post.id?.toString() || post.pk?.toString();248 if (!postId || seen.has(postId)) continue;249
250 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {251 if (post.is_paid_partnership !== false) continue;252 }253
254 seen.add(postId);255 const user = getCachedUser(post.user);256 const isReel = post.product_type === "clips" || post.media_type === 2;257
258 results.push({259 id: postId,260 code: post.code,261 user,262 type: getPostType(post.media_type),263 text: post.caption?.text,264 image: getImageUrl(post),265 video: post.video_versions?.[0]?.url,266 likes: post.like_count?.toString(),267 commentsCount: post.comment_count?.toString(),268 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,269 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,270 commentsDisabled: post.comments_disabled,271 accessibilityCaption: post.accessibility_caption,272 raw: post273 });274 }275 }276
277 return results;278}279
280window.preparePostsFromReact = function(data) {281 const results = [];282 const seen = new Set();283
284 for (const item of data) {285 if (item && "isFetching" in item && Array.isArray(item.posts)) {286 for (const post of item.posts) {287 const postId = post.id?.toString();288 if (!postId || seen.has(postId)) continue;289 if (post.isSponsored) continue;290
291 seen.add(postId);292 const owner = post.owner;293 const isReel = post.productType === "clips" || post.isVideo;294
295 results.push({296 id: postId,297 code: post.code,298 user: owner ? {299 username: owner.username,300 full_name: owner.fullName,301 profile_pic_url: owner.profilePictureUrl,302 is_verified: owner.isVerified303 } : null,304 type: getSavedPostType(post),305 text: post.caption,306 image: post.src || post.displayResources?.[0]?.src,307 video: post.videoResources?.[0]?.src,308 likes: post.numLikes?.toString(),309 commentsCount: null,310 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,311 date: post.postedAt ? new Date(post.postedAt * 1000).toISOString() : null,312 commentsDisabled: post.commentsDisabled,313 accessibilityCaption: post.accessibilityCaption,314 raw: post315 });316 }317 continue;318 }319
320 if (item instanceof Map) {321 cacheUsersFromData(item);322 let posts = searchFieldMap(item, "sponsor_tags");323 if (posts.length === 0) {324 posts = searchFieldMap(item, "clips_tab_pinned_user_ids");325 }326 posts = posts.map(p => resolveRefs(p, item));327
328 for (const post of posts) {329 const postId = post.id?.toString() || post.pk?.toString();330 if (!postId || seen.has(postId)) continue;331
332 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {333 if (post.is_paid_partnership !== false) continue;334 }335
336 seen.add(postId);337 const user = getCachedUser(post.user);338 const isReel = post.product_type === "clips" || post.media_type === 2;339
340 results.push({341 id: postId,342 code: post.code,343 user,344 type: getPostType(post.media_type),345 text: post.caption?.text,346 image: getImageUrl(post),347 video: post.video_versions?.[0]?.url,348 likes: post.like_count?.toString(),349 commentsCount: post.comment_count?.toString(),350 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,351 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,352 commentsDisabled: post.comments_disabled,353 accessibilityCaption: post.accessibility_caption,354 raw: post355 });356 }357 continue;358 }359
360 const edges = item?.data?.xdt_api__v1__feed__user_timeline_graphql_connection?.edges;361 if (edges) {362 for (const edge of edges) {363 const post = edge.node;364 const postId = post.id?.toString() || post.pk?.toString();365 if (!postId || seen.has(postId)) continue;366
367 seen.add(postId);368 const isReel = post.product_type === "clips" || post.media_type === 2;369
370 results.push({371 id: postId,372 code: post.code,373 user: post.user,374 type: getPostType(post.media_type),375 text: post.caption?.text,376 image: post.image_versions2?.candidates?.[0]?.url,377 video: post.video_versions?.[0]?.url,378 likes: post.like_count?.toString(),379 commentsCount: post.comment_count?.toString(),380 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,381 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,382 commentsDisabled: post.comments_disabled,383 accessibilityCaption: post.accessibility_caption,384 raw: edge385 });386 }387 continue;388 }389
390 cacheUsersFromData(item);391 let posts = searchField(item, "media_notes");392 if (posts.length === 0) {393 posts = searchField(item, "clips_tab_pinned_user_ids");394 }395
396 for (const post of posts) {397 const postId = post.id?.toString() || post.pk?.toString();398 if (!postId || seen.has(postId)) continue;399
400 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {401 if (post.is_paid_partnership !== false) continue;402 }403
404 seen.add(postId);405 const user = getCachedUser(post.user);406 const isReel = post.product_type === "clips" || post.media_type === 2;407
408 results.push({409 id: postId,410 code: post.code,411 user,412 type: getPostType(post.media_type),413 text: post.caption?.text,414 image: getImageUrl(post),415 video: post.video_versions?.[0]?.url,416 likes: post.like_count?.toString(),417 commentsCount: post.comment_count?.toString(),418 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,419 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,420 commentsDisabled: post.comments_disabled,421 accessibilityCaption: post.accessibility_caption,422 raw: post423 });424 }425 }426
427 return results;428}429
430window.formatPosts = function(prepared) {431 return prepared.map(item => ({432 id: item.id,433 id2: item.code,434 highlight: { id2: item.code },435 raw: item.raw,436 formatted: {437 "id": item.id,438 "Post Author": item.user?.username || "Not Available",439 "Post Author Full Name": item.user?.full_name || item.user?.fullName || "Not Available",440 "Post Author Image": item.user?.profile_pic_url || item.user?.profilePictureUrl || "Not Available",441 "Post Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",442 "Post Author Is Verified": formatBoolean(item.user?.is_verified ?? item.user?.isVerified),443 "Post Type": item.type || "Not Available",444 "Post Text": item.text || "Not Available",445 "Post Image": item.image || "Not Available",446 "Post Video": item.video || "Not Available",447 "Post Likes": item.likes || "Not Available",448 "Post Comments Count": item.commentsCount || "Not Available",449 "Post URL": item.url || "Not Available",450 "Post Date": item.date || "Not Available",451 "Is Comments Disabled": formatBoolean(item.commentsDisabled),452 "Post Accessibility Caption": item.accessibilityCaption || "Not Available"453 }454 }));455}456
457window.posts = function(rawData) {458 try {459 let prepared = [];460 const seen = new Set();461
462 for (const entry of rawData) {463 let batch = [];464 if (entry.metadata.method === "request") {465 batch = preparePostsFromRequest(entry.data);466 } else if (entry.metadata.method === "dom") {467 batch = preparePostsFromDom(entry.data);468 } else if (entry.metadata.method === "react") {469 batch = preparePostsFromReact(entry.data);470 }471 for (const item of batch) {472 if (!seen.has(item.id)) {473 seen.add(item.id);474 prepared.push(item);475 }476 }477 }478
479 return formatPosts(prepared);480 } catch (error) {481 return [];482 }483}484
485// ── STORIES ──486
487window.prepareStories = function(rawData) {488 const results = [];489 const seen = new Set();490 const userStore = new Map();491
492 for (const entry of rawData) {493 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];494 for (const data of dataItems) {495 const stories = searchField(data, "story_cta");496 searchField(data, "profile_pic_url").forEach(user => {497 userStore.set(user.id, user);498 });499
500 for (const item of stories) {501 const storyId = item.pk?.toString() || item.id?.toString();502 if (!storyId || seen.has(storyId)) continue;503 seen.add(storyId);504
505 let user = item.user;506 if (user && !user.profile_pic_url && userStore.get(user.id)) {507 user = userStore.get(user.id);508 }509
510 results.push({511 id: storyId,512 type: getPostType(item.media_type),513 image: item.image_versions2?.candidates?.[0]?.url,514 video: item.video_versions?.[0]?.url,515 link: item.pk && user?.username ? `https://www.instagram.com/stories/${user.username}/${item.pk}/` : null,516 date: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,517 expiringAt: item.expiring_at ? new Date(item.expiring_at * 1000).toISOString() : null,518 accessibilityCaption: item.accessibility_caption,519 user,520 bloksFullName: item.story_bloks_stickers?.[0]?.bloks_sticker?.sticker_data?.ig_mention?.full_name,521 raw: item522 });523 }524 }525 }526
527 return results;528}529
530window.formatStories = function(prepared) {531 return prepared.map(item => ({532 id: item.id,533 highlight: { id: item.id },534 raw: item.raw,535 formatted: {536 "id": item.id,537 "Story Type": item.type || "Not Available",538 "Story Image": item.image || "Not Available",539 "Story Video": item.video || "Not Available",540 "Story Link": item.link || "Not Available",541 "Story Date": item.date || "Not Available",542 "Story Expiring At": item.expiringAt || "Not Available",543 "Story Accessibility Caption": item.accessibilityCaption || "Not Available",544 "Author Username": item.user?.username || "Not Available",545 "Author Full Name": item.user?.full_name || item.bloksFullName || "Not Available",546 "Author Image": item.user?.profile_pic_url || "Not Available",547 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",548 "Author Is Verified": formatBoolean(item.user?.is_verified)549 }550 }));551}552
553window.stories = function(rawData) {554 try {555 const prepared = prepareStories(rawData);556 return formatStories(prepared);557 } catch (error) {558 return [];559 }560}561
562// ── POST DETAILS ──563
564window.preparePostDetails = function(rawData) {565 const results = [];566 const seen = new Set();567
568 for (const entry of rawData) {569 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];570
571 for (const data of dataItems) {572 let items = [];573
574 if (data instanceof Map) {575 items = searchFieldMap(data, "coauthor_producers");576 items = items.map(item => resolveRefs(item, data));577 } else {578 items = searchField(data, "coauthor_producers");579 }580
581 for (const item of items) {582 const postId = item.pk?.toString() || item.id?.toString();583 if (!location.href.includes(item.code) && !location.href.includes("/reels/")) continue;584 if (!postId || seen.has(postId)) continue;585 seen.add(postId);586
587 const user = item.user || item.owner;588 const caption = item.caption;589 const captionText = typeof caption === "string" ? caption : caption?.text;590
591 const isReel = item.product_type === "clips" || item.media_type === 2;592 const isCarousel = item.media_type === 8 || item.carousel_media_count > 0;593
594 let imageUrl = item.image_versions2?.candidates?.[0]?.url595 || item.carousel_media?.[0]?.image_versions2?.candidates?.[0]?.url;596
597 let videoUrl = item.video_versions?.[0]?.url598 || item.carousel_media?.[0]?.video_versions?.[0]?.url;599
600 let taggedUsers = null;601 if (item.usertags?.in?.length > 0) {602 taggedUsers = item.usertags.in.map(tag => tag.user?.username).filter(Boolean).join(", ");603 }604
605 results.push({606 id: postId,607 code: item.code,608 type: getPostType(item.media_type),609 caption: captionText,610 likeCount: item.like_count?.toString(),611 commentCount: item.comment_count?.toString(),612 viewCount: item.view_count?.toString() || item.play_count?.toString(),613 user,614 url: item.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${item.code}/` : null,615 imageUrl,616 videoUrl,617 coverImage: item.image_versions2?.candidates?.[0]?.url,618 postedAt: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,619 location: item.location?.name,620 duration: item.video_duration?.toString(),621 hasAudio: item.has_audio,622 audioTitle: item.clips_metadata?.original_sound_info?.original_audio_title,623 audioArtist: item.clips_metadata?.original_sound_info?.ig_artist?.username,624 taggedUsers,625 isCarousel,626 carouselCount: item.carousel_media_count?.toString() || item.carousel_media?.length?.toString(),627 originalWidth: item.original_width?.toString(),628 originalHeight: item.original_height?.toString(),629 isPaidPartnership: item.is_paid_partnership,630 commentsDisabled: item.comments_disabled,631 likesDisabled: item.like_and_view_counts_disabled,632 accessibilityCaption: item.accessibility_caption,633 raw: item634 });635 }636 }637 }638
639 return results;640}641
642window.formatPostDetails = function(prepared) {643 return prepared.map(item => ({644 id: item.id,645 id2: item.code,646 highlight: { id: item.id },647 raw: item.raw,648 formatted: {649 "id": item.id,650 "Post Type": item.type || "Not Available",651 "Caption": item.caption || "Not Available",652 "Like Count": item.likeCount || "Not Available",653 "Comment Count": item.commentCount || "Not Available",654 "View Count": item.viewCount || "Not Available",655 "Author Username": item.user?.username || "Not Available",656 "Author Full Name": item.user?.full_name || "Not Available",657 "Author ID": item.user?.pk?.toString() || item.user?.id?.toString() || "Not Available",658 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",659 "Author Is Verified": formatBoolean(item.user?.is_verified),660 "Author Profile Pic": item.user?.profile_pic_url || item.user?.hd_profile_pic_url_info?.url || "Not Available",661 "Post URL": item.url || "Not Available",662 "Image URL": item.imageUrl || "Not Available",663 "Video URL": item.videoUrl || "Not Available",664 "Cover Image": item.coverImage || "Not Available",665 "Posted At": item.postedAt || "Not Available",666 "Location": item.location || "Not Available",667 "Duration": item.duration || "Not Available",668 "Has Audio": formatBoolean(item.hasAudio),669 "Audio Title": item.audioTitle || "Not Available",670 "Audio Artist": item.audioArtist || "Not Available",671 "Tagged Users": item.taggedUsers || "Not Available",672 "Is Carousel": formatBoolean(item.isCarousel),673 "Carousel Count": item.carouselCount || "Not Available",674 "Original Width": item.originalWidth || "Not Available",675 "Original Height": item.originalHeight || "Not Available",676 "Is Paid Partnership": formatBoolean(item.isPaidPartnership),677 "Comments Disabled": formatBoolean(item.commentsDisabled),678 "Likes Disabled": formatBoolean(item.likesDisabled),679 "Accessibility Caption": item.accessibilityCaption || "Not Available"680 }681 }));682}683
684window.postDetails = function(rawData) {685 try {686 const prepared = preparePostDetails(rawData);687 return formatPostDetails(prepared);688 } catch (error) {689 return [];690 }691}692
693// ── COMMENTS ──694
695window.prepareComments = function(rawData) {696 const results = [];697 const seen = new Set();698
699 for (const entry of rawData) {700 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];701 for (const data of dataItems) {702 const comments = searchField(data, "comment_like_count");703
704 for (const item of comments) {705 const commentId = item.pk?.toString() || item.id?.toString();706 if (!commentId || seen.has(commentId)) continue;707 seen.add(commentId);708
709 const commentDate = item.created_at ? new Date(item.created_at * 1000).toISOString() : null;710
711 results.push({712 id: commentId,713 date: commentDate,714 text: item.text,715 media: item.giphy_media_info?.first_party_cdn_proxied_images?.fixed_height?.url,716 likes: item.comment_like_count?.toString(),717 user: item.user,718 isReply: !!item.parent_comment_id,719 parentCommentId: item.parent_comment_id?.toString(),720 childCommentCount: item.child_comment_count,721 raw: item722 });723 }724 }725 }726
727 return results;728}729
730window.formatComments = function(prepared) {731 return prepared.map(item => ({732 id: item.id,733 id2: item.date,734 highlight: { id2: item.date },735 raw: item.raw,736 formatted: {737 "id": item.id,738 "Comment Text": item.text || "Not Available",739 "Comment Media": item.media || "Not Available",740 "Comment Date": item.date || "Not Available",741 "Comment Likes": item.likes || "Not Available",742 "Author Username": item.user?.username || "Not Available",743 "Author ID": item.user?.pk || item.user?.id || "Not Available",744 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",745 "Author Is Verified": formatBoolean(item.user?.is_verified),746 "Is Reply": formatBoolean(item.isReply),747 "Parent Comment ID": item.parentCommentId || "Not Available",748 "Has Replies": formatBoolean(item.childCommentCount > 0),749 "Reply Count": item.childCommentCount?.toString() || "Not Available"750 }751 }));752}753
754window.comments = function(rawData) {755 try {756 const prepared = prepareComments(rawData);757 return formatComments(prepared);758 } catch (error) {759 return [];760 }761}762
763// ── PROFILE ──764
765window.prepareProfile = function(rawData) {766 const results = [];767 const seen = new Set();768
769 for (const entry of rawData) {770 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];771
772 for (const data of dataItems) {773 let users = [];774
775 if (data instanceof Map) {776 users = searchFieldMap(data, "biography");777 users = users.map(user => resolveRefs(user, data));778 } else {779 users = searchField(data, "biography");780 }781
782 for (const item of users) {783 const userId = item.pk?.toString() || item.id?.toString();784 if (!userId || seen.has(userId)) continue;785 seen.add(userId);786
787 results.push({788 id: userId,789 username: item.username,790 fullName: item.full_name,791 biography: item.biography,792 externalUrl: item.external_url,793 followerCount: item.follower_count?.toString(),794 followingCount: item.following_count?.toString(),795 mediaCount: item.media_count?.toString(),796 isVerified: item.is_verified,797 isPrivate: item.is_private,798 isBusiness: item.is_business,799 category: item.category,800 profilePicUrl: item.hd_profile_pic_url_info?.url || item.profile_pic_url,801 bioLinks: item.bio_links?.map(link => link.url),802 raw: item803 });804 }805 }806 }807
808 return results;809}810
811window.formatProfile = function(prepared) {812 return prepared.map(item => ({813 id: item.id,814 id2: item.username,815 highlight: { id: item.id },816 raw: item.raw,817 formatted: {818 "id": item.id,819 "Username": item.username || "Not Available",820 "Full Name": item.fullName || "Not Available",821 "Biography": item.biography || "Not Available",822 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",823 "External URL": item.externalUrl || "Not Available",824 "Follower Count": item.followerCount || "Not Available",825 "Following Count": item.followingCount || "Not Available",826 "Post Count": item.mediaCount || "Not Available",827 "Is Verified": formatBoolean(item.isVerified),828 "Is Private": formatBoolean(item.isPrivate),829 "Is Business": formatBoolean(item.isBusiness),830 "Category": item.category || "Not Available",831 "Profile Picture URL": item.profilePicUrl || "Not Available",832 "All Bio Links": joinList(item.bioLinks)833 }834 }));835}836
837window.profile = function(rawData) {838 try {839 const prepared = prepareProfile(rawData);840 return formatProfile(prepared);841 } catch (error) {842 return [];843 }844}845
846// ── FOLLOWERS / FOLLOWING ──847
848window.prepareUsers = function(rawData) {849 const results = [];850 const seen = new Set();851
852 for (const entry of rawData) {853 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];854 for (const data of dataItems) {855 const users = searchField(data, "profile_pic_url");856
857 for (const item of users) {858 const userId = item.pk?.toString() || item.id?.toString();859 if (!userId || seen.has(userId)) continue;860 seen.add(userId);861
862 results.push({863 id: userId,864 username: item.username,865 fullName: item.full_name,866 isPrivate: item.is_private,867 isVerified: item.is_verified,868 profilePicUrl: item.profile_pic_url,869 hasAnonPic: item.has_anonymous_profile_picture,870 latestStoryTime: item.latest_reel_media ? new Date(item.latest_reel_media * 1000).toISOString() : null,871 raw: item872 });873 }874 }875 }876
877 return results;878}879
880window.formatUsers = function(prepared) {881 return prepared.map(item => ({882 id: item.id,883 id2: item.username,884 highlight: { id2: item.username },885 raw: item.raw,886 formatted: {887 "id": item.id,888 "Username": item.username || "Not Available",889 "Full Name": item.fullName || "Not Available",890 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",891 "Is Private": formatBoolean(item.isPrivate),892 "Is Verified": formatBoolean(item.isVerified),893 "Profile Picture URL": item.profilePicUrl || "Not Available",894 "Has Anonymous Profile Picture": formatBoolean(item.hasAnonPic),895 "Latest Story Time": item.latestStoryTime || "Not Available"896 }897 }));898}899
900window.followers = function(rawData) {901 try {902 const prepared = prepareUsers(rawData);903 return formatUsers(prepared);904 } catch (error) {905 return [];906 }907}908
909window.following = function(rawData) {910 try {911 const prepared = prepareUsers(rawData);912 return formatUsers(prepared);913 } catch (error) {914 return [];915 }916}{ "title": "Instagram Comments of a Post Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "urls": { "title": "URLs", "type": "array", "items": { "type": "string" }, "description": "URLs to scrape. Must match: instagram.com", "editor": "stringList", "default": [ "https://www.instagram.com/p/DWZitMjgHp1/" ] }, "maxItems": { "title": "Max items", "type": "integer", "description": "Maximum number of items to extract per URL", "default": 100, "minimum": 1 }, "enablePagination": { "title": "Paginate", "type": "boolean", "description": "Load more results beyond the first page.", "default": true }, "maxPages": { "title": "Max pagination steps", "type": "integer", "description": "Maximum number of pagination actions (pages, scrolls, or loads)", "default": 1, "minimum": 1 } }, "required": [ "urls" ]}{ "actorSpecification": 1, "fields": { "Comment Text": { "type": "string", "title": "Comment Text" }, "Comment Media": { "type": "string", "title": "Comment Media" }, "Comment Date": { "type": "string", "title": "Comment Date" }, "Comment Likes": { "type": "string", "title": "Comment Likes" }, "Author Username": { "type": "string", "title": "Author Username" }, "Author ID": { "type": "string", "title": "Author ID" }, "Author URL": { "type": "string", "title": "Author URL" }, "Author Is Verified": { "type": "string", "title": "Author Is Verified" }, "Is Reply": { "type": "string", "title": "Is Reply" }, "Parent Comment ID": { "type": "string", "title": "Parent Comment ID" }, "Has Replies": { "type": "string", "title": "Has Replies" }, "Reply Count": { "type": "string", "title": "Reply Count" } }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table", "properties": {} } } }}