Instagram Post Comments Scraper
Pricing
from $2.00 / 1,000 results
Extract Instagram comment text, dates, likes, and author details from post pages. Powered by Instagram Scraper.
Pricing
from $2.00 / 1,000 results
Rating
0.0
(0)
Developer
Nifty
Maintained by CommunityActor stats
0
Bookmarked
29
Total users
6
Monthly active users
10 days ago
Last modified
Categories
Share
{ "actorSpecification": 1, "name": "instagram-comments-scraper", "title": "Instagram Comments of a Post Scraper", "version": "3.2", "dockerfile": "./Dockerfile", "input": "./INPUT_SCHEMA.json", "storages": { "dataset": { "actorSpecification": 1, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table" } } } } }}FROM apify/actor-node-playwright-chrome:20
COPY package.json ./RUN npm install --omit=dev --omit=optional
COPY . ./
CMD npm start{ "name": "instagram-comments-scraper", "version": "3.2", "type": "module", "main": "src/main.js", "scripts": { "start": "node src/main.js" }, "dependencies": { "crawlee": "^3.12.0", "playwright": "^1.49.0", "apify": "^3.2.0", "user-agents": "^1.1.0", "playwright-extra": "^4.3.0", "puppeteer-extra-plugin-stealth": "^2.11.0" }}{ "app": { "name": "Instagram Scraper", "targetName": "Instagram", "targetURL": "https://instagram.com", "id": "instagram-scraper", "title": "I-Social Scraper: Export Posts, Profiles, Reels & More", "description": "Instagram Scraper: Extract Instagram comments, stories, followers & more. Export to CSV, JSON, XLSX with ease.", "version": "3.2.2", "backend": "https://nifty.codes", "hostnames": [ "instagram.com" ], "matches": [ "*://*.instagram.com/*" ], "status": "Published", "colors": { "background": "#f80261", "elements": "#ffffff" }, "longDescription": "This extension provides a comprehensive toolkit for extracting publicly available data from Instagram profiles, posts, reels, stories, and comments directly within your browser. All data collection and processing happens locally on your device. Built-in monitoring tools let you track page changes and receive alerts over time.\n\nKey Features\n✓ Monitor pages for changes and receive alerts via webhook or ntfy.sh\n✓ All scraping and exports processed locally in your browser\n✓ Export to CSV, Excel (XLSX), JSON, HTML Table, and Raw JSON\n✓ Automatic pagination for feeds, follower lists, and comment threads\n✓ Batch processing for multiple URLs in a single session\n✓ Customizable field selection and conditional formatting\n\nWhat Data You Can Extract\n\nPosts and Reels: Author details, post text, media URLs, like and comment counts, timestamps, and accessibility captions. Works on home feeds, profile grids, reels tabs, saved posts, and tagged posts.\n\nPost Details: Engagement metrics (likes, comments, views), location data, audio information, tagged users, carousel content, duration, and paid partnership status.\n\nComments: Comment text, timestamps, like counts, reply hierarchies, author usernames, and verification status.\n\nStories: Media URLs, expiration times, story type, accessibility captions, and author information.\n\nProfiles: Username, full name, biography, follower and following counts, post count, verification status, privacy status, business category, and external links.\n\nFollowers and Following: Usernames, full names, profile URLs, verification and privacy status, profile pictures, and latest story timestamps.\n\nMonitoring and Alerts\nPage Watcher lets you track changes on any Instagram page and get notified when something updates. Set conditions on specific fields you care about, such as follower count or new posts, and receive alerts via webhook or ntfy.sh when those conditions are met.\n\nBatch Processing\nBatch URL Opener allows you to queue multiple Instagram URLs and scrape them all in one session. Add URLs manually or auto-collect them from a list page, then batch-process every link and export the combined results.\n\nExport Formats\n- CSV\n- Excel (XLSX)\n- JSON\n- HTML Table\n- Raw JSON\n\nPerfect For\nThis tool is ideal for archiving social media content, tracking public engagement metrics, and compiling profile data from Instagram. It suits researchers, marketers, and anyone who needs structured exports of publicly visible information.\n\nSupported Regions\nThis extension works with Instagram globally and is compatible across all regions where the platform is accessible.\n\nPrivacy and Security\nAll scraping and data exports are processed locally on your machine. General usage metrics are collected to help improve the extension, but no scraped data is transmitted to external servers.\n\nUsage Notice\nThis extension provides tools for extracting publicly available data. Users are responsible for ensuring their use complies with applicable terms of service and local regulations. The developer provides the tool and is not liable for how it is used.\n\nTechnical Support\nFor assistance or questions, contact hello@nifty.codes.\n\nTrademark\nInstagram is a registered trademark of Meta Platforms, Inc. This extension is an independent tool and is not affiliated with, endorsed by, authorized, maintained, or sponsored by Meta Platforms, Inc." }, "options": { "posts": { "id": "posts", "title": "Posts in a List", "description": "Works on feeds, profiles, reels, saved posts", "instructions": [ "Go to any page that shows a list of posts (home feed, profile, reels tab, saved, etc.)", "Make sure posts are visible and fully loaded" ], "type": "list", "urlField": "Post URL", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Post Author", "fixes": [], "timeout": 10000, "fields": [ "Post Author", "Post Author Full Name", "Post Author Image", "Post Author URL", "Post Author Is Verified", "Post Type", "Post Text", "Post Image", "Post Video", "Post Likes", "Post Comments Count", "Post URL", "Post Date", "Is Comments Disabled", "Post Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "media_notes" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "clips_tab_pinned_user_ids" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "sponsor_tags" }, "parse": true } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "xdt_api__v1__feed__user_timeline_graphql_connection" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "media_count" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 1, "group": "deepReelSearch", "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "attemptLimit": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "clips_tab_pinned_user_ids" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "url", "operator": "contains", "value": "/reels/" }, { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "meta[property='og:type'][content='profile']" } ] } ] } ] } }, { "delay": 0, "attempts": 3, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 3, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "isFetching", "posts" ], "limit": 1, "ancestorLevel": 0 }, "condition": [ { "type": "url", "operator": "contains", "value": "/saved/" } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href*='/p/{{{id2}}}/'], a[href*='/reel/{{{id2}}}/']" }, { "method": "closest", "query": "article" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "preset", "query": "window" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] }, "condition": [ { "type": "url", "operator": "endsWith", "value": "instagram.com/" } ] }, "postDetails": { "id": "postDetails", "title": "Post Details", "description": "Works on individual post or reel pages", "instructions": [ "Open a specific post or reel on Instagram", "Wait for all the details to fully load" ], "type": "detail", "unit": { "singular": "Post", "plural": "Posts" }, "frontField": "Caption", "fixes": [], "timeout": 10000, "fields": [ "Post Type", "Caption", "Like Count", "Comment Count", "View Count", "Author Username", "Author Full Name", "Author ID", "Author URL", "Author Is Verified", "Author Profile Pic", "Post URL", "Image URL", "Video URL", "Cover Image", "Posted At", "Location", "Duration", "Has Audio", "Audio Title", "Audio Artist", "Tagged Users", "Is Carousel", "Carousel Count", "Original Width", "Original Height", "Is Paid Partnership", "Comments Disabled", "Likes Disabled", "Accessibility Caption" ], "scraping": { "attempts": 0, "delay": 1000, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 0, "attempts": 1, "searchConfig": { "data": [ "coauthor_producers" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 0, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "coauthor_producers" ], "limit": 1, "ancestorLevel": 1 } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 0, "attempts": 1, "searchConfig": { "data": [ "clips_metadata" ] }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "react", "delay": 0, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "clips_metadata" ], "limit": 1, "ancestorLevel": 1 }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "window", "delay": 0, "attempts": 1, "searchConfig": { "nestedKey": "location.href" }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels" } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "article[role='presentation']" } ] }, "paginate": { "delay": 2500, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "preset", "query": "window" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 }, "condition": [ { "type": "url", "operator": "contains", "value": "instagram.com/reels/" } ] } ] } }, "comments": { "id": "comments", "title": "Comments of a Post", "description": "Works on post pages with comments", "instructions": [ "Go to a post page on Instagram", "Scroll down to the comments section", "Make sure comments are visible on the page" ], "type": "list", "urlField": "Author URL", "unit": { "singular": "Comment", "plural": "Comments" }, "frontField": "Comment Text", "fixes": [], "timeout": 10000, "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "comments" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "comment_like_count" }, "parse": true } ] } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "time[datetime*='{{{id2}}}']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "article ul" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 }, "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "article ul" } ] } ] }, { "type": "click", "target": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" }, { "method": "closest", "query": "button" } ], "condition": [ { "type": "element", "operator": "exists", "value": [ { "method": "querySelector", "query": "svg[aria-label='Load more comments']" } ] } ] }, { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "svg[aria-label='Comment']" }, { "method": "closest", "query": "section" }, { "method": "parentElement", "query": "" }, { "method": "parentElement", "query": "" }, { "method": "querySelector", "query": ":scope > div:nth-child(3)" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "stories": { "id": "stories", "title": "Stories", "description": "Works on stories viewer pages", "instructions": [ "Go to a user's stories on Instagram", "Make sure the story is playing or visible" ], "type": "list", "urlField": "Story Link", "unit": { "singular": "Story", "plural": "Stories" }, "frontField": "Story Date", "fixes": [], "timeout": 10000, "fields": [ "Story Type", "Story Image", "Story Video", "Story Link", "Story Date", "Story Expiring At", "Story Accessibility Caption", "Author Username", "Author Full Name", "Author Image", "Author URL", "Author Is Verified" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "data": [ "story_cta" ] } } ] } }, { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "dom", "delay": 500, "attempts": 1, "searchConfig": [ { "type": "selector", "query": "script", "all": true, "key": "data", "value": "text", "filter": { "contains": "story_cta" }, "parse": true } ] } ] } } ], "log": false }, "condition": [ { "type": "url", "operator": "contains", "value": "stories" } ] }, "profile": { "id": "profile", "title": "Profile Details", "description": "Works on user profile pages", "instructions": [ "Navigate to a user's profile page on Instagram", "Make sure their profile information is visible" ], "type": "detail", "unit": { "singular": "Profile", "plural": "Profiles" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Biography", "Profile URL", "External URL", "Follower Count", "Following Count", "Post Count", "Is Verified", "Is Private", "Is Business", "Category", "Profile Picture URL", "All Bio Links" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/graphql/query", "data": [ "follower_count", "following_count", "biography" ] } } ] } }, { "delay": 0, "attempts": 1, "attemptLimit": 1, "methods": { "A": [ { "type": "react", "delay": 500, "attempts": 1, "searchConfig": { "reactSelector": "div[id*='mount_0_0']", "reactKey": "__reactContainer", "searchProperty": [ "full_name", "following_count", "biography" ], "limit": 1, "ancestorLevel": 1 } } ] } } ], "log": false } }, "followers": { "id": "followers", "title": "Followers of a User", "description": "Works on the followers popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the followers count to open the followers list", "Make sure the followers list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Follower", "plural": "Followers" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/followers/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } }, "following": { "id": "following", "title": "Following of a User", "description": "Works on the following popup dialog", "instructions": [ "Navigate to a user's profile on Instagram", "Click on the following count to open the following list", "Make sure the following list is visible" ], "type": "list", "urlField": "Profile URL", "unit": { "singular": "Account", "plural": "Accounts" }, "frontField": "Username", "fixes": [], "timeout": 10000, "fields": [ "Username", "Full Name", "Profile URL", "Is Private", "Is Verified", "Profile Picture URL", "Has Anonymous Profile Picture", "Latest Story Time" ], "scraping": { "attempts": 0, "delay": 1500, "partials": [ { "delay": 0, "attempts": 1, "methods": { "A": [ { "type": "request", "delay": 500, "attempts": 1, "searchConfig": { "url": "/following/" } } ] } } ], "log": false }, "highlight": { "color": "#F80261", "instructions": [ { "method": "querySelector", "query": "a[href='/{{{id2}}}/']" }, { "method": "closest", "query": "div[role='button']" } ] }, "paginate": { "delay": 4000, "instructions": [ { "type": "scroll", "scrollType": "to", "target": [ { "method": "querySelector", "query": "div[style*='overflow: hidden auto']" }, { "method": "parentElement" } ], "amount": { "type": "preset", "value": "bottom" }, "steps": { "amount": 1, "delay": 0 } } ] } } }, "tests": [ { "option_id": "posts", "title": "Test Posts on Home Feed", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/", "url_updated_at": "2026-04-13T16:45:46.350Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts on Profile", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T16:49:39.489Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Saved", "flow": [ { "action": "open_url", "data": { "url": "https://www.instagram.com/biketonin/saved/all-posts/", "url_updated_at": "2026-05-12T04:23:50.105Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Reels Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/reels/", "https://www.instagram.com/markrober/reels/", "https://www.instagram.com/cristiano/reels/" ], "url_updated_at": "2026-04-13T16:51:58.657Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "posts", "title": "Test Posts in Tagged Tab", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/tagged/", "https://www.instagram.com/cristiano/tagged/", "https://www.instagram.com/markrober/tagged/" ], "url_updated_at": "2026-04-13T16:53:01.569Z" } }, { "action": "select_option", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "posts" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "posts" } }, { "action": "cancel_option", "data": { "optionId": "posts" } } ] }, { "option_id": "postDetails", "title": "Test Post Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T16:54:49.214Z" } }, { "action": "select_option", "data": { "optionId": "postDetails" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "postDetails" } }, { "action": "cancel_option", "data": { "optionId": "postDetails" } } ] }, { "option_id": "comments", "title": "Test Post Comments", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/p/DWZitMjgHp1/", "https://www.instagram.com/p/DXAUDExAMIG/", "https://www.instagram.com/p/DEIRwAdvBnA/" ], "url_updated_at": "2026-04-13T17:15:44.201Z" } }, { "action": "select_option", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "comments" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "comments" } }, { "action": "cancel_option", "data": { "optionId": "comments" } } ] }, { "option_id": "stories", "title": "Test Stories", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/stories/highlights/17929440964811872/", "https://www.instagram.com/stories/highlights/17942782034297155/" ], "url_updated_at": "2026-04-13T17:18:12.361Z" } }, { "action": "select_option", "data": { "optionId": "stories" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "stories" } }, { "action": "cancel_option", "data": { "optionId": "stories" } } ] }, { "option_id": "profile", "title": "Test Profile Details", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:25:37.449Z" } }, { "action": "select_option", "data": { "optionId": "profile" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "found", "data": { "optionId": "profile" } }, { "action": "cancel_option", "data": { "optionId": "profile" } } ] }, { "option_id": "followers", "title": "Test Followers of a User", "flow": [ { "action": "open_url", "data": { "url": [ "https://www.instagram.com/mrbeast/", "https://www.instagram.com/cristiano/", "https://www.instagram.com/markrober/" ], "url_updated_at": "2026-04-13T17:28:55.288Z" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "click", "data": { "selector": "a[href*='/followers/']" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "select_option", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 10000 } }, { "action": "paginate", "data": { "optionId": "followers" } }, { "action": "wait", "data": { "timeout": 7000 } }, { "action": "found", "data": { "optionId": "followers" } }, { "action": "cancel_option", "data": { "optionId": "followers" } } ] } ]}1// Browser-side scraping engine — injected into page via page.evaluate()2// All methods run in the real browser context (not Node.js)3
4window.scrapingEngine = {5
6 _shouldSkip: function(obj) {7 try {8 if (obj instanceof Date || obj instanceof RegExp || obj instanceof Error) return true;9 // Check for Window objects (including cross-origin)10 try { if (obj === obj.window) return true; } catch(e) { return true; }11 // Check valueOf — may throw on cross-origin12 try {13 if (typeof obj.valueOf === 'function') {14 var val = obj.valueOf();15 if (val !== obj && (typeof val !== 'object' || val === null)) return true;16 }17 } catch(e) { return true; }18 } catch(e) { return true; }19 return false;20 },21
22 fastDeepSearchField: function(obj, fieldName, opts) {23 opts = opts || {};24 var limit = opts.limit === undefined ? Infinity : opts.limit;25 var maxDepth = opts.maxDepth === undefined ? Infinity : opts.maxDepth;26 var ancestorLevel = opts.ancestorLevel || 0;27 var flexLimit = opts.flexLimit || false;28 var flexLimitThreshold = opts.flexLimitThreshold || 50000;29 var maxBreadth = opts.maxBreadth || Infinity;30 if (!obj || typeof obj !== 'object') return [];31
32 var self = this;33 var matcher = this.buildMatcher(fieldName);34 var results = [];35 var seen = new WeakSet();36 var stack = [[obj, 0, [obj]]];37
38 // flexLimit tracking39 var defaultFlexGap = flexLimitThreshold;40 var flexBuffer = 15000;41 var flexIterations = 0;42 var flexLastMatchAt = 0;43 var flexGaps = [];44
45 while (stack.length > 0 && (flexLimit || results.length < limit)) {46 if (flexLimit) {47 flexIterations++;48 var gap = flexIterations - flexLastMatchAt;49 var sorted = flexGaps.slice().sort(function(a, b) { return a - b; });50 var median = sorted.length > 0 ? sorted[sorted.length >> 1] : 0;51 var threshold = sorted.length > 0 ? (median + flexBuffer) : defaultFlexGap;52 if (gap > threshold) break;53 }54 var item = stack.pop();55 var current = item[0], depth = item[1], path = item[2];56 if (current == null || typeof current !== 'object') continue;57 if (self._shouldSkip(current)) continue;58 try { if (seen.has(current)) continue; } catch(e) { continue; }59 try { seen.add(current); } catch(e) { continue; }60
61 if (matcher(current)) {62 var result = current;63 if (ancestorLevel > 0) {64 var idx = path.length - 1 - ancestorLevel;65 if (idx >= 0) result = path[idx]; else continue;66 }67 results.push(result);68
69 if (flexLimit) {70 var matchGap = flexIterations - flexLastMatchAt;71 if (flexLastMatchAt > 0) flexGaps.push(matchGap);72 flexLastMatchAt = flexIterations;73 }74
75 if (!flexLimit && results.length >= limit) break;76 }77
78 if (depth >= maxDepth) continue;79
80 try {81 if (Array.isArray(current)) {82 var cappedLen = Math.min(current.length, maxBreadth);83 for (var i = cappedLen - 1; i >= 0; i--) {84 var v = current[i];85 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);86 }87 } else {88 var keys;89 try { keys = Object.keys(current); } catch(e) { continue; }90 var cappedKeysLen = Math.min(keys.length, maxBreadth);91 for (var i = cappedKeysLen - 1; i >= 0; i--) {92 var v;93 try { v = current[keys[i]]; } catch(e) { continue; }94 if (v && typeof v === 'object' && !self._shouldSkip(v)) stack.push([v, depth + 1, path.concat([v])]);95 }96 }97 } catch(e) { }98 }99 return results;100 },101
102 buildMatcher: function(fieldSpec) {103 var self = this;104 if (typeof fieldSpec === 'string') {105 return function(obj) { return fieldSpec in obj; };106 }107 if (Array.isArray(fieldSpec)) {108 if (fieldSpec.length === 0) return function() { return true; };109 if (Array.isArray(fieldSpec[0])) {110 var groups = fieldSpec.map(function(group) { return group.map(function(f) { return self.buildMatcher(f); }); });111 return function(obj) { return groups.some(function(group) { return group.every(function(m) { return m(obj); }); }); };112 }113 var matchers = fieldSpec.map(function(f) { return self.buildMatcher(f); });114 return function(obj) { return matchers.every(function(m) { return m(obj); }); };115 }116 if (typeof fieldSpec === 'object' && fieldSpec.field) {117 var field = fieldSpec.field, value = fieldSpec.value, mode = fieldSpec.mode || 'equal';118 if (value === undefined || value === null) return function(obj) { return field in obj; };119 return function(obj) {120 if (!(field in obj)) return false;121 var v = String(obj[field]).toLowerCase();122 var expected = String(value).toLowerCase();123 return mode === 'includes' ? v.indexOf(expected) !== -1 : v === expected;124 };125 }126 return function() { return false; };127 },128
129 searchReactRoot: function(element, searchConfig) {130 var reactKey = searchConfig.reactKey || '__reactFiber';131 var searchProperty = searchConfig.searchProperty;132 var limit = searchConfig.limit || 1;133 var ancestorLevel = searchConfig.ancestorLevel || 0;134 var maxDepth = searchConfig.maxDepth || Infinity;135 var flexLimit = searchConfig.flexLimit || false;136 var flexLimitThreshold = searchConfig.flexLimitThreshold || 50000;137 var maxBreadth = searchConfig.maxBreadth || Infinity;138 var keys = Object.keys(element);139 for (var i = 0; i < keys.length; i++) {140 if (keys[i].indexOf(reactKey) === 0) {141 return this.fastDeepSearchField(element[keys[i]], searchProperty, { limit: limit, maxDepth: maxDepth, ancestorLevel: ancestorLevel, flexLimit: flexLimit, flexLimitThreshold: flexLimitThreshold, maxBreadth: maxBreadth });142 }143 }144 return null;145 },146
147 DOMSearch: function(searchConfig) {148 if (!Array.isArray(searchConfig)) return [];149 var self = this;150 var results = [];151 for (var q = 0; q < searchConfig.length; q++) {152 var query = searchConfig[q];153 if (!query || !query.key) continue;154 try {155 var elements = self._getElements(query, document);156 if (query.filter) elements = self._filterElements(elements, query.filter);157 if (elements.length === 0) { results.push({ key: query.key, value: null }); continue; }158
159 if (query.nested && query.nested.length > 0) {160 // Nested queries: for each matched element, run sub-queries relative to it161 var nestedResults = self._processNested(elements, query.nested);162 var parsed = query.parse ? nestedResults.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : nestedResults;163 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });164 } else {165 var vals = self._extractValues(elements, query);166 var parsed = query.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;167 results.push({ key: query.key, value: query.all ? parsed : parsed[0] || null });168 }169 } catch(e) { results.push({ key: query.key, value: null }); }170 }171 return results;172 },173
174 _getElements: function(query, baseEl) {175 var elements = [];176 if (query.type === 'xpath') {177 var xr = document.evaluate(query.query, baseEl, null, query.all ? XPathResult.ORDERED_NODE_ITERATOR_TYPE : XPathResult.FIRST_ORDERED_NODE_TYPE, null);178 if (query.all) { var n = xr.iterateNext(); while (n) { elements.push(n); n = xr.iterateNext(); } }179 else { if (xr.singleNodeValue) elements.push(xr.singleNodeValue); }180 } else if (query.type === 'selector') {181 elements = query.all ? Array.from(baseEl.querySelectorAll(query.query)) : (baseEl.querySelector(query.query) ? [baseEl.querySelector(query.query)] : []);182 } else if (query.type === 'document') {183 elements = [baseEl];184 }185 return elements;186 },187
188 _filterElements: function(elements, filter) {189 if (!filter) return elements;190 return elements.filter(function(el) {191 var text = el.textContent || '';192 if (filter.contains) {193 if (Array.isArray(filter.contains)) { if (!filter.contains.every(function(s) { return text.indexOf(s) !== -1; })) return false; }194 else if (text.indexOf(filter.contains) === -1) return false;195 }196 if (filter.notContains) {197 if (Array.isArray(filter.notContains)) { if (filter.notContains.some(function(s) { return text.indexOf(s) !== -1; })) return false; }198 else if (text.indexOf(filter.notContains) !== -1) return false;199 }200 if (filter.startsWith && !text.trimStart().startsWith(filter.startsWith)) return false;201 if (filter.endsWith && !text.trimEnd().endsWith(filter.endsWith)) return false;202 if (typeof filter.minLength === 'number' && text.length < filter.minLength) return false;203 if (typeof filter.maxLength === 'number' && text.length > filter.maxLength) return false;204 return true;205 });206 },207
208 _extractValues: function(elements, query) {209 var value = query.value || 'text';210 var custom = query.custom || null;211 return elements.map(function(el) {212 if (value === 'text') return el.textContent || '';213 if (value === 'innerText') return el.innerText || '';214 if (value === 'html') return el.innerHTML || '';215 if (value === 'attribute' && custom && custom.attributeKey) return el.getAttribute(custom.attributeKey) || null;216 return el.textContent || '';217 });218 },219
220 _processNested: function(baseElements, nestedQueries) {221 var self = this;222 var results = [];223 for (var i = 0; i < baseElements.length; i++) {224 var baseEl = baseElements[i];225 var elementResult = {};226 for (var n = 0; n < nestedQueries.length; n++) {227 var nq = nestedQueries[n];228 if (!nq || !nq.key) continue;229 try {230 var childElements = self._getElements(nq, baseEl);231 if (nq.filter) childElements = self._filterElements(childElements, nq.filter);232 if (childElements.length === 0) { elementResult[nq.key] = null; continue; }233
234 if (nq.nested && nq.nested.length > 0) {235 var deepNested = self._processNested(childElements, nq.nested);236 var parsed = nq.parse ? deepNested.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : deepNested;237 elementResult[nq.key] = nq.all ? parsed : parsed[0] || null;238 } else {239 var vals = self._extractValues(childElements, nq);240 var parsed = nq.parse ? vals.map(function(v) { try { return JSON.parse(v); } catch(e) { return v; } }) : vals;241 elementResult[nq.key] = nq.all ? parsed : vals[0] || null;242 }243 } catch(e) { elementResult[nq.key] = null; }244 }245 results.push(elementResult);246 }247 return results;248 }249};1// Anti-detection enhancements — applied in preNavigationHooks and postNavigationHooks2
3import UserAgent from 'user-agents';4
5const TRACKER_DOMAINS = [6 'google-analytics.com', 'googletagmanager.com', 'facebook.net',7 'doubleclick.net', 'hotjar.com', 'segment.com', 'mixpanel.com',8 'amplitude.com', 'fullstory.com', 'clarity.ms', 'newrelic.com',9 'sentry.io', 'bugsnag.com', 'datadoghq.com', 'mouseflow.com',10 'crazyegg.com', 'optimizely.com', 'hubspot.com', 'intercom.io',11 'drift.com', 'tawk.to', 'livechatinc.com'12];13
14const COOKIE_SELECTORS = [15 '[data-testid="cookie-policy-dialog-accept-button"]',16 'button[id*="accept"]', 'button[class*="accept"]',17 'button[class*="consent"]', '#onetrust-accept-btn-handler',18 '.cookie-consent-accept', '[aria-label*="Accept"]',19 '[aria-label*="accept"]', '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',20 '.cc-btn.cc-allow', '#gdpr-cookie-accept', '.js-cookie-consent-agree'21];22
23export async function applyPreNavigation(page, enhancements) {24 if (enhancements.blockTrackers) {25 await page.route('**/*', (route) => {26 const url = route.request().url();27 if (TRACKER_DOMAINS.some(d => url.includes(d))) return route.abort();28 return route.continue();29 });30 }31
32 if (enhancements.rotateUserAgent) {33 const ua = new UserAgent({ deviceCategory: 'desktop' });34 await page.setExtraHTTPHeaders({ 'User-Agent': ua.toString() });35 }36}37
38export async function applyPostNavigation(page, enhancements) {39 if (enhancements.dismissCookies) {40 try {41 for (const sel of COOKIE_SELECTORS) {42 const btn = await page.$(sel);43 if (btn) { await btn.click().catch(() => {}); break; }44 }45 } catch { }46 }47
48 // Post-navigation delay (radio: none / random / custom)49 const postNav = enhancements.postNavigation || 'none';50 if (postNav === 'random') {51 await page.waitForTimeout(1000 + Math.floor(Math.random() * 2000));52 } else if (postNav === 'custom') {53 const delay = typeof enhancements.postNavigationDelay === 'number' ? enhancements.postNavigationDelay : 2000;54 await page.waitForTimeout(delay);55 }56}1import { log } from 'crawlee';2
3/**4 * Execute a custom user script in the browser context.5 * The script should define: async function main(context) { ... }6 * context = { url, hostname, pathname, searchParams, optionKey, timing }7 */8export async function executeCustomScript(page, enhancements, context) {9 const code = enhancements.customScriptCode;10 if (!code) return;11
12 try {13 log.info('Custom script: executing (' + context.timing + ')');14 await page.evaluate(async ({ scriptCode, ctx }) => {15 const fn = new Function('context', `return (async function() { ${scriptCode} }).call(null)`);16 await fn(ctx);17 }, { scriptCode: code, ctx: context });18 log.info('Custom script: completed');19 } catch (err) {20 log.warning('Custom script error: ' + err.message);21 }22}23
24/**25 * Execute a beforeLoad custom script in Node context.26 * Can return a modified URL string.27 */28export function executeBeforeLoadScript(enhancements, url) {29 const code = enhancements.customScriptCode;30 if (!code) return url;31
32 try {33 const parsed = new URL(url);34 const context = {35 url,36 hostname: parsed.hostname,37 pathname: parsed.pathname,38 searchParams: Object.fromEntries(parsed.searchParams),39 timing: 'beforeLoad'40 };41 const fn = new Function('context', code);42 const result = fn(context);43 if (typeof result === 'string') {44 log.info('Custom script (beforeLoad): URL modified to ' + result);45 return result;46 }47 } catch (err) {48 log.warning('Custom script (beforeLoad) error: ' + err.message);49 }50 return url;51}1// Data processing enhancements — raw data, criteria filtering2
3export function formatForDataset(item, enhancements) {4 if (enhancements.rawData) {5 return { ...item.formatted, _raw: item.raw };6 }7 return item.formatted || item;8}9
10// ── Criteria filtering ───────────────────────────────────────11
12export function checkCondition(value, condition) {13 const { operator, value: target, caseSensitive } = condition;14 const str = String(value ?? '');15 const cmp = caseSensitive ? str : str.toLowerCase();16 const tgt = caseSensitive ? target : target.toLowerCase();17 switch (operator) {18 case 'equals': return cmp === tgt;19 case 'notEquals': return cmp !== tgt;20 case 'contains': return cmp.includes(tgt);21 case 'notContains': return !cmp.includes(tgt);22 case 'regex': try { return new RegExp(target, caseSensitive ? '' : 'i').test(str); } catch { return false; }23 case 'greaterThan': return parseFloat(str) > parseFloat(target);24 case 'lessThan': return parseFloat(str) < parseFloat(target);25 case 'dateBefore': return new Date(str) < new Date(target);26 case 'dateOn': return new Date(str).toDateString() === new Date(target).toDateString();27 case 'dateAfter': return new Date(str) > new Date(target);28 default: return true;29 }30}31
32export function applyCriteria(items, criteria) {33 if (!criteria || typeof criteria !== 'object' || Object.keys(criteria).length === 0) return items;34 return items.filter(item => {35 for (const [field, conditions] of Object.entries(criteria)) {36 if (!Array.isArray(conditions)) continue;37 const val = item[field];38 for (const cond of conditions) {39 if (!checkCondition(val, cond)) return false;40 }41 }42 return true;43 });44}1// Pagination enhancement — executes scroll/click pagination from config2// Mirrors extension's AutoPaginationManager: data-driven stopping with retry attempts3
4export async function runPagination(page, option, enhancements, processInBrowser) {5 if (!enhancements.pagination || !option.paginate) return [];6
7 const log = (await import('crawlee')).log;8 const paginateConfig = option.paginate;9 const maxPages = typeof enhancements.maxPages === 'number' ? enhancements.maxPages : 0; // 0 = infinite10 const delay = paginateConfig.delay || 3000;11 const maxAttempts = 2;12 const allResults = [];13
14 let pageNum = 0;15 let attempts = 0;16
17 log.info('Pagination: starting, maxPages=' + (maxPages || 'infinite') + ', delay=' + delay + 'ms, instructions=' + (paginateConfig.instructions || []).length);18
19 while (true) {20 if (maxPages > 0 && pageNum >= maxPages) {21 log.info('Pagination: reached maxPages limit (' + maxPages + ')');22 break;23 }24
25 pageNum++;26 log.info('Pagination: page ' + pageNum + (maxPages > 0 ? '/' + maxPages : '') + ' url=' + page.url());27
28 let acted = await executeInstructions(page, paginateConfig.instructions || [], log);29
30 if (!acted) {31 log.info('Pagination: waiting 8s for page to finish rendering...');32 await page.waitForTimeout(8000);33 acted = await executeInstructions(page, paginateConfig.instructions || [], log);34 }35
36 if (!acted) {37 attempts++;38 log.info('Pagination: nothing executed after wait, attempt ' + attempts + '/' + maxAttempts);39 if (attempts >= maxAttempts) {40 log.info('Pagination: stopping — no pagination action possible');41 break;42 }43 continue;44 }45
46 await page.waitForTimeout(delay);47
48 const newItems = await processInBrowser(page);49
50 log.info('Pagination: page ' + pageNum + ' got ' + newItems.length + ' new items');51
52 if (newItems.length > 0) {53 allResults.push(...newItems);54 attempts = 0;55 log.info('Pagination: total so far ' + allResults.length + ' items, continuing');56 } else {57 attempts++;58 log.info('Pagination: no new data, attempt ' + attempts + '/' + maxAttempts);59 if (attempts >= maxAttempts) {60 log.info('Pagination: stopping after ' + maxAttempts + ' attempts with no new data');61 break;62 }63 }64 }65
66 log.info('Pagination: done, ' + pageNum + ' pages, ' + allResults.length + ' total new items');67 return allResults;68}69
70// Execute pagination instructions (scroll, click, delay) with full target chain support71async function executeInstructions(page, instructions, log) {72 let anyExecuted = false;73
74 for (let i = 0; i < instructions.length; i++) {75 const instruction = instructions[i];76 try {77 if (instruction.condition) {78 const condMet = await checkCondition(page, instruction.condition);79 if (!condMet) {80 log.info('Pagination: instruction[' + i + '] condition not met, skipping');81 continue;82 }83 }84
85 anyExecuted = true;86
87 if (instruction.type === 'delay') {88 await page.waitForTimeout(instruction.amount || 1000);89 } else if (instruction.type === 'scroll') {90 await executeScroll(page, instruction, log);91 } else if (instruction.type === 'click') {92 await executeClick(page, instruction, log);93 }94 } catch (err) {95 log.info('Pagination: instruction[' + i + '] error: ' + err.message);96 }97 }98
99 if (!anyExecuted) {100 log.info('Pagination: all conditions failed, no action taken');101 }102
103 return anyExecuted;104}105
106// Scroll with full support: scrollTo bottom, scrollBy, steps, presets, chained targets107async function executeScroll(page, instruction, log) {108 const { scrollType, amount, steps } = instruction;109 const el = await findElement(page, instruction.target);110
111 const stepCount = steps?.amount || 1;112 const stepDelay = steps?.delay || 100;113
114 if (scrollType === 'to' && amount?.type === 'preset' && amount.value === 'bottom') {115 for (let s = 0; s < stepCount; s++) {116 if (el) {117 await el.evaluate((e, args) => {118 e.scrollTo(0, (e.scrollHeight / args.total) * (args.step + 1));119 }, { step: s, total: stepCount });120 } else {121 await page.evaluate(({ step, total }) => {122 window.scrollTo(0, (document.body.scrollHeight / total) * (step + 1));123 }, { step: s, total: stepCount });124 }125 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);126 }127 log.info('Pagination: scrolled to bottom' + (stepCount > 1 ? ' in ' + stepCount + ' steps' : ''));128 } else if (scrollType === 'by') {129 const scrollAmount = amount?.value || 0;130 const stepSize = scrollAmount / stepCount;131 for (let s = 0; s < stepCount; s++) {132 if (el) {133 await el.evaluate((e, amt) => e.scrollBy(0, amt), stepSize);134 } else {135 await page.evaluate((amt) => window.scrollBy(0, amt), stepSize);136 }137 if (stepDelay > 0 && s < stepCount - 1) await page.waitForTimeout(stepDelay);138 }139 log.info('Pagination: scrolled by ' + scrollAmount + 'px');140 } else {141 if (el) {142 await el.evaluate(e => e.scrollTo(0, e.scrollHeight));143 } else {144 await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));145 }146 log.info('Pagination: scrolled');147 }148}149
150// Click with chained target support151async function executeClick(page, instruction, log) {152 const target = instruction.target;153 const step = Array.isArray(target) ? target[0] : target;154 const query = step?.query;155 const method = step?.method;156
157 if (query) {158 const clicked = await page.evaluate(({ query, method }) => {159 let el;160 if (method === 'xpath') {161 el = document.evaluate(query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;162 } else {163 el = document.querySelector(query);164 }165 if (el) { el.click(); return true; }166 return false;167 }, { query, method });168
169 if (clicked) {170 log.info('Pagination: clicked');171 } else {172 log.info('Pagination: click target not found');173 }174 } else {175 log.info('Pagination: no click query');176 }177}178
179// Find element with chained target support (matches extension's unified {method, query} format)180async function findElement(page, target) {181 if (!target) return null;182
183 if (!Array.isArray(target)) target = [target];184
185 let el = null;186 for (const step of target) {187 if (!step.method) continue;188
189 if (step.method === 'querySelector') {190 el = el ? await applyMethod(page, el, step) : await page.$(step.query);191 } else if (step.method === 'xpath') {192 el = el ? await applyMethod(page, el, step) : await page.$('xpath=' + step.query);193 } else if (step.method === 'preset') {194 if (['window', 'document'].includes(step.query?.toLowerCase())) return null;195 if (step.query?.toLowerCase() === 'body') el = await page.$('body');196 } else if (el) {197 el = await applyMethod(page, el, step);198 }199 if (!el) return null;200 }201 return el;202}203
204// Apply chained DOM methods (matches extension's Paginater.applyMethod)205async function applyMethod(page, el, step) {206 const { method, query } = step;207 try {208 switch (method) {209 case 'closest':210 return await el.evaluateHandle((e, q) => e.closest(q), query);211 case 'parentElement':212 return await el.evaluateHandle(e => e.parentElement);213 case 'querySelector':214 return await el.evaluateHandle((e, q) => e.querySelector(q), query);215 case 'nextElementSibling':216 return await el.evaluateHandle(e => e.nextElementSibling);217 case 'previousElementSibling':218 return await el.evaluateHandle(e => e.previousElementSibling);219 case 'firstElementChild':220 return await el.evaluateHandle(e => e.firstElementChild);221 case 'lastElementChild':222 return await el.evaluateHandle(e => e.lastElementChild);223 default:224 return null;225 }226 } catch {227 return null;228 }229}230
231async function checkCondition(page, conditions) {232 if (!Array.isArray(conditions)) return true;233 return page.evaluate((conds) => {234 for (const cond of conds) {235 if (cond.type === 'element') {236 const target = cond.value?.[0];237 if (!target) continue;238 let el = null;239 if (target.method === 'querySelector') {240 el = document.querySelector(target.query);241 } else if (target.method === 'xpath') {242 el = document.evaluate(target.query, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;243 }244 if (cond.operator === 'exists' && !el) return false;245 if (cond.operator === 'not_exists' && el) return false;246 } else if (cond.type === 'url') {247 const url = window.location.href;248 if (cond.operator === 'contains' && !url.includes(cond.value)) return false;249 if (cond.operator === 'notContains' && url.includes(cond.value)) return false;250 if (cond.operator === 'startsWith' && !url.startsWith(cond.value)) return false;251 if (cond.operator === 'endsWith' && !url.endsWith(cond.value)) return false;252 if (cond.operator === 'equals' && url !== cond.value) return false;253 if (cond.operator === 'notEquals' && url === cond.value) return false;254 }255 }256 return true;257 }, conditions);258}1// Performance enhancements — resource blocking, crawler config, session pool2
3export async function applyResourceBlocking(page, enhancements) {4 const blockTypes = [];5 if (enhancements.blockImages) blockTypes.push('image');6 if (enhancements.blockMedia) blockTypes.push('media', 'font');7 if (enhancements.blockStylesheets) blockTypes.push('stylesheet');8
9 if (blockTypes.length > 0) {10 await page.route('**/*', (route) => {11 if (blockTypes.includes(route.request().resourceType())) return route.abort();12 return route.continue();13 });14 }15}16
17export async function getCrawlerOptions(enhancements) {18 const { Actor } = await import('apify');19 const opts = {};20
21 // Max timeout (0 = no limit)22 const timeout = typeof enhancements.maxTimeout === 'number' ? enhancements.maxTimeout : 120;23 opts.requestHandlerTimeoutSecs = timeout === 0 ? 999999 : timeout;24
25 // Max concurrent tabs — use autoscaling instead of hard concurrency26 const maxTabs = typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1;27 opts.maxConcurrency = maxTabs;28 opts.minConcurrency = 1;29 if (maxTabs > 1) {30 opts.autoscaledPoolOptions = {31 desiredConcurrencyRatio: 0.85,32 scaleUpStepRatio: 0.5,33 scaleDownStepRatio: 0.2,34 snapshotterOptions: {35 maxBlockedMillis: 2000,36 }37 };38 }39
40 // Session pool — proper Apify SessionPool integration41 if (enhancements.sessionPool) {42 const maxUsage = typeof enhancements.sessionMaxUsage === 'number' ? enhancements.sessionMaxUsage : 5;43 opts.useSessionPool = true;44 opts.persistCookiesPerSession = true;45 opts.sessionPoolOptions = {46 maxPoolSize: 20,47 sessionOptions: {48 maxAgeSecs: 300,49 maxUsageCount: maxUsage,50 }51 };52 }53
54 // Proxy configuration55 if (enhancements.proxyApifyResidential) {56 opts.proxyConfiguration = await Actor.createProxyConfiguration({57 groups: ['RESIDENTIAL']58 });59 } else if (enhancements.proxyApifyDatacenter) {60 opts.proxyConfiguration = await Actor.createProxyConfiguration();61 }62
63 return opts;64}{ "stealth": true, "waitUntil": "domcontentloaded", "postNavigation": "none", "navigationTimeout": 45, "maxTimeout": 0, "retryNavigation": 0, "blockImages": true, "blockMedia": true, "blockStylesheets": true, "blockTrackers": true, "actorMemory": 2048, "maxTabs": 3, "reuseContext": true, "retryIfEmpty": true, "retryMaxAttempts": 2, "abortOnLoss": true, "abortWarmupUrls": 5, "abortWarmupSeconds": 60, "_optionKey": "comments"}1// DOM scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 return window.scrapingEngine.DOMSearch(searchConfig);8 }, method.searchConfig);9
10 return rawData;11}1// React fiber scraping handler2
3export async function scrape(page, option, enhancements) {4 // Extract the single method from the synthetic option5 const method = option.scraping.partials[0].methods.A[0];6 const sc = method.searchConfig;7
8 const rawData = await page.evaluate(({ selector, searchConfig }) => {9 const el = document.querySelector(selector);10 if (!el) return null;11 return window.scrapingEngine.searchReactRoot(el, searchConfig);12 }, { selector: sc.reactSelector, searchConfig: sc });13
14 return rawData;15}1// Network request interception handler2import { log } from 'crawlee';3
4export async function scrape(page, option, enhancements, capturedResponses) {5 if (capturedResponses.length === 0) return null;6
7 // Use per-page processed timestamps stored on the array itself8 const processedTimestamps = capturedResponses._processedTimestamps || (capturedResponses._processedTimestamps = new Set());9
10 // Filter out already-processed responses11 const newResponses = capturedResponses.filter(r => r._timestamp && !processedTimestamps.has(r._timestamp));12 if (newResponses.length === 0) return null;13
14 // Mark as processed15 for (const r of newResponses) {16 processedTimestamps.add(r._timestamp);17 }18
19 return newResponses.map(r => {20 const { _timestamp, ...data } = r;21 return data;22 });23}24
25// Called from main.js preNavigationHooks to set up response listeners before navigation26export function setupInterception(page, option, capturedResponses) {27 const searchConfigs = [];28 for (const partial of option.scraping?.partials || []) {29 for (const [, methods] of Object.entries(partial.methods || {})) {30 for (const method of methods) {31 if (method.type === 'request') searchConfigs.push(method.searchConfig);32 }33 }34 }35
36 page.on('response', async (response) => {37 const url = response.url();38 for (const sc of searchConfigs) {39 const matchUrl = sc.url ? url.includes(sc.url) : true;40 if (!matchUrl) continue;41 try {42 const body = await response.text();43 let json;44 try { json = JSON.parse(body); } catch { continue; }45 const dataMatch = sc.data46 ? (Array.isArray(sc.data) ? sc.data.every(d => body.includes(d)) : body.includes(sc.data))47 : true;48 if (dataMatch) {49 json._timestamp = Date.now();50 capturedResponses.push(json);51 }52 } catch { }53 }54 });55}1// Window object scraping handler2
3export async function scrape(page, option, enhancements) {4 const method = option.scraping.partials[0].methods.A[0];5
6 const rawData = await page.evaluate((searchConfig) => {7 const nestedKey = searchConfig.nestedKey;8 const keys = Array.isArray(nestedKey) ? nestedKey : [nestedKey];9 const results = [];10 for (const key of keys) {11 let value = window;12 for (const part of key.split('.')) {13 if (value == null) break;14 value = value[part];15 }16 if (value != null) {17 if (Array.isArray(value)) results.push(...value);18 else results.push(value);19 }20 }21 return results.length > 0 ? results : null;22 }, method.searchConfig);23
24 return rawData;25}1import { PlaywrightCrawler, Dataset, log, KeyValueStore } from 'crawlee';2import { Actor } from 'apify';3import { fileURLToPath } from 'url';4import { dirname, join } from 'path';5import { readFileSync } from 'fs';6
7import { applyPreNavigation, applyPostNavigation } from './enhancements/anti-detection.js';8import { applyResourceBlocking, getCrawlerOptions } from './enhancements/performance.js';9import { runPagination } from './enhancements/pagination.js';10import { formatForDataset } from './enhancements/data.js';11import { executeCustomScript, executeBeforeLoadScript } from './enhancements/custom-script.js';12
13// Import ALL handlers14import * as reactHandler from './handlers/react.js';15import * as requestHandler from './handlers/request.js';16import * as domHandler from './handlers/dom.js';17import * as windowHandler from './handlers/window.js';18
19const handlerMap = {20 react: reactHandler,21 request: requestHandler,22 dom: domHandler,23 window: windowHandler24};25
26const __dirname = dirname(fileURLToPath(import.meta.url));27
28// ── Load config files ────────────────────────────────────────────29const config = JSON.parse(readFileSync(join(__dirname, 'config.json'), 'utf-8'));30const enhancements = JSON.parse(readFileSync(join(__dirname, 'enhancements.json'), 'utf-8'));31
32// ── Debug flags (set via test tab debug toggles) ─────────────────33const dbg = enhancements._debug || {};34const D = {35 config: dbg.config || false,36 navigation: dbg.navigation || false,37 injection: dbg.injection || false,38 partials: dbg.partials || false,39 rawData: dbg.rawData || false,40 processing: dbg.processing || false,41 pagination: dbg.pagination || false,42 requests: dbg.requests || false,43 browser: dbg.browser || false,44 performance: dbg.performance || false,45 screenshot: dbg.screenshot || false,46};47
48// Read script.js and engine.js as strings — they run in the BROWSER, not Node.js49const scriptCode = readFileSync(join(__dirname, 'script.js'), 'utf-8');50const engineCode = readFileSync(join(__dirname, 'engine.js'), 'utf-8');51
52const optionKey = enhancements._optionKey;53const option = config.options[optionKey];54if (!option) {55 log.error('Option "' + optionKey + '" not found in config');56 process.exit(1);57}58
59if (D.config) {60 log.info('DEBUG[config] optionKey=' + optionKey);61 log.info('DEBUG[config] enhancements=' + JSON.stringify(Object.keys(enhancements).filter(k => !k.startsWith('_') && enhancements[k])));62 log.info('DEBUG[config] hostnames=' + JSON.stringify(config.app.hostnames));63 log.info('DEBUG[config] fields=' + JSON.stringify(option.fields?.length || 0));64}65
66const hostnames = config.app.hostnames || [];67
68function validateUrl(url) {69 try {70 const parsed = new URL(url);71 return hostnames.some(h => parsed.hostname === h || parsed.hostname.endsWith('.' + h));72 } catch {73 return false;74 }75}76
77// Check if any partial uses request interception78function hasRequestMethods(opt) {79 for (const partial of opt.scraping?.partials || []) {80 for (const methods of Object.values(partial.methods || {})) {81 for (const method of methods) {82 if (method.type === 'request') return true;83 }84 }85 }86 return false;87}88
89const needsRequestInterception = hasRequestMethods(option);90
91function getPartialMethodType(partial) {92 if (!partial?.methods) return 'unknown';93 for (const methods of Object.values(partial.methods)) {94 for (const method of methods) {95 if (method.type) return method.type;96 }97 }98 return 'unknown';99}100
101// ── Signature tracking (mirrors extension's processedSignatures) ──────102
103let processedSignatures = new Set();104
105function hashString(str) {106 let hash = 0;107 for (let i = 0; i < str.length; i++) {108 const char = str.charCodeAt(i);109 hash = ((hash << 5) - hash) + char;110 hash = hash & hash;111 }112 return Math.abs(hash).toString(16).padStart(8, '0');113}114
115function generateSignature(data) {116 const seen = new WeakSet();117 return hashString(JSON.stringify(data, function(key, value) {118 if (typeof value === 'object' && value !== null) {119 if (seen.has(value)) return '[Circular]';120 seen.add(value);121 }122 return value;123 }));124}125
126// ── Build enhancement overrides for a retry attempt ──────────────127
128function getRetryEnhancements(attempt) {129 if (!enhancements.retryIfEmpty || !enhancements.retryEnhancements) return enhancements;130 const overrides = enhancements.retryEnhancements[attempt];131 if (!overrides) return enhancements;132 return { ...enhancements, ...overrides };133}134
135// ── Inject code into browser ─────────────────────────────────────136
137async function injectIntoPage(page) {138 if (D.injection) log.info('DEBUG[injection] Injecting engine.js and script.js into page');139 await page.evaluate((code) => {140 if (!window.scrapingEngine) new Function(code)();141 }, engineCode);142
143 await page.evaluate((code) => {144 if (!window.__scriptInjected) {145 new Function(code)();146 window.__scriptInjected = true;147 }148 }, scriptCode);149
150 if (D.browser) {151 const info = await page.evaluate((optKey) => {152 const fns = Object.keys(window).filter(k => { try { return typeof window[k] === 'function'; } catch { return false; } });153 const customFns = fns.filter(k => !['addEventListener','removeEventListener','dispatchEvent','eval','isFinite','isNaN','parseFloat','parseInt','decodeURI','decodeURIComponent','encodeURI','encodeURIComponent','escape','unescape','toString','valueOf'].includes(k) && !k.startsWith('on') && !k.startsWith('webkit') && k.length > 2);154 return {155 processFnExists: typeof window[optKey] === 'function',156 searchField: typeof window.searchField,157 scrapingEngine: typeof window.scrapingEngine,158 scriptInjected: !!window.__scriptInjected,159 customFnCount: customFns.length,160 sampleFns: customFns.slice(0, 15).join(',')161 };162 }, optionKey);163 log.info('DEBUG[browser] ' + JSON.stringify(info));164 }165
166 if (D.injection) log.info('DEBUG[injection] Done');167}168
169// ── Execute a single method (mirrors controller.js executeMethod) ──170
171async function executeMethod(page, method, capturedResponses) {172 const type = method.type;173 const handler = handlerMap[type];174 if (!handler) return { success: false, error: 'Unknown method type: ' + type };175
176 if (D.partials) log.info('DEBUG[partials] Executing method type=' + type + ' delay=' + (method.delay || 0) + ' attempts=' + (method.attempts || 1));177
178 try {179 let rawData;180 if (type === 'request') {181 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements, capturedResponses);182 } else {183 rawData = await handler.scrape(page, { scraping: { partials: [{ methods: { A: [method] } }], delay: method.delay || 0 } }, enhancements);184 }185
186 if (rawData && (Array.isArray(rawData) ? rawData.length > 0 : true)) {187 if (D.rawData) {188 log.info('DEBUG[rawData] type=' + type + ' isArray=' + Array.isArray(rawData) + ' length=' + (Array.isArray(rawData) ? rawData.length : 'N/A'));189 if (Array.isArray(rawData) && rawData.length > 0) {190 const sample = rawData[0];191 log.info('DEBUG[rawData] firstItem type=' + typeof sample + ' keys=' + (typeof sample === 'object' && sample ? Object.keys(sample).slice(0, 8).join(',') : 'N/A'));192 }193 }194 return { success: true, data: rawData };195 }196 if (D.partials) log.info('DEBUG[partials] method type=' + type + ' returned no data');197 return { success: false, error: 'No data returned' };198 } catch (err) {199 if (D.partials) log.error('DEBUG[partials] method type=' + type + ' error: ' + err.message);200 return { success: false, error: err.message };201 }202}203
204// ── Evaluate method conditions in browser context ────────────────205
206async function evaluateConditions(page, conditions) {207 if (!conditions || !Array.isArray(conditions) || conditions.length === 0) return true;208 return page.evaluate((conds) => {209 function evalSingle(cond) {210 const { type, operator, value } = cond;211 if (!type || !operator || value === undefined) return false;212 if (type === 'url') {213 const url = window.location.href;214 switch (operator) {215 case 'equals': return url === value;216 case 'contains': return url.includes(value);217 case 'startsWith': return url.startsWith(value);218 case 'endsWith': return url.endsWith(value);219 case 'regex': try { return new RegExp(value).test(url); } catch { return false; }220 case 'notEquals': return url !== value;221 case 'notContains': return !url.includes(value);222 default: return false;223 }224 }225 if (type === 'element') {226 if (!Array.isArray(value)) return false;227 // Check element existence/visibility using selectors from value array228 for (const v of value) {229 const query = v.query || v;230 if (typeof query !== 'string') continue;231 const el = document.querySelector(query);232 if (operator === 'exists' && !el) return false;233 if (operator === 'not_exists' && el) return false;234 if (operator === 'visible') {235 if (!el) return false;236 const rect = el.getBoundingClientRect();237 const style = window.getComputedStyle(el);238 if (rect.width === 0 || rect.height === 0 || style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;239 }240 }241 return true;242 }243 return false;244 }245
246 function evalGroup(arr) {247 if (!Array.isArray(arr) || arr.length === 0) return true;248 // If first element is array → OR logic249 if (Array.isArray(arr[0])) {250 return arr.some(sub => evalGroup(sub));251 }252 // AND logic253 return arr.every(item => {254 if (Array.isArray(item)) return evalGroup(item);255 if (typeof item === 'object') return evalSingle(item);256 return false;257 });258 }259 return evalGroup(conds);260 }, conditions);261}262
263// ── Process partials (mirrors controller.js processInstructions) ──264
265async function processAllPartials(page, capturedResponses) {266 const scraping = option.scraping;267 if (!scraping?.partials) return [];268
269 const partials = scraping.partials;270 const qualityOrder = ['A', 'B', 'C'];271
272 // Run all partials in parallel273 const partialPromises = partials.map(async (partial, pi) => {274 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] delay=' + (partial.delay || 0) + ' attempts=' + (partial.attempts || 1) + ' group=' + (partial.group || 'none') + ' methods=' + Object.keys(partial.methods || {}).join(','));275 if (partial.delay) await page.waitForTimeout(partial.delay);276
277 const partialAttempts = partial.attempts || 1;278
279 for (let attempt = 0; attempt < partialAttempts; attempt++) {280 for (const quality of qualityOrder) {281 const methods = partial.methods?.[quality];282 if (!methods || !Array.isArray(methods)) continue;283
284 for (const method of methods) {285 if (!method?.type) continue;286
287 // Check method conditions288 if (method.condition && Array.isArray(method.condition)) {289 const condMet = await evaluateConditions(page, method.condition);290 if (!condMet) {291 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] method condition not met, skipping');292 continue;293 }294 }295
296 if (method.delay) await page.waitForTimeout(method.delay);297
298 const methodAttempts = method.attempts || 1;299 for (let mAttempt = 0; mAttempt < methodAttempts; mAttempt++) {300 const result = await executeMethod(page, method, capturedResponses);301 if (result.success) {302 const signature = generateSignature(result.data);303 const alreadyProcessed = processedSignatures.has(signature);304 if (!alreadyProcessed) processedSignatures.add(signature);305 if (D.partials) log.info('DEBUG[partials] partial[' + pi + '] signature=' + signature + ' alreadyProcessed=' + alreadyProcessed);306 return { success: true, data: result.data, partialIndex: pi, alreadyProcessed };307 }308 if (mAttempt < methodAttempts - 1) await page.waitForTimeout(500);309 }310 }311 }312
313 if (attempt < partialAttempts - 1) await page.waitForTimeout(500);314 }315
316 return { success: false, partialIndex: pi };317 });318
319 const results = await Promise.all(partialPromises);320
321 // Group logic — mirrors extension's processInstructions322 const groupedPartials = new Map();323 const independentIndices = [];324
325 partials.forEach((partial, index) => {326 const group = partial.group;327 if (group) {328 const groups = Array.isArray(group) ? group : (typeof group === 'string' ? [group] : []);329 if (groups.length > 0) {330 groups.forEach(g => {331 if (!groupedPartials.has(g)) groupedPartials.set(g, []);332 groupedPartials.get(g).push(index);333 });334 } else {335 independentIndices.push(index);336 }337 } else {338 independentIndices.push(index);339 }340 });341
342 // Check group signatures (mirrors extension's processScrapingResults)343 for (const [group, indices] of groupedPartials.entries()) {344 const groupResults = results.filter(r => indices.includes(r?.partialIndex));345 const allSuccess = groupResults.every(r => r?.success);346 if (allSuccess) {347 const groupData = groupResults.reduce((acc, r) => [...acc, ...(Array.isArray(r.data) ? r.data : [r.data])], []);348 const groupSig = generateSignature(groupData);349 if (processedSignatures.has(groupSig)) {350 groupResults.forEach(r => { r.alreadyProcessed = true; });351 if (D.partials) log.info('DEBUG[partials] group "' + group + '" signature already processed');352 } else {353 processedSignatures.add(groupSig);354 }355 }356 }357
358 // Check if at least one group is fully satisfied359 let groupSatisfied = groupedPartials.size === 0;360 for (const [group, indices] of groupedPartials.entries()) {361 const allSuccess = indices.every(i => results[i]?.success);362 if (allSuccess) {363 groupSatisfied = true;364 if (D.partials) log.info('DEBUG[partials] group "' + group + '" satisfied');365 break;366 }367 }368
369 // Check independent partials (at least one must succeed)370 const independentSuccess = independentIndices.length === 0 || independentIndices.some(i => results[i]?.success);371
372 if (!groupSatisfied && !independentSuccess) {373 if (D.partials) log.info('DEBUG[partials] no groups satisfied and no independent successes');374 return [];375 }376
377 // Filter out already-processed results, collect only new data (mirrors extension line 631)378 const newResults = results.filter(r => r?.success && r.data && !r.alreadyProcessed);379 if (D.partials) log.info('DEBUG[partials] ' + results.filter(r => r?.success).length + ' successful partials, ' + newResults.length + ' new (not already processed)');380
381 if (newResults.length === 0) {382 if (D.partials) log.info('DEBUG[partials] all partials already processed, skipping');383 return [];384 }385
386 const allRawData = [];387 for (const r of newResults) {388 const methodType = getPartialMethodType(partials[r.partialIndex]);389 allRawData.push({390 data: r.data,391 metadata: {392 method: methodType,393 location: r.partialIndex + '0A',394 timestamp: Date.now()395 }396 });397 }398 return allRawData;399}400
401// ── Process raw data in the browser using script.js ──────────────402
403async function processRawDataInBrowser(page, allRawData) {404 if (allRawData.length === 0) {405 if (D.processing) log.info('DEBUG[processing] No raw data to process');406 return [];407 }408
409 if (D.processing) log.info('DEBUG[processing] partials=' + allRawData.length);410
411 const combinedRawData = allRawData;412
413 const results = await page.evaluate(({ rawData, optionKey, debug }) => {414 const processFn = window[optionKey];415 if (!processFn) {416 if (debug) console.log('DEBUG[processing] processFn not found for: ' + optionKey);417 return [];418 }419 try {420 const input = Array.isArray(rawData) ? rawData : [rawData];421 if (debug) console.log('DEBUG[processing] calling ' + optionKey + '() with ' + input.length + ' items');422 const items = processFn(input);423 if (!Array.isArray(items)) return items ? [{ id: null, formatted: items }] : [];424 if (debug) console.log('DEBUG[processing] ' + optionKey + '() returned ' + items.length + ' items');425 return items.map(item => ({426 id: item.id || null,427 raw: item.raw || null,428 formatted: item.formatted || item429 }));430 } catch (err) {431 console.error('Processing error:', err.message, err.stack);432 return [];433 }434 }, { rawData: combinedRawData, optionKey, debug: D.processing });435
436 if (D.processing) log.info('DEBUG[processing] browser returned ' + results.length + ' items');437
438 return results;439}440
441// ── Scrape a page with given enhancements (used for initial + retry) ──442
443async function scrapePage(page, enh, capturedResponses, seenIds) {444 // Apply wait-until strategy445 const waitUntil = enh.waitUntil || 'networkidle';446 if (waitUntil !== 'none' && waitUntil !== 'commit') {447 await page.waitForLoadState(waitUntil).catch(() => {});448 if (D.navigation) log.info('DEBUG[navigation] ' + waitUntil + ' reached');449 }450
451 if (D.screenshot) {452 try {453 const buf = await page.screenshot({ fullPage: false });454 const store = await KeyValueStore.open();455 await store.setValue('debug-before-scrape-' + Date.now(), buf, { contentType: 'image/png' });456 log.info('DEBUG[screenshot] saved pre-scrape screenshot');457 } catch { }458 }459
460 if (D.requests) {461 log.info('DEBUG[requests] capturedResponses=' + capturedResponses.length);462 capturedResponses.forEach(function(r, i) { log.info('DEBUG[requests] [' + i + '] keys=' + Object.keys(r).slice(0, 5).join(',')); });463 }464
465 await injectIntoPage(page);466
467 // Polling loop — mirrors extension's scraping.attempts + scraping.delay468 const scrapingAttempts = option.scraping?.attempts ?? 1;469 const scrapingDelay = option.scraping?.delay || 0;470 const maxScrapingAttempts = Math.min(scrapingAttempts === 0 ? 5 : scrapingAttempts, 5);471
472 let items = [];473 for (let attempt = 1; attempt <= maxScrapingAttempts; attempt++) {474 const rawData = await processAllPartials(page, capturedResponses);475 items = await processRawDataInBrowser(page, rawData);476
477 if (items.length > 0) {478 if (attempt > 1 && D.partials) log.info('DEBUG[partials] scraping succeeded on attempt ' + attempt);479 break;480 }481
482 if (attempt < maxScrapingAttempts && scrapingDelay > 0) {483 if (D.partials) log.info('DEBUG[partials] attempt ' + attempt + ' yielded 0 items, waiting ' + scrapingDelay + 'ms');484 await page.waitForTimeout(scrapingDelay);485 await injectIntoPage(page); // Re-inject in case page changed486 }487 }488
489 // Deduplicate against seenIds490 const newItems = [];491 for (const item of items) {492 const id = item.id || JSON.stringify(item.formatted || item).slice(0, 100);493 if (D.processing) log.info('DEBUG[dedup] id=' + (item.id ? item.id : 'hash:' + id.slice(0, 60)) + ' seen=' + seenIds.has(id));494 if (seenIds.has(id)) continue;495 seenIds.add(id);496 newItems.push(item);497 }498 if (D.processing) log.info('DEBUG[dedup] ' + items.length + ' items -> ' + newItems.length + ' new (seenIds total=' + seenIds.size + ')');499 return newItems;500}501
502// ── Main ─────────────────────────────────────────────────────────503
504await Actor.init();505const input = await Actor.getInput();506
507if (!input?.urls?.length) {508 log.error('No URLs provided in input');509 await Actor.exit();510 process.exit(1);511}512
513const urls = input.urls514 .map(u => typeof u === 'string' ? u : u.url)515 .filter(u => {516 if (!validateUrl(u)) {517 log.warning('Skipping invalid URL: ' + u);518 return false;519 }520 return true;521 });522
523if (urls.length === 0) {524 log.error('No valid URLs after filtering');525 await Actor.exit();526 process.exit(1);527}528
529const maxItems = input.maxItems || 100;530const enablePagination = input.enablePagination !== undefined ? input.enablePagination : !!enhancements.pagination;531const maxPages = typeof input.maxPages === 'number' ? input.maxPages : (enhancements.maxPages || 1);532
533// ── Watchdog & abort tracking ────────────────────────────────534let lastRowTime = Date.now();535const WATCHDOG_TIMEOUT = 100000;536const watchdog = setInterval(() => {537 if (Date.now() - lastRowTime > WATCHDOG_TIMEOUT) {538 log.error('Watchdog: no new rows for ' + (WATCHDOG_TIMEOUT / 1000) + 's, aborting');539 process.exit(1);540 }541}, 15000);542
543const runState = { urlsProcessed: 0, totalResults: 0, startTime: Date.now(), aborted: false };544const navTimeout = typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout * 1000 : 30000;545const navRetries = typeof enhancements.retryNavigation === 'number' ? enhancements.retryNavigation : 0;546
547function shouldAbort() {548 if (!enhancements.abortOnLoss || runState.totalResults > 0) return false;549 const warmupUrls = typeof enhancements.abortWarmupUrls === 'number' ? enhancements.abortWarmupUrls : 5;550 const warmupSecs = typeof enhancements.abortWarmupSeconds === 'number' ? enhancements.abortWarmupSeconds : 60;551 const elapsedSecs = (Date.now() - runState.startTime) / 1000;552 return runState.urlsProcessed >= warmupUrls && elapsedSecs >= warmupSecs;553}554
555// ── Process a single URL on a given page ─────────────────────556
557async function processUrl(page, url, capturedResponses) {558 if (runState.aborted) { log.info('Skipping (aborted): ' + url); return; }559 if (shouldAbort()) {560 runState.aborted = true;561 log.warning('Aborting: no results after ' + runState.urlsProcessed + ' URLs and ' + Math.round((Date.now() - runState.startTime) / 1000) + 's');562 return;563 }564
565 log.info('Processing: ' + url);566 const _t0 = Date.now();567
568 // Reset per-URL state569 processedSignatures = new Set();570
571 await applyPostNavigation(page, enhancements);572
573 if (D.navigation) {574 const pageUrl = page.url();575 const title = await page.title();576 log.info('DEBUG[navigation] landed on url=' + pageUrl.slice(0, 150) + ' title=' + title);577 }578
579 // Custom script — afterLoad, afterDelay, concurrent timing580 let concurrentScriptPromise = null;581 if (enhancements.customScript && enhancements.customScriptCode && enhancements.customScriptTiming !== 'beforeLoad') {582 const timing = enhancements.customScriptTiming || 'afterLoad';583 const parsedUrl = new URL(url);584 const ctx = {585 url,586 hostname: parsedUrl.hostname,587 pathname: parsedUrl.pathname,588 searchParams: Object.fromEntries(parsedUrl.searchParams),589 optionKey,590 timing591 };592
593 if (timing === 'afterLoad') {594 await executeCustomScript(page, enhancements, ctx);595 } else if (timing === 'afterDelay') {596 const delay = typeof enhancements.customScriptDelay === 'number' ? enhancements.customScriptDelay : 2000;597 await page.waitForTimeout(delay);598 await executeCustomScript(page, enhancements, ctx);599 } else if (timing === 'concurrent') {600 concurrentScriptPromise = executeCustomScript(page, enhancements, ctx);601 }602 }603
604 const seenIds = new Set();605 let totalExtracted = 0;606
607 const initialResults = await scrapePage(page, enhancements, capturedResponses, seenIds);608
609 // Retry If Empty610 let results = initialResults;611 if (results.length === 0 && enhancements.retryIfEmpty) {612 const maxRetries = typeof enhancements.retryMaxAttempts === 'number' ? enhancements.retryMaxAttempts : 3;613 log.info('Retry: 0 results, attempting up to ' + maxRetries + ' retries');614
615 for (let attempt = 0; attempt < maxRetries; attempt++) {616 const retryEnh = getRetryEnhancements(attempt);617 log.info('Retry: attempt ' + (attempt + 1) + '/' + maxRetries + (retryEnh.newContext ? ' (new context)' : ''));618
619 let retryPage = page;620 let retryContext = null;621 let retryCaptured = capturedResponses;622
623 if (retryEnh.newContext) {624 // Fresh browser context — new cookies, storage, fingerprint625 retryContext = await page.context().browser().newContext();626 retryPage = await retryContext.newPage();627 retryCaptured = [];628 await applyResourceBlocking(retryPage, retryEnh);629 await applyPreNavigation(retryPage, retryEnh);630 if (needsRequestInterception) {631 requestHandler.setupInterception(retryPage, option, retryCaptured);632 }633 }634
635 await retryPage.goto(url, { timeout: navTimeout, waitUntil: 'commit' });636 await applyPostNavigation(retryPage, retryEnh);637
638 const retryWait = retryEnh.waitUntil || 'networkidle';639 if (retryWait !== 'none' && retryWait !== 'commit') {640 await retryPage.waitForLoadState(retryWait).catch(() => {});641 }642
643 results = await scrapePage(retryPage, retryEnh, retryCaptured, seenIds);644
645 // Clean up new context if we created one646 if (retryContext) {647 await retryContext.close().catch(() => {});648 }649
650 if (results.length > 0) {651 log.info('Retry: attempt ' + (attempt + 1) + ' succeeded with ' + results.length + ' items');652 break;653 }654 }655 }656
657 const initialFormatted = results.map(item => formatForDataset(item, enhancements));658 for (const item of initialFormatted) {659 if (totalExtracted >= maxItems) break;660 await Dataset.pushData(item); lastRowTime = Date.now();661 totalExtracted++;662 }663
664 if (option.paginate && enablePagination) {665 await runPagination(666 page, option, { ...enhancements, pagination: true, maxPages },667 async (pg) => {668 const newItems = await scrapePage(pg, enhancements, capturedResponses, seenIds);669 const formatted = newItems.map(item => formatForDataset(item, enhancements));670 for (const item of formatted) {671 if (totalExtracted >= maxItems) break;672 await Dataset.pushData(item); lastRowTime = Date.now();673 totalExtracted++;674 }675 return formatted;676 }677 );678 }679
680 runState.urlsProcessed++;681 runState.totalResults += totalExtracted;682
683 if (D.performance) log.info('DEBUG[performance] total time=' + ((Date.now() - _t0) / 1000).toFixed(1) + 's items=' + totalExtracted);684 log.info('Extracted ' + totalExtracted + ' items from ' + url);685}686
687// ── Run ──────────────────────────────────────────────────────688
689const crawlerOptions = await getCrawlerOptions(enhancements);690
691if (enhancements.reuseContext) {692 // Reuse context mode: split URLs across N tabs, each tab navigates its URLs sequentially via goto693 const tabCount = Math.min(typeof enhancements.maxTabs === 'number' ? enhancements.maxTabs : 1, urls.length);694
695 // Distribute URLs round-robin across tabs696 const tabUrls = Array.from({ length: tabCount }, () => []);697 urls.forEach((url, i) => tabUrls[i % tabCount].push(url));698
699 // Each tab gets its first URL as the crawler request, remaining URLs navigated via goto700 const crawler = new PlaywrightCrawler({701 headless: true,702 maxRequestsPerCrawl: tabCount,703 ...crawlerOptions,704 maxConcurrency: tabCount,705 minConcurrency: tabCount,706 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,707
708 preNavigationHooks: [709 async ({ page, request }) => {710 await applyResourceBlocking(page, enhancements);711 await applyPreNavigation(page, enhancements);712 // Custom script — beforeLoad timing (Node context, can modify URL)713 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {714 request.url = executeBeforeLoadScript(enhancements, request.url);715 }716 // Create per-page capturedResponses, stored on the page object717 if (needsRequestInterception) {718 page._capturedResponses = [];719 requestHandler.setupInterception(page, option, page._capturedResponses);720 }721 }722 ],723
724 async requestHandler({ page, request }) {725 const capturedResponses = page._capturedResponses || [];726 // Find which tab's URL list this request belongs to727 const tabIndex = tabUrls.findIndex(t => t[0] === request.url);728 const myUrls = tabIndex >= 0 ? tabUrls[tabIndex] : [request.url];729
730 // Process first URL (already navigated by crawler)731 await processUrl(page, request.url, capturedResponses);732
733 // Navigate remaining URLs on the same page734 for (let i = 1; i < myUrls.length; i++) {735 if (runState.aborted) break;736 try {737 const gotoWait = enhancements.waitUntil || 'networkidle';738 await page.goto(myUrls[i], { timeout: navTimeout, waitUntil: gotoWait === 'none' ? 'commit' : gotoWait });739 await processUrl(page, myUrls[i], capturedResponses);740 } catch (err) {741 log.error('Navigation failed for ' + myUrls[i] + ': ' + err.message);742 runState.urlsProcessed++;743 }744 }745 },746
747 maxRequestRetries: navRetries,748 async failedRequestHandler({ request }) {749 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);750 runState.urlsProcessed++;751 }752 });753
754 await crawler.run(tabUrls.map(t => ({ url: t[0] })));755} else {756 // Normal mode: one request per URL, crawler manages contexts757 const crawler = new PlaywrightCrawler({758 headless: true,759 maxRequestsPerCrawl: urls.length,760 navigationTimeoutSecs: typeof enhancements.navigationTimeout === 'number' ? enhancements.navigationTimeout : 30,761 ...crawlerOptions,762
763 preNavigationHooks: [764 async ({ page, request }) => {765 await applyResourceBlocking(page, enhancements);766 await applyPreNavigation(page, enhancements);767 if (enhancements.customScript && enhancements.customScriptTiming === 'beforeLoad' && enhancements.customScriptCode) {768 request.url = executeBeforeLoadScript(enhancements, request.url);769 }770 if (needsRequestInterception) {771 page._capturedResponses = [];772 requestHandler.setupInterception(page, option, page._capturedResponses);773 }774 }775 ],776
777 async requestHandler({ page, request }) {778 const capturedResponses = page._capturedResponses || [];779 await processUrl(page, request.url, capturedResponses);780 },781
782 maxRequestRetries: navRetries,783 async failedRequestHandler({ request }) {784 log.error('Request failed after ' + (navRetries + 1) + ' attempts: ' + request.url);785 runState.urlsProcessed++;786 }787 });788
789 await crawler.run(urls.map(url => ({ url })));790}791
792clearInterval(watchdog);793await Actor.exit();1window.searchField = function(obj, fieldName, parentDepth = 0, results = [], parents = []) {2 if (typeof obj !== "object" || obj === null) {3 return results;4 }5
6 for (const key in obj) {7 if (key === fieldName) {8 let ancestorIndex = parents.length - parentDepth;9 if (ancestorIndex >= 0) {10 results.push(parents[ancestorIndex] || obj);11 } else {12 results.push(obj);13 }14 }15 if (typeof obj[key] === "object" && obj[key] !== null) {16 searchField(obj[key], fieldName, parentDepth, results, [...parents, obj]);17 }18 }19 return results;20}21
22window.searchFieldMap = function(data, fieldName, parentDepth = 0) {23 const results = [];24
25 if (!(data instanceof Map)) {26 return searchField(data, fieldName, parentDepth, results);27 }28
29 for (const [, value] of data) {30 if (typeof value === "object" && value !== null) {31 if (fieldName in value) {32 results.push(value);33 }34 const nested = searchField(value, fieldName, parentDepth, []);35 results.push(...nested);36 }37 }38
39 return results;40}41
42window.getValue = function(value, fallback = "Not Available") {43 if (value === null || value === undefined || value === "") return fallback;44 return String(value);45}46
47window.formatBoolean = function(value) {48 if (typeof value === "boolean") return value ? "Yes" : "No";49 return "Not Available";50}51
52window.joinList = function(items, separator = " || ") {53 if (!Array.isArray(items)) return "Not Available";54 const filtered = items.filter(Boolean);55 return filtered.length > 0 ? filtered.join(separator) : "Not Available";56}57
58window.moduleExport = function(exports) {59 if (typeof module !== "undefined" && module.exports) {60 module.exports = exports;61 } else {62 Object.assign(window, exports);63 }64}65
66window.userCache = new Map();67window.postDetailsCache = new Map();68
69window.getCachedUser = function(user) {70 if (!user) return null;71
72 const userId = user.pk?.toString() || user.id?.toString();73 if (!userId) return user;74
75 if (user.username) {76 userCache.set(userId, user);77 return user;78 }79
80 if (userCache.has(userId)) {81 return { ...user, ...userCache.get(userId) };82 }83
84 return user;85}86
87window.resolveRefs = function(obj, map, visited = new Set()) {88 if (typeof obj !== "object" || obj === null) {89 return obj;90 }91
92 if (!(map instanceof Map)) {93 return obj;94 }95
96 if (obj.__ref) {97 const refKey = obj.__ref;98 if (visited.has(refKey)) return obj;99 visited.add(refKey);100
101 if (map.has(refKey)) {102 const resolved = map.get(refKey);103 return resolveRefs(resolved, map, visited);104 }105 return obj;106 }107
108 if (obj.__refs && Array.isArray(obj.__refs)) {109 return obj.__refs.map(refKey => {110 if (visited.has(refKey)) return { __ref: refKey };111 visited.add(refKey);112
113 if (map.has(refKey)) {114 const resolved = map.get(refKey);115 return resolveRefs(resolved, map, visited);116 }117 return { __ref: refKey };118 });119 }120
121 if (obj.__id) {122 const refKey = obj.__id;123 if (!visited.has(refKey) && map.has(refKey)) {124 visited.add(refKey);125 const lookedUp = map.get(refKey);126 const merged = { ...obj, ...lookedUp };127 const resolved = {};128 for (const key in merged) {129 resolved[key] = resolveRefs(merged[key], map, visited);130 }131 return resolved;132 }133 }134
135 if (Array.isArray(obj)) {136 return obj.map(item => resolveRefs(item, map, visited));137 }138
139 const resolved = {};140 for (const key in obj) {141 resolved[key] = resolveRefs(obj[key], map, visited);142 }143 return resolved;144}145
146window.getPostType = function(mediaType) {147 switch (mediaType) {148 case 1: return "Photo";149 case 2: return "Video";150 case 8: return "Carousel";151 default: return null;152 }153}154
155window.getSavedPostType = function(item) {156 if (item.isSidecar) return "Carousel";157 if (item.isVideo) return "Video";158 return "Photo";159}160
161window.getImageUrl = function(item) {162 const candidates = item.image_versions2?.candidates;163 if (Array.isArray(candidates) && candidates[0]?.url) {164 return candidates[0].url;165 }166
167 const resolverData = item["$r:client__srcSet"]?.__resolverSnapshot?.data;168 if (resolverData?.image_versions2?.candidates?.[0]?.url) {169 return resolverData.image_versions2.candidates[0].url;170 }171
172 return null;173}174
175window.getCurrentInstagramUrl = function() {176 if (typeof location === "undefined") return "";177 return location.href || "";178}179
180window.isInstagramReelsUrl = function() {181 const currentUrl = getCurrentInstagramUrl();182 return currentUrl.includes("instagram.com/reels/") || currentUrl.includes("instagram.com/reel/");183}184
185window.getCurrentInstagramReelsCode = function() {186 if (!isInstagramReelsUrl()) return null;187 return getInstagramCodeFromUrl(getCurrentInstagramUrl());188}189
190window.getInstagramCodeFromUrl = function(url) {191 if (!url) return null;192 try {193 const parsedUrl = new URL(url);194 const parts = parsedUrl.pathname.split("/").filter(Boolean);195 return ["p", "reel", "reels"].includes(parts[0]) && parts[1] ? parts[1] : null;196 } catch (error) {197 const match = url.match(/instagram\.com\/(?:p|reel|reels)\/([^/?#]+)/);198 return match ? match[1] : null;199 }200}201
202window.cacheUsersFromData = function(data) {203 if (!data || typeof data !== "object") return;204 let users = [];205 if (data instanceof Map) {206 users = searchFieldMap(data, "full_name");207 users = users.map(item => resolveRefs(item, data));208 } else {209 users = searchField(data, "full_name");210 }211 for (const user of users) {212 if (user && user.username) {213 getCachedUser(user);214 }215 }216}217
218window.getPostDetailValueCount = function(item) {219 return [220 item.caption,221 item.likeCount,222 item.commentCount,223 item.viewCount,224 item.user?.username,225 item.user?.full_name,226 item.user?.profile_pic_url,227 item.imageUrl,228 item.videoUrl,229 item.postedAt,230 item.duration,231 item.hasAudio,232 item.audioTitle,233 item.audioArtist,234 item.isPaidPartnership,235 item.commentsDisabled,236 item.accessibilityCaption237 ].filter(value => value !== null && value !== undefined && value !== "").length;238}239
240window.mergePostDetails = function(existing, incoming) {241 if (!existing) return incoming;242 const preferred = getPostDetailValueCount(incoming) > getPostDetailValueCount(existing) ? incoming : existing;243 const fallback = preferred === incoming ? existing : incoming;244 return {245 ...fallback,246 ...preferred,247 user: {248 ...(fallback.user || {}),249 ...(preferred.user || {})250 },251 raw: preferred.raw || fallback.raw252 };253}254
255window.getPostDetailCandidates = function(data) {256 if (!data || typeof data !== "object") return [];257 let candidates = [];258 if (data instanceof Map) {259 candidates = [260 ...searchFieldMap(data, "coauthor_producers"),261 ...searchFieldMap(data, "code")262 ].map(item => resolveRefs(item, data));263 } else {264 candidates = [265 ...searchField(data, "coauthor_producers"),266 ...searchField(data, "code")267 ];268 }269
270 return candidates.filter(item => {271 if (!item || typeof item !== "object") return false;272 const key = item.code || item.pk?.toString() || item.id?.toString();273 return Boolean(key);274 });275}276
277// ── POSTS ──278
279window.isReelPostDetailItem = function(item) {280 return item?.code && (item.product_type === "clips" || item.media_type === 2 || item.clips_metadata);281}282
283window.preparePostDetailItem = function(item) {284 const postId = item.pk?.toString() || item.id?.toString();285 if (!postId) return null;286
287 const user = getCachedUser(item.user || item.owner || item.caption?.user);288 const caption = item.caption;289 const captionText = typeof caption === "string" ? caption : caption?.text;290 const isReel = item.product_type === "clips" || item.media_type === 2;291 const isCarousel = item.media_type === 8 || item.carousel_media_count > 0;292
293 let imageUrl = item.image_versions2?.candidates?.[0]?.url294 || item.carousel_media?.[0]?.image_versions2?.candidates?.[0]?.url;295
296 let videoUrl = item.video_versions?.[0]?.url297 || item.carousel_media?.[0]?.video_versions?.[0]?.url;298
299 let taggedUsers = null;300 if (item.usertags?.in?.length > 0) {301 taggedUsers = item.usertags.in.map(tag => tag.user?.username).filter(Boolean).join(", ");302 }303
304 return {305 id: postId,306 code: item.code,307 type: getPostType(item.media_type),308 caption: captionText,309 likeCount: item.like_count?.toString(),310 commentCount: item.comment_count?.toString(),311 viewCount: item.view_count?.toString() || item.play_count?.toString(),312 user,313 url: item.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${item.code}/` : null,314 imageUrl,315 videoUrl,316 coverImage: item.image_versions2?.candidates?.[0]?.url,317 postedAt: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,318 location: item.location?.name,319 duration: item.video_duration?.toString(),320 hasAudio: item.has_audio,321 audioTitle: item.clips_metadata?.original_sound_info?.original_audio_title,322 audioArtist: item.clips_metadata?.original_sound_info?.ig_artist?.username,323 taggedUsers,324 isCarousel,325 carouselCount: item.carousel_media_count?.toString() || item.carousel_media?.length?.toString(),326 originalWidth: item.original_width?.toString(),327 originalHeight: item.original_height?.toString(),328 isPaidPartnership: item.is_paid_partnership,329 commentsDisabled: item.comments_disabled,330 likesDisabled: item.like_and_view_counts_disabled,331 accessibilityCaption: item.accessibility_caption,332 raw: item333 };334}335
336window.cachePostDetailItem = function(item) {337 if (!isReelPostDetailItem(item)) return;338 const prepared = preparePostDetailItem(item);339 if (!prepared) return;340 const cached = postDetailsCache.get(item.code);341 if (!cached) {342 postDetailsCache.set(item.code, item);343 return;344 }345 const cachedPrepared = preparePostDetailItem(cached);346 const merged = mergePostDetails(cachedPrepared, prepared);347 postDetailsCache.set(item.code, merged.raw);348}349
350window.preparePostsFromRequest = function(data) {351 const results = [];352 const seen = new Set();353
354 for (const item of data) {355 let posts = searchField(item, "media_notes");356 if (posts.length === 0) {357 posts = searchField(item, "clips_tab_pinned_user_ids");358 }359
360 for (const post of posts) {361 const postId = post.id?.toString() || post.pk?.toString();362 if (!postId || seen.has(postId)) continue;363
364 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {365 if (post.is_paid_partnership !== false) continue;366 }367
368 seen.add(postId);369 const user = getCachedUser(post.user);370 const isReel = post.product_type === "clips" || post.media_type === 2;371
372 results.push({373 id: postId,374 code: post.code,375 user,376 type: getPostType(post.media_type),377 text: post.caption?.text,378 image: getImageUrl(post),379 video: post.video_versions?.[0]?.url,380 likes: post.like_count?.toString(),381 commentsCount: post.comment_count?.toString(),382 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,383 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,384 commentsDisabled: post.comments_disabled,385 accessibilityCaption: post.accessibility_caption,386 raw: post387 });388 }389 }390
391 return results;392}393
394window.preparePostsFromDom = function(data) {395 const results = [];396 const seen = new Set();397
398 for (const kv of data) {399 const parsed = typeof kv === "string" ? JSON.parse(kv) : kv;400 let posts = searchField(parsed, "media_notes");401 if (posts.length === 0) {402 posts = searchField(parsed, "clips_tab_pinned_user_ids");403 }404
405 for (const post of posts) {406 const postId = post.id?.toString() || post.pk?.toString();407 if (!postId || seen.has(postId)) continue;408
409 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {410 if (post.is_paid_partnership !== false) continue;411 }412
413 seen.add(postId);414 const user = getCachedUser(post.user);415 const isReel = post.product_type === "clips" || post.media_type === 2;416
417 results.push({418 id: postId,419 code: post.code,420 user,421 type: getPostType(post.media_type),422 text: post.caption?.text,423 image: getImageUrl(post),424 video: post.video_versions?.[0]?.url,425 likes: post.like_count?.toString(),426 commentsCount: post.comment_count?.toString(),427 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,428 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,429 commentsDisabled: post.comments_disabled,430 accessibilityCaption: post.accessibility_caption,431 raw: post432 });433 }434 }435
436 return results;437}438
439window.preparePostsFromReact = function(data) {440 const results = [];441 const seen = new Set();442
443 for (const item of data) {444 if (item && "isFetching" in item && Array.isArray(item.posts)) {445 for (const post of item.posts) {446 const postId = post.id?.toString();447 if (!postId || seen.has(postId)) continue;448 if (post.isSponsored) continue;449
450 seen.add(postId);451 const owner = post.owner;452 const isReel = post.productType === "clips" || post.isVideo;453
454 results.push({455 id: postId,456 code: post.code,457 user: owner ? {458 username: owner.username,459 full_name: owner.fullName,460 profile_pic_url: owner.profilePictureUrl,461 is_verified: owner.isVerified462 } : null,463 type: getSavedPostType(post),464 text: post.caption,465 image: post.src || post.displayResources?.[0]?.src,466 video: post.videoResources?.[0]?.src,467 likes: post.numLikes?.toString(),468 commentsCount: null,469 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,470 date: post.postedAt ? new Date(post.postedAt * 1000).toISOString() : null,471 commentsDisabled: post.commentsDisabled,472 accessibilityCaption: post.accessibilityCaption,473 raw: post474 });475 }476 continue;477 }478
479 if (item instanceof Map) {480 cacheUsersFromData(item);481 let posts = searchFieldMap(item, "sponsor_tags");482 if (posts.length === 0) {483 posts = searchFieldMap(item, "clips_tab_pinned_user_ids");484 }485 posts = posts.map(p => resolveRefs(p, item));486
487 for (const post of posts) {488 const postId = post.id?.toString() || post.pk?.toString();489 if (!postId || seen.has(postId)) continue;490
491 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {492 if (post.is_paid_partnership !== false) continue;493 }494
495 seen.add(postId);496 const user = getCachedUser(post.user);497 const isReel = post.product_type === "clips" || post.media_type === 2;498
499 results.push({500 id: postId,501 code: post.code,502 user,503 type: getPostType(post.media_type),504 text: post.caption?.text,505 image: getImageUrl(post),506 video: post.video_versions?.[0]?.url,507 likes: post.like_count?.toString(),508 commentsCount: post.comment_count?.toString(),509 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,510 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,511 commentsDisabled: post.comments_disabled,512 accessibilityCaption: post.accessibility_caption,513 raw: post514 });515 }516 continue;517 }518
519 const edges = item?.data?.xdt_api__v1__feed__user_timeline_graphql_connection?.edges;520 if (edges) {521 for (const edge of edges) {522 const post = edge.node;523 const postId = post.id?.toString() || post.pk?.toString();524 if (!postId || seen.has(postId)) continue;525
526 seen.add(postId);527 const isReel = post.product_type === "clips" || post.media_type === 2;528
529 results.push({530 id: postId,531 code: post.code,532 user: post.user,533 type: getPostType(post.media_type),534 text: post.caption?.text,535 image: post.image_versions2?.candidates?.[0]?.url,536 video: post.video_versions?.[0]?.url,537 likes: post.like_count?.toString(),538 commentsCount: post.comment_count?.toString(),539 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,540 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,541 commentsDisabled: post.comments_disabled,542 accessibilityCaption: post.accessibility_caption,543 raw: edge544 });545 }546 continue;547 }548
549 cacheUsersFromData(item);550 let posts = searchField(item, "media_notes");551 if (posts.length === 0) {552 posts = searchField(item, "clips_tab_pinned_user_ids");553 }554
555 for (const post of posts) {556 const postId = post.id?.toString() || post.pk?.toString();557 if (!postId || seen.has(postId)) continue;558
559 if (!("clips_tab_pinned_user_ids" in post) && ("sponsor_tags" in post)) {560 if (post.is_paid_partnership !== false) continue;561 }562
563 seen.add(postId);564 const user = getCachedUser(post.user);565 const isReel = post.product_type === "clips" || post.media_type === 2;566
567 results.push({568 id: postId,569 code: post.code,570 user,571 type: getPostType(post.media_type),572 text: post.caption?.text,573 image: getImageUrl(post),574 video: post.video_versions?.[0]?.url,575 likes: post.like_count?.toString(),576 commentsCount: post.comment_count?.toString(),577 url: post.code ? `https://www.instagram.com/${isReel ? "reel" : "p"}/${post.code}/` : null,578 date: post.taken_at ? new Date(post.taken_at * 1000).toISOString() : null,579 commentsDisabled: post.comments_disabled,580 accessibilityCaption: post.accessibility_caption,581 raw: post582 });583 }584 }585
586 return results;587}588
589window.formatPosts = function(prepared) {590 return prepared.map(item => ({591 id: item.id,592 id2: item.code,593 highlight: { id2: item.code },594 raw: item.raw,595 formatted: {596 "id": item.id,597 "Post Author": item.user?.username || "Not Available",598 "Post Author Full Name": item.user?.full_name || item.user?.fullName || "Not Available",599 "Post Author Image": item.user?.profile_pic_url || item.user?.profilePictureUrl || "Not Available",600 "Post Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",601 "Post Author Is Verified": formatBoolean(item.user?.is_verified ?? item.user?.isVerified),602 "Post Type": item.type || "Not Available",603 "Post Text": item.text || "Not Available",604 "Post Image": item.image || "Not Available",605 "Post Video": item.video || "Not Available",606 "Post Likes": item.likes || "Not Available",607 "Post Comments Count": item.commentsCount || "Not Available",608 "Post URL": item.url || "Not Available",609 "Post Date": item.date || "Not Available",610 "Is Comments Disabled": formatBoolean(item.commentsDisabled),611 "Post Accessibility Caption": item.accessibilityCaption || "Not Available"612 }613 }));614}615
616window.posts = function(rawData) {617 try {618 if (isInstagramReelsUrl()) return [];619
620 let prepared = [];621 const seen = new Set();622
623 for (const entry of rawData) {624 let batch = [];625 if (entry.metadata.method === "request") {626 batch = preparePostsFromRequest(entry.data);627 } else if (entry.metadata.method === "dom") {628 batch = preparePostsFromDom(entry.data);629 } else if (entry.metadata.method === "react") {630 batch = preparePostsFromReact(entry.data);631 }632 for (const item of batch) {633 if (!seen.has(item.id)) {634 seen.add(item.id);635 prepared.push(item);636 }637 }638 }639
640 return formatPosts(prepared);641 } catch (error) {642 return [];643 }644}645
646// ── STORIES ──647
648window.prepareStories = function(rawData) {649 const results = [];650 const seen = new Set();651 const userStore = new Map();652
653 for (const entry of rawData) {654 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];655 for (const data of dataItems) {656 const stories = searchField(data, "story_cta");657 searchField(data, "profile_pic_url").forEach(user => {658 userStore.set(user.id, user);659 });660
661 for (const item of stories) {662 const storyId = item.pk?.toString() || item.id?.toString();663 if (!storyId || seen.has(storyId)) continue;664 seen.add(storyId);665
666 let user = item.user;667 if (user && !user.profile_pic_url && userStore.get(user.id)) {668 user = userStore.get(user.id);669 }670
671 results.push({672 id: storyId,673 type: getPostType(item.media_type),674 image: item.image_versions2?.candidates?.[0]?.url,675 video: item.video_versions?.[0]?.url,676 link: item.pk && user?.username ? `https://www.instagram.com/stories/${user.username}/${item.pk}/` : null,677 date: item.taken_at ? new Date(item.taken_at * 1000).toISOString() : null,678 expiringAt: item.expiring_at ? new Date(item.expiring_at * 1000).toISOString() : null,679 accessibilityCaption: item.accessibility_caption,680 user,681 bloksFullName: item.story_bloks_stickers?.[0]?.bloks_sticker?.sticker_data?.ig_mention?.full_name,682 raw: item683 });684 }685 }686 }687
688 return results;689}690
691window.formatStories = function(prepared) {692 return prepared.map(item => ({693 id: item.id,694 highlight: { id: item.id },695 raw: item.raw,696 formatted: {697 "id": item.id,698 "Story Type": item.type || "Not Available",699 "Story Image": item.image || "Not Available",700 "Story Video": item.video || "Not Available",701 "Story Link": item.link || "Not Available",702 "Story Date": item.date || "Not Available",703 "Story Expiring At": item.expiringAt || "Not Available",704 "Story Accessibility Caption": item.accessibilityCaption || "Not Available",705 "Author Username": item.user?.username || "Not Available",706 "Author Full Name": item.user?.full_name || item.bloksFullName || "Not Available",707 "Author Image": item.user?.profile_pic_url || "Not Available",708 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",709 "Author Is Verified": formatBoolean(item.user?.is_verified)710 }711 }));712}713
714window.stories = function(rawData) {715 try {716 const prepared = prepareStories(rawData);717 return formatStories(prepared);718 } catch (error) {719 return [];720 }721}722
723// ── POST DETAILS ──724
725window.preparePostDetails = function(rawData) {726 const byPostKey = new Map();727 const reelsCode = getCurrentInstagramReelsCode();728 const currentUrl = getCurrentInstagramUrl();729
730 for (const entry of rawData) {731 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];732 const entryUrl = entry.metadata?.url || entry.metadata?.href || entry.url || entry.sourceUrl;733 const entryCode = getInstagramCodeFromUrl(entryUrl);734
735 for (const data of dataItems) {736 cacheUsersFromData(data);737 }738
739 for (const data of dataItems) {740 const items = getPostDetailCandidates(data);741
742 for (const item of items) {743 cachePostDetailItem(item);744 const prepared = preparePostDetailItem(item);745 if (!prepared) continue;746
747 const targetCode = entryCode || reelsCode;748 if (targetCode) {749 if (item.code !== targetCode) continue;750 } else if (!item.code || !currentUrl.includes(item.code)) {751 continue;752 }753 const postKey = item.code || prepared.id;754 byPostKey.set(postKey, mergePostDetails(byPostKey.get(postKey), prepared));755 }756 }757 }758
759 if (reelsCode && postDetailsCache.has(reelsCode)) {760 console.log("Instagram postDetails cache check", {761 currentUrl,762 reelsCode,763 cacheSize: postDetailsCache.size,764 cacheKeys: Array.from(postDetailsCache.keys()),765 cacheHit: true766 });767 const cachedPrepared = preparePostDetailItem(postDetailsCache.get(reelsCode));768 if (cachedPrepared) {769 byPostKey.set(reelsCode, mergePostDetails(byPostKey.get(reelsCode), cachedPrepared));770 }771 } else if (reelsCode) {772 console.log("Instagram postDetails cache check", {773 currentUrl,774 reelsCode,775 cacheSize: postDetailsCache.size,776 cacheKeys: Array.from(postDetailsCache.keys()),777 cacheHit: false778 });779 }780
781 return Array.from(byPostKey.values());782}783
784window.formatPostDetails = function(prepared) {785 return prepared.map(item => ({786 id: item.id,787 id2: item.code,788 highlight: { id: item.id },789 raw: item.raw,790 formatted: {791 "id": item.id,792 "Post Type": item.type || "Not Available",793 "Caption": item.caption || "Not Available",794 "Like Count": item.likeCount || "Not Available",795 "Comment Count": item.commentCount || "Not Available",796 "View Count": item.viewCount || "Not Available",797 "Author Username": item.user?.username || "Not Available",798 "Author Full Name": item.user?.full_name || "Not Available",799 "Author ID": item.user?.pk?.toString() || item.user?.id?.toString() || "Not Available",800 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",801 "Author Is Verified": formatBoolean(item.user?.is_verified),802 "Author Profile Pic": item.user?.profile_pic_url || item.user?.hd_profile_pic_url_info?.url || "Not Available",803 "Post URL": item.url || "Not Available",804 "Image URL": item.imageUrl || "Not Available",805 "Video URL": item.videoUrl || "Not Available",806 "Cover Image": item.coverImage || "Not Available",807 "Posted At": item.postedAt || "Not Available",808 "Location": item.location || "Not Available",809 "Duration": item.duration || "Not Available",810 "Has Audio": formatBoolean(item.hasAudio),811 "Audio Title": item.audioTitle || "Not Available",812 "Audio Artist": item.audioArtist || "Not Available",813 "Tagged Users": item.taggedUsers || "Not Available",814 "Is Carousel": formatBoolean(item.isCarousel),815 "Carousel Count": item.carouselCount || "Not Available",816 "Original Width": item.originalWidth || "Not Available",817 "Original Height": item.originalHeight || "Not Available",818 "Is Paid Partnership": formatBoolean(item.isPaidPartnership),819 "Comments Disabled": formatBoolean(item.commentsDisabled),820 "Likes Disabled": formatBoolean(item.likesDisabled),821 "Accessibility Caption": item.accessibilityCaption || "Not Available"822 }823 }));824}825
826window.postDetails = function(rawData) {827 console.log(rawData)828 try {829 const prepared = preparePostDetails(rawData);830 return formatPostDetails(prepared);831 } catch (error) {832 return [];833 }834}835
836// ── COMMENTS ──837
838window.prepareComments = function(rawData) {839 const results = [];840 const seen = new Set();841
842 for (const entry of rawData) {843 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];844 for (const data of dataItems) {845 const comments = searchField(data, "comment_like_count");846
847 for (const item of comments) {848 const commentId = item.pk?.toString() || item.id?.toString();849 if (!commentId || seen.has(commentId)) continue;850 seen.add(commentId);851
852 const commentDate = item.created_at ? new Date(item.created_at * 1000).toISOString() : null;853
854 results.push({855 id: commentId,856 date: commentDate,857 text: item.text,858 media: item.giphy_media_info?.first_party_cdn_proxied_images?.fixed_height?.url,859 likes: item.comment_like_count?.toString(),860 user: item.user,861 isReply: !!item.parent_comment_id,862 parentCommentId: item.parent_comment_id?.toString(),863 childCommentCount: item.child_comment_count,864 raw: item865 });866 }867 }868 }869
870 return results;871}872
873window.formatComments = function(prepared) {874 return prepared.map(item => ({875 id: item.id,876 id2: item.date,877 highlight: { id2: item.date },878 raw: item.raw,879 formatted: {880 "id": item.id,881 "Comment Text": item.text || "Not Available",882 "Comment Media": item.media || "Not Available",883 "Comment Date": item.date || "Not Available",884 "Comment Likes": item.likes || "Not Available",885 "Author Username": item.user?.username || "Not Available",886 "Author ID": item.user?.pk || item.user?.id || "Not Available",887 "Author URL": item.user?.username ? `https://www.instagram.com/${item.user.username}/` : "Not Available",888 "Author Is Verified": formatBoolean(item.user?.is_verified),889 "Is Reply": formatBoolean(item.isReply),890 "Parent Comment ID": item.parentCommentId || "Not Available",891 "Has Replies": formatBoolean(item.childCommentCount > 0),892 "Reply Count": item.childCommentCount?.toString() || "Not Available"893 }894 }));895}896
897window.comments = function(rawData) {898 try {899 const prepared = prepareComments(rawData);900 return formatComments(prepared);901 } catch (error) {902 return [];903 }904}905
906// ── PROFILE ──907
908window.prepareProfile = function(rawData) {909 const results = [];910 const seen = new Set();911
912 for (const entry of rawData) {913 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];914
915 for (const data of dataItems) {916 let users = [];917
918 if (data instanceof Map) {919 users = searchFieldMap(data, "biography");920 users = users.map(user => resolveRefs(user, data));921 } else {922 users = searchField(data, "biography");923 }924
925 for (const item of users) {926 const userId = item.pk?.toString() || item.id?.toString();927 if (!userId || seen.has(userId)) continue;928 seen.add(userId);929
930 results.push({931 id: userId,932 username: item.username,933 fullName: item.full_name,934 biography: item.biography,935 externalUrl: item.external_url,936 followerCount: item.follower_count?.toString(),937 followingCount: item.following_count?.toString(),938 mediaCount: item.media_count?.toString(),939 isVerified: item.is_verified,940 isPrivate: item.is_private,941 isBusiness: item.is_business,942 category: item.category,943 profilePicUrl: item.hd_profile_pic_url_info?.url || item.profile_pic_url,944 bioLinks: item.bio_links?.map(link => link.url),945 raw: item946 });947 }948 }949 }950
951 return results;952}953
954window.formatProfile = function(prepared) {955 return prepared.map(item => ({956 id: item.id,957 id2: item.username,958 highlight: { id: item.id },959 raw: item.raw,960 formatted: {961 "id": item.id,962 "Username": item.username || "Not Available",963 "Full Name": item.fullName || "Not Available",964 "Biography": item.biography || "Not Available",965 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",966 "External URL": item.externalUrl || "Not Available",967 "Follower Count": item.followerCount || "Not Available",968 "Following Count": item.followingCount || "Not Available",969 "Post Count": item.mediaCount || "Not Available",970 "Is Verified": formatBoolean(item.isVerified),971 "Is Private": formatBoolean(item.isPrivate),972 "Is Business": formatBoolean(item.isBusiness),973 "Category": item.category || "Not Available",974 "Profile Picture URL": item.profilePicUrl || "Not Available",975 "All Bio Links": joinList(item.bioLinks)976 }977 }));978}979
980window.profile = function(rawData) {981 try {982 const prepared = prepareProfile(rawData);983 return formatProfile(prepared);984 } catch (error) {985 return [];986 }987}988
989// ── FOLLOWERS / FOLLOWING ──990
991window.prepareUsers = function(rawData) {992 const results = [];993 const seen = new Set();994
995 for (const entry of rawData) {996 const dataItems = Array.isArray(entry.data) ? entry.data : [entry.data];997 for (const data of dataItems) {998 const users = searchField(data, "profile_pic_url");999
1000 for (const item of users) {1001 const userId = item.pk?.toString() || item.id?.toString();1002 if (!userId || seen.has(userId)) continue;1003 seen.add(userId);1004
1005 results.push({1006 id: userId,1007 username: item.username,1008 fullName: item.full_name,1009 isPrivate: item.is_private,1010 isVerified: item.is_verified,1011 profilePicUrl: item.profile_pic_url,1012 hasAnonPic: item.has_anonymous_profile_picture,1013 latestStoryTime: item.latest_reel_media ? new Date(item.latest_reel_media * 1000).toISOString() : null,1014 raw: item1015 });1016 }1017 }1018 }1019
1020 return results;1021}1022
1023window.formatUsers = function(prepared) {1024 return prepared.map(item => ({1025 id: item.id,1026 id2: item.username,1027 highlight: { id2: item.username },1028 raw: item.raw,1029 formatted: {1030 "id": item.id,1031 "Username": item.username || "Not Available",1032 "Full Name": item.fullName || "Not Available",1033 "Profile URL": item.username ? `https://www.instagram.com/${item.username}/` : "Not Available",1034 "Is Private": formatBoolean(item.isPrivate),1035 "Is Verified": formatBoolean(item.isVerified),1036 "Profile Picture URL": item.profilePicUrl || "Not Available",1037 "Has Anonymous Profile Picture": formatBoolean(item.hasAnonPic),1038 "Latest Story Time": item.latestStoryTime || "Not Available"1039 }1040 }));1041}1042
1043window.followers = function(rawData) {1044 try {1045 const prepared = prepareUsers(rawData);1046 return formatUsers(prepared);1047 } catch (error) {1048 return [];1049 }1050}1051
1052window.following = function(rawData) {1053 try {1054 const prepared = prepareUsers(rawData);1055 return formatUsers(prepared);1056 } catch (error) {1057 return [];1058 }1059}{ "title": "Instagram Comments of a Post Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "urls": { "title": "URLs", "type": "array", "items": { "type": "string" }, "description": "URLs to scrape. Must match: instagram.com", "editor": "stringList", "default": [ "https://www.instagram.com/p/DWZitMjgHp1/" ] }, "maxItems": { "title": "Max items", "type": "integer", "description": "Maximum number of items to extract per URL", "default": 100, "minimum": 1 }, "enablePagination": { "title": "Paginate", "type": "boolean", "description": "Load more results beyond the first page.", "default": true }, "maxPages": { "title": "Max pagination steps", "type": "integer", "description": "Maximum number of pagination actions (pages, scrolls, or loads)", "default": 1, "minimum": 1 } }, "required": [ "urls" ]}{ "actorSpecification": 1, "fields": { "Comment Text": { "type": "string", "title": "Comment Text" }, "Comment Media": { "type": "string", "title": "Comment Media" }, "Comment Date": { "type": "string", "title": "Comment Date" }, "Comment Likes": { "type": "string", "title": "Comment Likes" }, "Author Username": { "type": "string", "title": "Author Username" }, "Author ID": { "type": "string", "title": "Author ID" }, "Author URL": { "type": "string", "title": "Author URL" }, "Author Is Verified": { "type": "string", "title": "Author Is Verified" }, "Is Reply": { "type": "string", "title": "Is Reply" }, "Parent Comment ID": { "type": "string", "title": "Parent Comment ID" }, "Has Replies": { "type": "string", "title": "Has Replies" }, "Reply Count": { "type": "string", "title": "Reply Count" } }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "Comment Text", "Comment Media", "Comment Date", "Comment Likes", "Author Username", "Author ID", "Author URL", "Author Is Verified", "Is Reply", "Parent Comment ID", "Has Replies", "Reply Count" ] }, "display": { "component": "table", "properties": {} } } }}