Instagram Post & Reel Scraper
Under maintenancePricing
from $1.00 / 1,000 posts
Scrapes public Instagram profile posts and reels, or looks up a single post/reel by URL. No login required.
Pricing
from $1.00 / 1,000 posts
Rating
0.0
(0)
Developer
Saketh Balijepalli
Maintained by CommunityActor stats
0
Bookmarked
3
Total users
1
Monthly active users
8 days ago
Last modified
Categories
Share
node_modulesdiststorage.git.scratch.envtestcoveragenode_modules/dist/storage/apify_storage/.env.DS_Store1## Agent skills2
3### Issue tracker4
5Issues and specs live as local markdown files under `.scratch/<feature-slug>/`. See `docs/agents/issue-tracker.md`.6
7### Triage labels8
9Default five canonical roles (needs-triage, needs-info, ready-for-agent, ready-for-human, wontfix). See `docs/agents/triage-labels.md`.10
11### Domain docs12
13Single-context — `CONTEXT.md` + `docs/adr/` at the repo root. See `docs/agents/domain.md`.{ "name": "instagram-apify-scraper", "version": "0.1.0", "type": "module", "private": true, "description": "Apify Actor that scrapes public Instagram posts and reels.", "main": "dist/main.js", "scripts": { "build": "tsc", "start": "npm run build && node dist/main.js", "typecheck": "tsc --noEmit -p tsconfig.typecheck.json", "test": "vitest run" }, "dependencies": { "apify": "^3.2.6", "crawlee": "^3.11.5", "playwright": "^1.47.0" }, "devDependencies": { "@crawlee/memory-storage": "^3.11.5", "@types/node": "^20.14.0", "typescript": "^5.5.4", "vitest": "^2.0.5" }}{ "compilerOptions": { "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", "lib": ["ES2022"], "outDir": "dist", "rootDir": "src", "strict": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "declaration": false, "sourceMap": false }, "include": ["src/**/*.ts"], "exclude": ["node_modules", "dist", "test", "storage"]}{ "extends": "./tsconfig.json", "compilerOptions": { "rootDir": "." }, "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["node_modules", "dist"]}FROM apify/actor-node-playwright-chrome:20 AS builder
COPY package*.json ./RUN npm install --include=dev --audit=false
COPY . ./RUN npm run build
FROM apify/actor-node-playwright-chrome:20
COPY package*.json ./RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional --audit=false \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
COPY /home/myuser/dist ./distCOPY . ./
CMD ["node", "dist/main.js"]{ "actorSpecification": 1, "name": "instagram-post-reel-scraper", "title": "Instagram Post & Reel Scraper", "description": "Scrapes public Instagram profile posts and reels, or looks up a single post/reel by URL. No login required.", "version": "0.1", "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": "./dataset_schema.json" }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "shortcode": { "type": "string", "description": "Instagram's short code for the item, taken from its URL." }, "ownerUsername": { "type": "string", "description": "Null if Instagram didn't include it in the response." }, "contentType": { "type": "string", "enum": ["post", "reel"], "description": "Content type" }, "caption": { "type": "string", "description": "Caption" }, "timestamp": { "type": "number", "description": "Unix timestamp, in seconds, of when the item was posted." }, "likeCount": { "type": "number", "description": "Like count" }, "commentCount": { "type": "number", "description": "Comment count" }, "mediaType": { "type": "string", "enum": ["image", "video", "carousel"], "description": "Media type" }, "mediaUrl": { "type": "string", "description": "A single representative URL: the video file for videos, or the cover/first image for images and carousels." } }, "required": ["shortcode", "contentType", "timestamp", "mediaType"] }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "shortcode", "ownerUsername", "contentType", "caption", "timestamp", "likeCount", "commentCount", "mediaType", "mediaUrl" ] }, "display": { "component": "table", "properties": { "shortcode": { "label": "Shortcode", "format": "text" }, "ownerUsername": { "label": "Owner", "format": "text" }, "contentType": { "label": "Type", "format": "text" }, "caption": { "label": "Caption", "format": "text" }, "timestamp": { "label": "Posted at", "format": "number" }, "likeCount": { "label": "Likes", "format": "number" }, "commentCount": { "label": "Comments", "format": "number" }, "mediaType": { "label": "Media type", "format": "text" }, "mediaUrl": { "label": "Media", "format": "link" } } } } }}{ "title": "Instagram Post & Reel Scraper input", "type": "object", "schemaVersion": 1, "properties": { "username": { "title": "Profile username", "type": "string", "editor": "textfield", "description": "Public Instagram username to scrape posts and reels from. Provide this or postOrReelUrl.", "prefill": "nasa", "default": "nasa" }, "postOrReelUrl": { "title": "Post or reel URL", "type": "string", "editor": "textfield", "description": "Direct URL to a single public Instagram post or reel to look up. Provide this or username." }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "editor": "proxy", "description": "Optional. Route requests through your own proxy (or Apify Proxy, at your own cost) to reduce the chance of being blocked. Left empty, the Actor runs proxy-less by default at no extra cost.", "prefill": { "useApifyProxy": false }, "default": { "useApifyProxy": false } } }, "required": []}{ "actorOutputSchemaVersion": 1, "title": "Instagram Post & Reel Scraper output", "properties": { "scrapedItems": { "type": "string", "title": "Scraped posts & reels", "template": "{{links.apiDefaultDatasetUrl}}/items?view=overview" } }}1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { MemoryStorage } from '@crawlee/memory-storage';4import { Actor } from 'apify';5import { describe, expect, it, vi } from 'vitest';6import { processRun } from '../src/actor.js';7import { BlockedRequestError } from '../src/instagram/challengeShell.js';8import { ItemNotFoundError, type ItemFetcher } from '../src/instagram/fetcher.js';9import { PrivateProfileError, ProfileNotFoundError, type ProfileFetcher } from '../src/instagram/profileFetcher.js';10import { InvalidInputError } from '../src/input.js';11
12async function createTestActor(): Promise<Actor> {13 const actor = new Actor({14 storageClient: new MemoryStorage({ persistStorage: false }),15 });16 await actor.init();17 return actor;18}19
20function loadFixture(name: string): unknown {21 const path = fileURLToPath(new URL(`./fixtures/${name}`, import.meta.url));22 return JSON.parse(readFileSync(path, 'utf8'));23}24
25function stubFetcher(fetchRawItem: ItemFetcher['fetchRawItem']): ItemFetcher {26 return { fetchRawItem };27}28
29function stubProfileFetcher(fetchProfileListing: ProfileFetcher['fetchProfileListing']): ProfileFetcher {30 return { fetchProfileListing };31}32
33const neverCalledFetcher = stubFetcher(async () => {34 throw new Error('fetcher should not be called for this input');35});36
37const neverCalledProfileFetcher = stubProfileFetcher(async () => {38 throw new Error('profileFetcher should not be called for this input');39});40
41describe('local Dataset storage harness', () => {42 it('can observe an item pushed to the Dataset', async () => {43 const actor = await createTestActor();44 const dataset = await actor.openDataset();45
46 await dataset.pushData({ shortcode: 'ABC123' });47
48 const { items } = await dataset.getData();49 expect(items).toEqual([{ shortcode: 'ABC123' }]);50 });51});52
53describe('processRun', () => {54 it('rejects with a clear error when neither username nor postOrReelUrl is provided', async () => {55 const actor = await createTestActor();56 const keyValueStore = await actor.openKeyValueStore();57 await keyValueStore.setValue('INPUT', {});58
59 await expect(processRun(actor, neverCalledFetcher, neverCalledProfileFetcher)).rejects.toThrow(InvalidInputError);60 });61
62 it('pushes a normalized reel item and charges the reel event, given a postOrReelUrl', async () => {63 const actor = await createTestActor();64 const keyValueStore = await actor.openKeyValueStore();65 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/reel/DbvqPafyqY-/' });66 const fetcher = stubFetcher(async () => loadFixture('reel-single.json'));67 const chargeSpy = vi.spyOn(actor, 'pushData');68
69 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).resolves.toBeUndefined();70
71 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbvqPafyqY-' }), 'reel-scraped');72 const dataset = await actor.openDataset();73 const { items } = await dataset.getData();74 expect(items).toEqual([expect.objectContaining({ shortcode: 'DbvqPafyqY-', contentType: 'reel' })]);75 });76
77 it('pushes a normalized post item and charges the post event, given a postOrReelUrl', async () => {78 const actor = await createTestActor();79 const keyValueStore = await actor.openKeyValueStore();80 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/p/DbbY9pdm6Q2/' });81 const fetcher = stubFetcher(async () => loadFixture('post-carousel.json'));82 const chargeSpy = vi.spyOn(actor, 'pushData');83
84 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).resolves.toBeUndefined();85
86 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }), 'post-scraped');87 const dataset = await actor.openDataset();88 const { items } = await dataset.getData();89 expect(items).toEqual([expect.objectContaining({ shortcode: 'DbbY9pdm6Q2', contentType: 'post' })]);90 });91
92 it('rejects with a clear error and leaves the Dataset empty when the item is not found', async () => {93 const actor = await createTestActor();94 const keyValueStore = await actor.openKeyValueStore();95 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/p/deleted000/' });96 const fetcher = stubFetcher(async () => {97 throw new ItemNotFoundError('This post or reel is not available.');98 });99
100 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).rejects.toThrow(ItemNotFoundError);101
102 const dataset = await actor.openDataset();103 const { items } = await dataset.getData();104 expect(items).toHaveLength(0);105 });106
107 it('rejects with PrivateProfileError and pushes nothing, given a private profile', async () => {108 const actor = await createTestActor();109 const keyValueStore = await actor.openKeyValueStore();110 await keyValueStore.setValue('INPUT', { username: 'a-private-account' });111 const profileFetcher = stubProfileFetcher(async () => {112 throw new PrivateProfileError('This profile is private.');113 });114
115 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).rejects.toThrow(PrivateProfileError);116
117 const dataset = await actor.openDataset();118 const { items } = await dataset.getData();119 expect(items).toHaveLength(0);120 });121
122 it('resolves with an explicit empty Dataset, given a public profile with zero items', async () => {123 const actor = await createTestActor();124 const keyValueStore = await actor.openKeyValueStore();125 await keyValueStore.setValue('INPUT', { username: 'an-empty-account' });126 const profileFetcher = stubProfileFetcher(async () => ({ items: [], limitReached: false }));127
128 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).resolves.toBeUndefined();129
130 const dataset = await actor.openDataset();131 const { items } = await dataset.getData();132 expect(items).toHaveLength(0);133 });134
135 it('fetches, normalizes, and pushes each discovered item at the correct permalink, given a populated profile', async () => {136 const actor = await createTestActor();137 const keyValueStore = await actor.openKeyValueStore();138 await keyValueStore.setValue('INPUT', { username: 'nasa' });139 const profileFetcher = stubProfileFetcher(async () => ({140 items: [141 { shortcode: 'DbvqPafyqY-', contentType: 'reel' },142 { shortcode: 'DbbY9pdm6Q2', contentType: 'post' },143 ],144 limitReached: false,145 }));146 const requestedUrls: string[] = [];147 const fetcher = stubFetcher(async (url) => {148 requestedUrls.push(url);149 return url.includes('/reel/') ? loadFixture('reel-single.json') : loadFixture('post-carousel.json');150 });151 const chargeSpy = vi.spyOn(actor, 'pushData');152
153 await expect(processRun(actor, fetcher, profileFetcher)).resolves.toBeUndefined();154
155 expect(requestedUrls).toEqual([156 'https://www.instagram.com/reel/DbvqPafyqY-/',157 'https://www.instagram.com/p/DbbY9pdm6Q2/',158 ]);159 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbvqPafyqY-' }), 'reel-scraped');160 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }), 'post-scraped');161 const dataset = await actor.openDataset();162 const { items } = await dataset.getData();163 expect(items).toHaveLength(2);164 });165
166 it('rejects with ProfileNotFoundError and pushes nothing, given an unrecognized profile', async () => {167 const actor = await createTestActor();168 const keyValueStore = await actor.openKeyValueStore();169 await keyValueStore.setValue('INPUT', { username: 'this-should-not-exist' });170 const profileFetcher = stubProfileFetcher(async () => {171 throw new ProfileNotFoundError('Could not find profile data.');172 });173
174 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).rejects.toThrow(ProfileNotFoundError);175
176 const dataset = await actor.openDataset();177 const { items } = await dataset.getData();178 expect(items).toHaveLength(0);179 });180
181 it('skips an item that fails with a content-level error and keeps scraping the rest of the profile', async () => {182 const actor = await createTestActor();183 const keyValueStore = await actor.openKeyValueStore();184 await keyValueStore.setValue('INPUT', { username: 'nasa' });185 const profileFetcher = stubProfileFetcher(async () => ({186 items: [187 { shortcode: 'DbvqPafyqY-', contentType: 'reel' },188 { shortcode: 'deleted000', contentType: 'post' },189 { shortcode: 'DbbY9pdm6Q2', contentType: 'post' },190 ],191 limitReached: false,192 }));193 const fetcher = stubFetcher(async (url) => {194 if (url.includes('deleted000')) throw new ItemNotFoundError('This post or reel is not available.');195 return url.includes('/reel/') ? loadFixture('reel-single.json') : loadFixture('post-carousel.json');196 });197
198 await expect(processRun(actor, fetcher, profileFetcher)).resolves.toBeUndefined();199
200 const dataset = await actor.openDataset();201 const { items } = await dataset.getData();202 expect(items).toEqual([203 expect.objectContaining({ shortcode: 'DbvqPafyqY-' }),204 expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }),205 ]);206 });207
208 it('reports a clear partial-results status message when the anonymous pagination limit is hit', async () => {209 const actor = await createTestActor();210 const keyValueStore = await actor.openKeyValueStore();211 await keyValueStore.setValue('INPUT', { username: 'a-very-active-account' });212 const profileFetcher = stubProfileFetcher(async () => ({ items: [], limitReached: true }));213 const statusSpy = vi.spyOn(actor, 'setStatusMessage').mockResolvedValue({} as never);214
215 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).resolves.toBeUndefined();216
217 expect(statusSpy).toHaveBeenCalledWith(expect.stringMatching(/incomplete|limit/i), expect.anything());218 });219
220 it('propagates BlockedRequestError from the profile listing fetch rather than swallowing it', async () => {221 const actor = await createTestActor();222 const keyValueStore = await actor.openKeyValueStore();223 await keyValueStore.setValue('INPUT', { username: 'nasa' });224 const profileFetcher = stubProfileFetcher(async () => {225 throw new BlockedRequestError('Instagram served an anti-bot challenge shell.');226 });227
228 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).rejects.toThrow(BlockedRequestError);229 });230
231 it('skips an item that fails with BlockedRequestError and keeps scraping the rest of the profile', async () => {232 // Unlike a BlockedRequestError from the profile *listing* fetch (tested above), a233 // per-item block is now recoverable (ticket 07): the fetcher's pool retires the234 // affected session internally, so the item is skipped rather than aborting the run.235 const actor = await createTestActor();236 const keyValueStore = await actor.openKeyValueStore();237 await keyValueStore.setValue('INPUT', { username: 'nasa' });238 const profileFetcher = stubProfileFetcher(async () => ({239 items: [240 { shortcode: 'blocked000', contentType: 'post' },241 { shortcode: 'DbvqPafyqY-', contentType: 'reel' },242 ],243 limitReached: false,244 }));245 const fetcher = stubFetcher(async (url) => {246 if (url.includes('blocked000')) throw new BlockedRequestError('Instagram served an anti-bot challenge shell.');247 return loadFixture('reel-single.json');248 });249
250 await expect(processRun(actor, fetcher, profileFetcher)).resolves.toBeUndefined();251
252 const dataset = await actor.openDataset();253 const { items } = await dataset.getData();254 expect(items).toEqual([expect.objectContaining({ shortcode: 'DbvqPafyqY-' })]);255 });256});1import { describe, expect, it } from 'vitest';2import { InvalidInputError, validateInput } from '../src/input.js';3
4describe('validateInput', () => {5 it('throws InvalidInputError when neither username nor postOrReelUrl is provided', () => {6 expect(() => validateInput({})).toThrow(InvalidInputError);7 expect(() => validateInput(null)).toThrow(InvalidInputError);8 });9
10 it('does not throw when a username is provided', () => {11 expect(() => validateInput({ username: 'natgeo' })).not.toThrow();12 });13
14 it('does not throw when a postOrReelUrl is provided', () => {15 expect(() =>16 validateInput({ postOrReelUrl: 'https://www.instagram.com/p/ABC123/' }),17 ).not.toThrow();18 });19});1import { describe, expect, it } from 'vitest';2import { createScraperRouter } from '../src/router.js';3
4describe('createScraperRouter', () => {5 it('registers a default handler that runs without throwing', async () => {6 const router = createScraperRouter();7 const defaultHandler = router.getHandler();8
9 // @ts-expect-error -- empty context stub; the placeholder handler ignores it entirely10 await expect(defaultHandler({})).resolves.toBeUndefined();11 });12});1import type { Actor } from 'apify';2import { validateInput, type ActorInput } from './input.js';3import { BlockedRequestError } from './instagram/challengeShell.js';4import { ItemNotFoundError, type ItemFetcher } from './instagram/fetcher.js';5import { normalizeRawItem, UnrecognizedItemShapeError } from './instagram/normalize.js';6import type { ProfileFetcher } from './instagram/profileFetcher.js';7import type { ContentType, ScrapedItem } from './instagram/types.js';8
9/**10 * Reads and validates the run's input, then fetches, normalizes, and pushes11 * item(s): a single item for `postOrReelUrl`, or every item discoverable on12 * a profile for `username` (which internally reuses the same per-item13 * fetch/normalize/charge pipeline once shortcodes are known). Takes an14 * already-`init()`-ed Actor instance and injected fetchers so it can be15 * exercised against an isolated in-memory storage instance and canned16 * responses in tests.17 *18 * When both inputs are given, `postOrReelUrl` takes priority and `username`19 * is ignored — see ticket 02 code review finding #7.20 */21export async function processRun(actor: Actor, fetcher: ItemFetcher, profileFetcher: ProfileFetcher): Promise<void> {22 const input = await actor.getInput<ActorInput>();23 validateInput(input);24
25 if (input.postOrReelUrl) {26 await scrapeSingleItem(actor, fetcher, input.postOrReelUrl);27 return;28 }29
30 if (input.username) {31 await scrapeProfile(actor, fetcher, profileFetcher, input.username);32 }33}34
35async function scrapeSingleItem(actor: Actor, fetcher: ItemFetcher, url: string): Promise<void> {36 const raw = await fetcher.fetchRawItem(url);37 const item = normalizeRawItem(raw);38 await pushScrapedItem(actor, item);39}40
41/**42 * Fetches every discovered item concurrently via `fetcher`, rather than one43 * Chromium launch per item in strict sequence (ticket 07's v1 limitation).44 * Actual concurrency is bounded by `fetcher`'s own pool (`BrowserPool`), not45 * here — this just fans every item's fetch out and lets calls beyond the46 * pool's size queue naturally.47 *48 * Uses `Promise.allSettled`, not `Promise.all`, deliberately: `Promise.all`49 * rejects as soon as the first unrecoverable error hits, abandoning sibling50 * tasks still mid-fetch or still queued for a pool slot — those would then51 * race against the caller's `fetcher.close()` (main.ts), and any still52 * queued would hang forever once the pool closes under them. Waiting for53 * every task to settle here means `close()` only ever runs once nothing is54 * still using the pool.55 *56 * Known consequence, not covered by any acceptance criteria: dataset items57 * now land in completion order, not listing order (a profile timeline is58 * newest-first, so this is a minor, user-visible ordering change). Pushing59 * happens inside each item's own task rather than being collected and60 * pushed at the end, since collecting-then-pushing would silently break the61 * user-confirmed partial-results guarantee (ticket 03 AC2: "stop at the62 * wall, report partial + reason") — an infra failure on a later item must63 * not discard items already successfully pushed.64 */65async function scrapeProfile(66 actor: Actor,67 fetcher: ItemFetcher,68 profileFetcher: ProfileFetcher,69 username: string,70): Promise<void> {71 const listing = await profileFetcher.fetchProfileListing(username);72
73 const results = await Promise.allSettled(74 listing.items.map(async ({ shortcode, contentType }) => {75 const raw = await fetcher.fetchRawItem(permalinkFor(shortcode, contentType));76 const item = normalizeRawItem(raw);77 await pushScrapedItem(actor, item);78 }),79 );80
81 for (const result of results) {82 if (result.status !== 'rejected') continue;83 const error = result.reason;84 // An item can become unavailable or unparseable between listing capture and85 // detail fetch, or blocked mid-scrape (its session already retired by the86 // fetcher's pool) — skip it and keep scraping the rest of the profile rather87 // than aborting the whole run (AC2: partial results, not silent/total failure).88 // Anything else (infra failures, unexpected shapes elsewhere) still propagates,89 // once every task has finished settling.90 if (91 !(error instanceof ItemNotFoundError) &&92 !(error instanceof UnrecognizedItemShapeError) &&93 !(error instanceof BlockedRequestError)94 ) {95 throw error;96 }97 }98
99 if (listing.limitReached) {100 await actor.setStatusMessage(101 "Reached Instagram's anonymous-access pagination limit before this profile's items were exhausted — results may be incomplete.",102 { isStatusMessageTerminal: true },103 );104 }105}106
107async function pushScrapedItem(actor: Actor, item: ScrapedItem): Promise<void> {108 // 'post-scraped' / 'reel-scraped' must match the event names registered109 // in the Actor's pay-per-event pricing config (ticket 05) — until that's110 // configured, pushData still succeeds but charges $0 (Apify SDK no-ops111 // charging for unregistered event names rather than throwing).112 await actor.pushData(item, `${item.contentType}-scraped`);113}114
115function permalinkFor(shortcode: string, contentType: ContentType): string {116 const segment = contentType === 'reel' ? 'reel' : 'p';117 return `https://www.instagram.com/${segment}/${shortcode}/`;118}1import type { ProxyConfigurationOptions } from 'apify';2
3export interface ActorInput {4 username?: string;5 postOrReelUrl?: string;6 proxyConfiguration?: ProxyConfigurationOptions & { useApifyProxy?: boolean };7}8
9export class InvalidInputError extends Error {}10
11export function validateInput(input: ActorInput | null): asserts input is ActorInput {12 if (!input || (!input.username && !input.postOrReelUrl)) {13 throw new InvalidInputError(14 'Provide either "username" (to scrape a profile) or "postOrReelUrl" (to look up a single post or reel).',15 );16 }17}1import { Actor } from 'apify';2import { processRun } from './actor.js';3import type { ActorInput } from './input.js';4import { BrowserItemFetcher } from './instagram/browserFetcher.js';5import { BrowserProfileFetcher } from './instagram/browserProfileFetcher.js';6import { toPlaywrightProxy, type PlaywrightProxy } from './instagram/proxy.js';7
8const actor = new Actor();9
10try {11 await actor.init();12
13 // Actor.createProxyConfiguration(undefined) does NOT mean "no proxy" — its default14 // parameter is {}, which falls through to real, auto-mode (billable) Apify Proxy on15 // the platform. v1 relied on that to enforce zero-spend: this guard only called16 // createProxyConfiguration when an explicit proxyConfiguration was present, so an17 // omitted key stayed proxy-less. v2 (ticket 08) flips the intent, not the guard:18 // input_schema.json's proxyConfiguration now carries a "default" (not "prefill" —19 // prefill is UI-only and never reaches the Actor at runtime, confirmed against20 // Apify's input-schema spec, AND already live-verified for a different property:21 // the 2026-08-31 autotest-input fix showed username's schema "default" really does22 // reach a run triggered with the key omitted). Whether the platform merges23 // "default" for an *object*-typed, proxy-editor property the same way is not24 // independently confirmed the same way yet — worth checking on the first live run25 // after this ships. If it turns out not to, the guard just silently stays26 // proxy-less (same as v1), not a crash — see the try/catch below for the failure27 // mode that IS a crash risk: an explicit or defaulted proxyConfiguration whose28 // requested group/tier the caller's account can't actually use.29 const input = await actor.getInput<ActorInput>();30 let proxyConfiguration;31 try {32 proxyConfiguration = input?.proxyConfiguration33 ? await actor.createProxyConfiguration(input.proxyConfiguration)34 : undefined;35 } catch (error) {36 // The real platform throws here (not just warns) when the caller's account can't37 // use the requested proxy group/tier — e.g. RESIDENTIAL requires a paid plan tier38 // not every Store user has. Defaulting proxy on must not turn "works proxy-less"39 // into "hard-fails for anyone without that access" — fall back to proxy-less and40 // say why, rather than crash the whole run over an unavailable proxy.41 const message = error instanceof Error ? error.message : String(error);42 await actor.setStatusMessage(`Could not set up the requested proxy (${message}) — continuing proxy-less.`);43 proxyConfiguration = undefined;44 }45 const proxyInfo = await proxyConfiguration?.newProxyInfo();46 const proxy: PlaywrightProxy | undefined = proxyInfo ? toPlaywrightProxy(proxyInfo) : undefined;47
48 const fetcher = new BrowserItemFetcher({ proxy });49 const profileFetcher = new BrowserProfileFetcher({ proxy });50
51 // BrowserItemFetcher now owns a browser pool (ticket 07) instead of52 // launching and closing per call — main.ts constructs it, so main.ts is53 // responsible for closing it, on both the success and error paths.54 try {55 await processRun(actor, fetcher, profileFetcher);56 } finally {57 await fetcher.close();58 }59 await actor.exit();60} catch (error) {61 const message = error instanceof Error ? error.message : String(error);62 await actor.exit({ statusMessage: message, exitCode: 1 });63}1import { Router, type BasicCrawlingContext, type RouterHandler } from 'crawlee';2
3/**4 * Placeholder proving Crawlee's Router wires up and compiles. `BasicCrawlingContext`5 * can't host a Playwright handler, so this is not itself the hybrid direct-API/6 * browser-automation routing point the spec describes — ticket 02 designs that7 * (most likely as two crawlers, one per context type, behind a shared dispatch8 * function) and will replace this module accordingly.9 *10 * Returns `RouterHandler`, not the base `Router` type — `RouterHandler` carries11 * the call signature that makes the result usable as a crawler's `requestHandler`12 * (or directly callable, as tests do).13 */14export function createScraperRouter(): RouterHandler<BasicCrawlingContext> {15 const router = Router.create<BasicCrawlingContext>();16
17 router.addDefaultHandler(async () => {18 // Intentionally a no-op — no scraping logic exists yet.19 });20
21 return router;22}1#!/usr/bin/env bash2#3# A wizard — walks a human through a manual procedure step by step.4# Generated by the /wizard skill.5#6# Everything above the "STAGES" marker is the wizard library: do not hand-edit7# it. Author the per-step stages below the marker.8
9set -euo pipefail10
11# ──────────────────────────────────────────────────────────────────────────12# Wizard library — delightful, consistent UX. Identical across every wizard.13# ──────────────────────────────────────────────────────────────────────────14
15if [[ -t 1 ]] && command -v tput >/dev/null 2>&1 && [[ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]]; then16 BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)17 BLUE=$(tput setaf 4); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); RED=$(tput setaf 1)18else19 BOLD=""; DIM=""; RESET=""; BLUE=""; GREEN=""; YELLOW=""; RED=""20fi21
22# Author sets this at the top of the stages section.23TOTAL_STAGES=024
25_STAGE_INDEX=026ENV_FILE="${ENV_FILE:-.env}"27WRITTEN_ENV=() # KEYs written to ENV_FILE this run28WRITTEN_SECRET=() # secret NAMEs set this run29SKIPPED=() # things we couldn't do (e.g. gh missing)30
31# _clear — wipe the terminal so only the current step is on screen. No-op when32# output isn't a terminal, so piped logs stay readable.33_clear() {34 [[ -t 1 ]] || return 035 if command -v tput >/dev/null 2>&1; then tput clear; else printf '\033[2J\033[3J\033[H'; fi36}37
38# banner "Title" — opening frame: what this wizard does.39banner() {40 _clear41 printf '\n%s%s %s%s\n' "$BOLD" "$BLUE" "$1" "$RESET"42 printf '%s %s stages%s\n\n' "$DIM" "$TOTAL_STAGES" "$RESET"43 printf '%s You drive the browser; this wizard tells you exactly what to do and\n' "$DIM"44 printf ' captures the values you copy back. Stop any time with Ctrl-C and re-run\n'45 printf ' later — it remembers values already saved.%s\n' "$RESET"46 pause "Ready to start?"47}48
49# stage "Name" — clear the screen, then announce a stage and show progress.50# Clearing keeps only the current step on screen.51stage() {52 _clear53 _STAGE_INDEX=$((_STAGE_INDEX + 1))54 printf '\n%s%s▸ Stage %s/%s · %s%s\n' \55 "$BOLD" "$BLUE" "$_STAGE_INDEX" "$TOTAL_STAGES" "$1" "$RESET"56}57
58# say "..." — a plain instruction line.59say() { printf ' %s\n' "$1"; }60# step "..." — a numbered-feeling action the human takes in the browser.61step() { printf ' %s•%s %s\n' "$BLUE" "$RESET" "$1"; }62note() { printf ' %s%s%s\n' "$DIM" "$1" "$RESET"; }63warn() { printf ' %s⚠ %s%s\n' "$YELLOW" "$1" "$RESET"; }64
65# open_url URL — open in the human's browser, cross-platform incl. WSL.66open_url() {67 local url="$1"68 printf ' %s↗ opening%s %s\n' "$GREEN" "$RESET" "$url"69 { if command -v wslview >/dev/null 2>&1; then wslview "$url"70 elif command -v explorer.exe >/dev/null 2>&1; then explorer.exe "$url"71 elif command -v xdg-open >/dev/null 2>&1; then xdg-open "$url"72 elif command -v open >/dev/null 2>&1; then open "$url"73 else warn "couldn't open a browser — visit it manually: $url"; fi74 } >/dev/null 2>&1 || warn "couldn't open a browser — visit it manually: $url"75}76
77# pause "msg" — wait for the human to confirm they've done the manual part.78pause() {79 printf ' %s%s%s ' "$DIM" "${1:-Press Enter to continue}" "$RESET"80 read -r _ || true81}82
83# confirm "question" — y/N gate; returns success on yes.84confirm() {85 local reply=""86 printf ' %s? %s [y/N] ' "$YELLOW" "$1"87 read -r reply || true88 [[ "$reply" =~ ^[Yy] ]]89}90
91# _existing KEY — current value of KEY in ENV_FILE, if any.92_existing() {93 [[ -f "$ENV_FILE" ]] || return 194 local line; line=$(grep -E "^${1}=" "$ENV_FILE" | tail -n1) || return 195 printf '%s' "${line#*=}"96}97
98# ask KEY "Prompt" — read a value into $KEY. Offers the existing .env value as99# a default on re-runs (Enter keeps it). Visible input (non-secret).100ask() {101 local key="$1" prompt="$2" current input102 current=$(_existing "$key" || true)103 if [[ -n "$current" ]]; then104 printf ' %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"105 else106 printf ' %s%s%s ' "$BOLD" "$prompt" "$RESET"107 fi108 read -r input || true109 [[ -z "$input" && -n "$current" ]] && input="$current"110 printf -v "$key" '%s' "$input"111}112
113# ask_secret KEY "Prompt" — like ask, but input is hidden.114ask_secret() {115 local key="$1" prompt="$2" current input116 current=$(_existing "$key" || true)117 if [[ -n "$current" ]]; then118 printf ' %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"119 else120 printf ' %s%s%s ' "$BOLD" "$prompt" "$RESET"121 fi122 read -rs input || true123 printf '\n'124 [[ -z "$input" && -n "$current" ]] && input="$current"125 printf -v "$key" '%s' "$input"126}127
128# write_env KEY VALUE — upsert KEY=VALUE into ENV_FILE (creates it; replaces129# any existing line). Idempotent.130write_env() {131 local key="$1" value="$2" tmp132 touch "$ENV_FILE"133 tmp=$(mktemp)134 grep -vE "^${key}=" "$ENV_FILE" > "$tmp" || true135 printf '%s=%s\n' "$key" "$value" >> "$tmp"136 mv "$tmp" "$ENV_FILE"137 WRITTEN_ENV+=("$key")138 printf ' %s✓ wrote%s %s → %s\n' "$GREEN" "$RESET" "$key" "$ENV_FILE"139}140
141# set_secret NAME VALUE — set a GitHub Actions repo secret via gh. Falls back142# to a warning (and records it) if gh is unavailable or unauthenticated.143set_secret() {144 local name="$1" value="$2"145 if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then146 if printf '%s' "$value" | gh secret set "$name" >/dev/null 2>&1; then147 WRITTEN_SECRET+=("$name")148 printf ' %s✓ set%s GitHub secret %s\n' "$GREEN" "$RESET" "$name"149 return150 fi151 fi152 SKIPPED+=("GitHub secret $name (set it manually: gh secret set $name)")153 warn "skipped GitHub secret $name — gh not ready; set it later"154}155
156# set_var NAME VALUE — set a GitHub Actions repo variable (non-secret).157set_var() {158 local name="$1" value="$2"159 if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then160 if gh variable set "$name" --body "$value" >/dev/null 2>&1; then161 printf ' %s✓ set%s GitHub variable %s\n' "$GREEN" "$RESET" "$name"162 return163 fi164 fi165 SKIPPED+=("GitHub variable $name")166 warn "skipped GitHub variable $name — gh not ready; set it later"167}168
169# finish — clear, then a closing summary of everything configured.170finish() {171 _clear172 printf '\n%s%s ✓ Setup complete%s\n' "$BOLD" "$GREEN" "$RESET"173 (( ${#WRITTEN_ENV[@]} )) && note "wrote ${#WRITTEN_ENV[@]} value(s) to $ENV_FILE: ${WRITTEN_ENV[*]}"174 (( ${#WRITTEN_SECRET[@]} )) && note "set ${#WRITTEN_SECRET[@]} GitHub secret(s): ${WRITTEN_SECRET[*]}"175 if (( ${#SKIPPED[@]} )); then176 printf '\n'; warn "still to do by hand:"177 for s in "${SKIPPED[@]}"; do note " - $s"; done178 fi179 printf '\n'180}181
182# ──────────────────────────────────────────────────────────────────────────183# STAGES — author this section. One stage() per step the human takes.184# Replace the example below. Set TOTAL_STAGES to match the stages you write.185# ──────────────────────────────────────────────────────────────────────────186
187TOTAL_STAGES=9188
189REPO_ROOT="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel 2>/dev/null || echo "$HOME/instagram-apify-scraper")"190
191banner "Publish instagram-post-reel-scraper to the Apify Store"192
193# ── Stage 1: Apify CLI install + login ─────────────────────────────────────194stage "Apify CLI — install & log in"195say "Nothing has pushed this Actor to Apify's platform yet — no CLI, no linked project."196if command -v apify >/dev/null 2>&1; then197 say "apify CLI already installed ($(apify --version 2>/dev/null))."198else199 if confirm "apify CLI not found — install it now via 'npm install -g apify-cli'?"; then200 npm install -g apify-cli201 else202 warn "can't continue without the CLI — install it manually, then re-run this wizard."203 exit 1204 fi205fi206step "Run 'apify login' below. It opens a browser to authenticate, or you can paste an API token"207step "from Console → Settings → Integrations → API token."208( cd "$REPO_ROOT" && apify login )209pause "Logged in? Press Enter to continue."210
211# ── Stage 2: Push the Actor ─────────────────────────────────────────────────212stage "Push the Actor to your account"213say "This uploads the current code as an Actor on your account. It is NOT public or listed yet —"214say "just present on the platform so the Console has something to configure."215step "If this is the first push, the CLI will ask whether to create a new Actor — say yes and"216step "accept the name from .actor/actor.json (instagram-post-reel-scraper)."217( cd "$REPO_ROOT" && apify push )218pause "Push finished successfully? Press Enter to continue."219
220# ── Stage 3: Competitor pricing research ────────────────────────────────────221stage "Research competitor pricing"222say "The spec requires pricing below the lowest current comparable competitor — this has to be"223say "checked live, not assumed from memory."224open_url "https://apify.com/store?search=instagram+scraper"225step "Skim the pay-per-event Instagram scrapers listed. Note the lowest per-event price you see"226step "among actors comparable to this one (single post/reel or profile post/reel scraping)."227ask LOWEST_COMPETITOR_PRICE "Lowest comparable competitor price you found (e.g. \$0.01 per item):"228note "Stage 4 will ask you to price both events below this."229
230# ── Stage 4: Configure pay-per-event pricing ────────────────────────────────231stage "Configure pay-per-event pricing"232open_url "https://console.apify.com/actors"233step "Open the Actor you just pushed (instagram-post-reel-scraper) → Publication tab → Monetization."234step "Add TWO events, matching these names exactly (the code charges these via Actor.pushData(item, eventName)):"235say " • name: post-scraped — title: e.g. \"Post scraped\" — description: one post pushed to the Dataset"236say " • name: reel-scraped — title: e.g. \"Reel scraped\" — description: one reel pushed to the Dataset"237step "Price BOTH below the competitor price from Stage 3 ($LOWEST_COMPETITOR_PRICE)."238ask POST_SCRAPED_PRICE "Price you set for post-scraped:"239ask REEL_SCRAPED_PRICE "Price you set for reel-scraped:"240pause "Both events saved in Console? Press Enter to continue."241
242# ── Stage 5: Display information ────────────────────────────────────────────243stage "Display information"244say "Still on the Publication tab:"245step "Add an Actor logo (any square image works for now — can be swapped later)."246step "Confirm the description pulls in cleanly and reads well."247step "Confirm the README renders correctly as the Store detail page — it now carries the"248step "Pricing, Maintenance, and Disclaimer sections written for ticket 05. Check the"249step "Disclaimer section is visible and unmodified by any Console-side formatting."250pause "Display Information section marked complete? Press Enter to continue."251
252# ── Stage 6: Sample output & schemas ────────────────────────────────────────253stage "Sample output & schemas"254note "Apify's docs don't spell out exact fields here — follow whatever the Publication tab's"255note "\"Sample Output\" and \"Schemas\" sections prompt you for on the page itself."256step "Sample Output: if not auto-populated, run the Actor once (single postOrReelUrl input against"257step "a real public post) and use that run's dataset item as the sample."258step "Schemas: accept the auto-generated output schema unless the Console flags something missing."259pause "Both sections marked complete? Press Enter to continue."260
261# ── Stage 7: Actor permissions ──────────────────────────────────────────────262stage "Actor permissions"263note "Again, no fixed field list from the docs — follow the Console page's own prompts."264step "Set whatever permission level the Publication tab asks for (this determines what a user"265step "running the Actor from the Store can see/access) — the default/minimal option is fine for v1."266pause "Actor Permissions section marked complete? Press Enter to continue."267
268# ── Stage 8: Publish ─────────────────────────────────────────────────────────269stage "Publish on Store"270warn "This is the irreversible step — the Actor becomes publicly listed and runnable by anyone."271say "All Publication tab sections should now show as complete (Display Information, Monetization,"272say "Sample Output, Schemas, Actor Permissions)."273if confirm "All sections complete and ready to go public — click 'Publish on Store' now?"; then274 step "Click 'Publish on Store' in the Console."275 pause "Published? Press Enter to continue."276else277 warn "stopping before publish — re-run this wizard (or just do Stage 8 by hand) when ready."278 SKIPPED+=("Publish on Store — deferred, run was stopped before the irreversible step")279fi280
281# ── Stage 9: Verify ──────────────────────────────────────────────────────────282stage "Verify the listing is live"283open_url "https://apify.com/store?search=instagram+post+reel+scraper"284step "Confirm your Actor appears in the Store search results with the two priced events showing."285pause "Confirmed live? Press Enter to continue."286
287finish288note "Recorded this run: competitor floor $LOWEST_COMPETITOR_PRICE, post-scraped $POST_SCRAPED_PRICE, reel-scraped $REEL_SCRAPED_PRICE."289# ──────────────────────────────────────────────────────────────────────────1Status: ready-for-agent2
3# Instagram Post/Reel Scraper — Apify Actor4
5## Problem Statement6
7Anyone who wants structured data out of public Instagram content — marketers pulling a competitor's post performance, researchers sampling reels, analysts tracking a brand's account — has to either build their own scraper (fragile, time-consuming, breaks constantly against Instagram's anti-bot measures) or pay one of the existing Apify Store scrapers, which are already commoditized around the same core feature set (profile posts, reels, hashtags, comments) at broadly similar prices.8
9Separately, the developer publishing this Actor wants to enter that market and generate revenue from it, but cannot take on any ongoing personal financial cost — not for proxies, not for third-party APIs, not for storage — at launch or at any point later.10
11## Solution12
13A v1 Apify Actor that scrapes public Instagram profile posts and reels, plus single post/reel URL lookups, with no login required, and publishes to the Apify Store under pay-per-event pricing set below the cheapest current competitor. The Actor returns structured metadata and direct media URLs (never downloads or stores media itself).14
15Every design choice pushes any real operating cost (proxying, authentication, future paid AI calls) onto the end user of the Actor via optional bring-your-own inputs, never onto the developer — so the product can be run and grown indefinitely at zero cost to its publisher.16
17Three further capabilities are planned as later phases, once v1 has validated demand: Stories/Highlights access (gated behind a user-supplied session cookie), reel transcription (self-hosted, open-source, no paid API), and follower/following counts and growth stats (not full list scraping, which carries too high a ban risk).18
19**Superseded 2026-08-26 — v2 direction change.** The zero-spend/price-undercutting strategy above was v1's launch strategy, not a permanent constraint. The developer/publisher has now decided to compete on feature parity with the market's dominant incumbent (Apify's own official Instagram Scraper actor) rather than on price alone, once v1's baseline is proven. This changes two decisions specifically: (1) the zero-spend principle no longer forbids the Actor from defaulting to Apify Proxy (a real, billable cost that must now be priced into pay-per-event rates rather than avoided), and (2) hashtag/search-based discovery, comments, mentions, and full profile info move from "never adopted" to in-scope v2 features. See the new "v2 — Feature parity" entries under Implementation Decisions and Out of Scope for what specifically changed and what's still excluded (full follower-list scraping and Apify's own paid *dedicated* scrapers — Ads, Reel downloads/transcripts, profile verification data — stay out of scope because Apify's own base Instagram Scraper doesn't include them either; matching it doesn't require them).20
21## User Stories22
231. As an Apify Store user, I want to scrape a public Instagram profile's posts by supplying just a username, so that I can get structured post data without logging in.242. As an Apify Store user, I want to scrape a public Instagram profile's reels by supplying just a username, so that I can get structured reel data without logging in.253. As an Apify Store user, I want to look up a single post or reel by URL, so that I don't have to scrape an entire profile when I only need one item.264. As an Apify Store user, I want the output to include direct media URLs rather than downloaded files, so that I can fetch or skip the media myself without paying for Actor storage I don't need.275. As an Apify Store user, I want the returned dataset fields (caption, timestamp, like count, comment count, media type, owner username, shortcode) to be consistent across posts and reels, so that I can process both content types with the same downstream code.286. As an Apify Store user, I want reel-specific fields (e.g. view count, duration) included when scraping reels, so that I don't lose data specific to that content type.297. As an Apify Store user, I want to run the Actor without providing any Instagram login credentials, so that I'm not exposing my own account to a ban risk just to get public data.308. As an Apify Store user, I want to optionally supply my own proxy configuration, so that I can improve reliability at scale if I choose to, without the Actor forcing a paid proxy cost on me by default.319. As an Apify Store user, I want clear pricing per event/content type, so that I know exactly what a run will cost before I trigger it.3210. As an Apify Store user, I want the listing to disclose that scraping Instagram may conflict with its Terms of Service, so that I can make an informed decision about my own usage risk.3311. As an Apify Store user, I want the Actor to fail gracefully (clear error, not a silent empty result) when a requested profile is private or a post/reel no longer exists, so that I can distinguish "no data" from "broken run."3412. As the developer/publisher, I want the Actor priced below the cheapest comparable competitor at launch, so that I can win initial volume and reviews without having a differentiated feature set yet.3513. As the developer/publisher, I want zero infrastructure or third-party API costs at v1, so that running and hosting the Actor never costs me money regardless of usage volume.3614. As the developer/publisher, I want a pay-per-event monetization model on the Apify Store, so that I'm paid per unit of value delivered (per post/reel/lookup) rather than a flat rental fee I might not recoup.3715. As the developer/publisher, I want to be able to ship v1 with a narrow scope and add Stories/Highlights, transcripts, and follower stats later, so that I can validate demand before investing further unpaid effort.3816. As the developer/publisher, I want to treat maintenance as best-effort/ship-and-revisit rather than committing to an SLA, so that the project doesn't require an ongoing time commitment I can't guarantee.3917. As an Apify Store user (future, v1.1), I want to supply my own Instagram session cookie to unlock Stories/Highlights scraping, so that I can access that content without the Actor's developer ever handling my credentials.4018. As an Apify Store user (future, v1.2), I want reels to come with an automatically generated transcript, so that I can search/analyze reel content as text without manually transcribing audio.4119. As an Apify Store user (future, v1.2), I want follower/following counts and growth stats for a profile, so that I can track account growth without the Actor attempting a full (high-ban-risk) follower list scrape.4220. As the developer/publisher (future phases), I want any paid dependency introduced later (e.g. a transcription API) to be funded by the end user (bring-your-own API key) rather than by me, so that the zero-spend constraint holds as the product grows.43
44### v2 — Feature parity (added 2026-08-26)45
4621. As an Apify Store user, I want to scrape comments from a post or reel URL, so that I can analyze audience engagement without opening Instagram myself.4722. As an Apify Store user, I want to scrape mentions from a profile URL, so that I can see what content a user has been tagged in.4823. As an Apify Store user, I want full profile info (bio, follower/following counts, verification status, external link, category) from a profile URL, so that I don't need a separate tool just for account-level metadata.4924. As an Apify Store user, I want to scrape posts and reels by hashtag, and get hashtag volume/metadata from a hashtag URL or search query, so that I can do trend and topic research, not just single-account research.5025. As an Apify Store user, I want place details from a location URL or search query, so that I can research location-tagged content.5126. As an Apify Store user, I want to search for Instagram users by a search query, so that I can discover accounts without already knowing their exact username.5227. As an Apify Store user, I want the Actor to reliably distinguish "Instagram blocked this request" from "this content genuinely doesn't exist," so that transient blocks don't masquerade as permanent 404s (sharpens story 11 now that reliability is a competitive differentiator, not just a nice-to-have).5328. As the developer/publisher, I want the Actor's default reliability (proxy rotation, fingerprint spoofing, session reuse) to be competitive with Apify's own incumbent scraper out of the box, so that "no proxy configured" isn't the Actor's default failure mode once price is no longer the only differentiator.54
55## Implementation Decisions56
57- **Runtime/SDK**: Node.js using Crawlee + the Apify SDK — Apify's own primary stack, with the deepest first-party docs/examples and the stack most incumbent Instagram scrapers on the Store are built on.58- **Scraping technique**: browser-driven (Playwright), loading each permalink directly and reading the data Instagram embeds server-side in the page rather than waiting on a client-side XHR (direct navigation doesn't reliably trigger one). **Corrected 2026-08-12 from an original hybrid direct-API-first design** — live capture confirmed plain HTTP requests get Instagram's anti-bot challenge shell, not real data, while the identical request succeeds from within a real browser session; no replicable direct-HTTP request recipe (session cookies/CSRF token) was captured. A lighter direct-HTTP path remains a possible future optimization, not part of v1.59- **Auth (v1)**: fully anonymous — no login, no session cookie required or accepted for the v1 feature set (profile posts, profile reels, single URL lookup). All v1 data must be reachable from Instagram's public web surface without authentication.60- **Auth (v1.1, Stories/Highlights only)**: optional `sessionCookie` Actor input, supplied by the end user for their own Instagram session. The Actor must never request, store, log, or persist this value beyond the lifetime of the run that uses it, and must never attempt to acquire a session via a login flow itself.61- **Proxy**: no bundled/paid proxy (e.g. no default use of Apify Proxy's paid tiers). An optional proxy-configuration Actor input lets the end user supply their own proxy; absent that input, the Actor runs proxy-less and accepts a higher block/ban rate as a known, documented limitation. **Superseded 2026-08-26, then reverted 2026-09-10**: briefly defaulted Apify Proxy (RESIDENTIAL, then the default datacenter pool) on, per the 2026-08-26 note below — but live testing (ticket 08) hit three distinct connection-failure modes (`ERR_TUNNEL_CONNECTION_FAILED` on both proxy groups, then `ERR_HTTP_RESPONSE_CODE_FAILURE`), none reproducible outside the live platform and none traceable to this project's own code (verified independently: proxy-wiring matches the Apify SDK's contract, and `channel: 'chrome'` + Playwright's proxy-auth injection both confirmed working locally against a real authenticated proxy). Leading theory: Instagram actively disrupts traffic from Apify's shared proxy IP pools specifically. **Reverted to the original v1 design above** (proxy-less by default, BYO proxy as an opt-in override) — see ticket 08's Comments for the full investigation.62- **Session management (added 2026-08-26)**: browser instances are reused across items within a run (bounded concurrency via a small worker pool) instead of one Chromium launch-and-close per item — the v1 code's per-item launch was an explicit known limitation (see ticket 03), not a permanent design choice. Sessions/cookies are pooled and retired on detected blocks rather than discarded per item.63- **Anti-detection (added 2026-08-26)**: `fingerprint-injector`/`fingerprint-generator` (already transitive Crawlee dependencies, previously unused) are wired into the Playwright browser context to reduce fingerprint-based blocking. This is additive to the existing browser-driven approach — it does not reopen the rejected direct-HTTP path (that rejection was based on a live anti-bot capture, not a missing layer; see the technique decision above).64- **Block vs. missing detection (added 2026-08-26)**: `BrowserItemFetcher`/`BrowserProfileFetcher` currently collapse "Instagram served an anti-bot challenge shell" and "this post/profile is genuinely gone" into the same not-found error path. These must be distinguished (e.g. by checking for the challenge shell's markers before concluding not-found) so blocked requests can be retried/rotated instead of silently reported as missing content — this directly implements story 27.65- **Media handling**: dataset items contain direct URLs to images/video, never downloaded binaries. No use of Apify Key-Value Store (or any storage) for media content, to keep storage cost at zero regardless of scale.66- **Output schema (v1, shared shape across posts and reels)**: shortcode/id, owner username, content type (post|reel), caption, timestamp, like count, comment count, media type (image|video|carousel), media URL(s). **Corrected 2026-08-12**: view count and duration dropped for reels — confirmed absent from the anonymous single-item reel query, not just unextracted. Posts and reels arrive via the **same** `xig_polaris_media` source schema on direct navigation (an earlier pass mistakenly concluded two separate schemas, based on data captured via a different, unused code path — client-side SPA link-clicks); one mapper derives `contentType`/`mediaType` from the `media_type`/`product_type` fields rather than dispatching between two schemas.67- **Input schema (v1)**: accepts either a profile username (to scrape that profile's posts+reels) or a direct post/reel URL (single-item lookup); optional proxy configuration input as described above.68- **Monetization**: pay-per-event pricing on the Apify Store, priced per distinct content-type event (e.g. "post scraped," "reel scraped," "URL lookup"), set below the lowest current comparable competitor price at launch. **Superseded 2026-08-26, then reverted 2026-09-10**: the "reassess per-event pricing to cover proxy cost" plan (ticket 10) is moot now that Proxy above has reverted to proxy-less-by-default — there's no proxy cost to absorb. Current live pricing ($0.001/event for both events) stands unchanged. Repricing once v2 events (comment/mention/hashtag/place/user-search) ship, to reflect a wider feature set, remains a live possibility independent of proxy cost.69- **Legal**: the Store listing carries a standard disclaimer (lawful/research use only, used at the runner's own risk, no guarantee of compliance with Instagram's Terms of Service).70- **Phasing** (drives ticket sequencing in `to-tickets`, not built as part of this spec's v1 slice):71 - v1: profile posts, profile reels, single post/reel URL lookup, anonymous only.72 - v1.1: Stories/Highlights via optional user-supplied session cookie.73 - v1.2: reel transcription via a self-hosted, open-source speech-to-text model running inside the Actor (no external paid API, so no BYO-key input is needed for this feature specifically — the zero-spend property holds for both developer and end user here).74 - v1.2 (or alongside): follower/following counts and growth stats only — explicit non-goal is full follower/following list scraping.75 - **v2 (added 2026-08-26) — feature parity with Apify's own Instagram Scraper**: reliability hardening (default proxy, session reuse/bounded concurrency, fingerprint injection, block-vs-missing detection) first, since it's a prerequisite for measuring whether the new scrape types work at all; then comments, mentions, full profile info, hashtag posts/reels + hashtag volume, place details, user search — matching Apify's own base actor's feature list. Apify's *dedicated* (separately sold) scrapers — Ads, Reel downloads/transcripts, profile location/verification data — are explicitly not part of this parity target, since they're not part of the incumbent's base actor either. **Updated 2026-09-10**: "default proxy" dropped from this list — see Proxy above. The other three reliability items (session reuse/concurrency, fingerprint injection, block-vs-missing detection) are proxy-independent and unaffected; block-vs-missing detection and session reuse/concurrency already shipped (tickets 06, 07).76- **Zero-spend principle** (v1-era, applies to launch cost avoidance only). **Superseded 2026-08-26, then reinstated 2026-09-10**: the 2026-08-26 note accepted proxy cost as a real, billed-to-the-runner cost of reliability — that's moot now that Proxy above has reverted to proxy-less-by-default (no proxy cost exists to accept or avoid). Third-party *paid* APIs (e.g. a commercial transcription API) are still to be avoided or made BYO-key where practical, consistent with the original v1-era principle, though not treated as an absolute rule blocking future features.77
78## Testing Decisions79
80- **Seam (single, highest-level)**: `Actor input → real pipeline (request routing, scraping, normalization, dataset push) → Apify Dataset output`. The only thing stubbed is the outbound HTTP layer to Instagram — canned GraphQL/JSON responses stand in for the network call. Everything else in the pipeline runs for real.81- **Harness**: Apify SDK's local storage emulation (`Actor.init()` against a local/in-memory dataset) — no real Apify Cloud run and no real network call to Instagram in tests.82- **What a good test asserts**: the shape and content of the resulting Dataset item(s) for a given canned input+response pair — never internal function calls or implementation details of the scraper.83- **Coverage required through this one seam**: profile-posts path, profile-reels path, single-URL-lookup path, and edge cases — private/inaccessible profile, deleted or missing post/reel, and an empty (no-posts) profile — each as a distinct canned network response.84- **Prior art**: none — this is a greenfield repo, so the first tests against this seam establish the pattern that subsequent tickets (v1.1, v1.2) should extend rather than replace.85
86## Out of Scope87
88- Login-based scraping, session/cookie handling, and Stories/Highlights access — deferred to v1.1, not part of this spec's buildable slice.89- Reel transcription — deferred to v1.2.90- Follower/following counts and growth stats — deferred to v1.2.91- Full follower/following list scraping — **still** out of scope indefinitely; rejected due to Instagram's aggressive rate-limiting/ban risk on that specific action. Not reopened by the 2026-08-26 v2 pivot — Apify's own base Instagram Scraper doesn't offer this either, so it isn't required for parity.92- ~~Hashtag or search-based discovery — never adopted during scoping; not part of any planned phase.~~ **Reopened 2026-08-26**: now in scope for v2 (hashtag posts/reels, hashtag volume, user search) — see Implementation Decisions' v2 phasing.93- Downloading or storing media binaries — permanently out of scope by design (URLs only). Not reopened by the v2 pivot — Apify's base actor also returns URLs, not files (media downloads are one of its separately sold dedicated scrapers).94- Any bundled/paid proxy service — out of scope by design (BYO proxy only). **Briefly reopened 2026-08-26, closed again 2026-09-10**: Apify Proxy defaulting on was tried and reverted after live testing showed it's not viable against Instagram specifically (three distinct connection-failure modes across two proxy groups — see Implementation Decisions' Proxy entry and ticket 08's Comments). BYO proxy remains supported as an override; a non-Apify third-party proxy was never tested and might behave differently, since the failure appears specific to Apify's shared IP pools being recognized, not proxies in general.95- Any developer-funded third-party *API* cost (e.g. a paid transcription API) — still avoided/BYO-key where practical, though no longer an absolute rule (see Zero-spend principle entry above). Proxy is no longer covered by this line — it's now an accepted, billed-to-the-runner cost.96- Apify's own separately sold *dedicated* scrapers' feature areas — Instagram Ads, Reel downloads/transcripts, profile location/verification data — not required for parity with the base Instagram Scraper actor and not adopted by the v2 pivot.97
98## Further Notes99
100- Pricing is a launch-time decision, not a permanent one: undercutting competitors is the only viable lever for v1 because it has no feature differentiation yet; once v1.1/v1.2 ship, pricing should be reassessed since Stories/transcripts/follower-stats give the Actor differentiated value to charge a premium for.101- Maintenance stance is explicitly best-effort ("ship it, revisit occasionally") — the Actor's Store listing and any user-facing documentation should set this expectation rather than implying an SLA, since Instagram changes its rendering/API frequently enough that breakage should be expected between fixes.102- This spec and its phasing came out of a structured design-tree interview (the `/grilling` session) rather than a pre-existing codebase — there is no prior code, ADRs, or domain glossary to reconcile against yet. `docs/agents/domain.md`'s "proceed silently if absent" rule applies as-is.103- **2026-08-26 — v2 pivot rationale.** The v1 strategy (undercut on price, zero developer spend, narrow anonymous-only surface) was explicitly a launch strategy pending demand validation (see original Solution section and story 15). The developer/publisher has now decided to pursue feature and reliability parity with Apify's own official Instagram Scraper actor instead of continuing to compete on price alone. This is a genuine business-model change (accepting a real, billed proxy cost; expanding scope to hashtag/comments/mentions/place/user-search) — not a silent scope-creep override of the v1 decisions, which is why it's recorded here with explicit strikethroughs/supersession notes on every v1 decision it touches, per `docs/agents/domain.md`'s guidance to surface rather than silently override conflicting prior decisions. Tickets 06+ under `.scratch/instagram-scraper/issues/` implement this phase; ticket ordering puts reliability hardening (06) before new content types (07+) because the new scrape types can't be reliably measured against a browser pipeline that still conflates "blocked" with "doesn't exist."1# Domain Docs2
3How the engineering skills should consume this repo's domain documentation when exploring the codebase.4
5## Before exploring, read these6
7- **`CONTEXT.md`** at the repo root, or8- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.9- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.10
11If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved.12
13## File structure14
15Single-context repo (most repos):16
17```18/19├── CONTEXT.md20├── docs/adr/21│ ├── 0001-event-sourced-orders.md22│ └── 0002-postgres-for-write-model.md23└── src/24```25
26## Use the glossary's vocabulary27
28When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.29
30If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`).31
32## Flag ADR conflicts33
34If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:35
36> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_1# Issue tracker: Local Markdown2
3Issues and specs for this repo live as markdown files in `.scratch/`.4
5## Conventions6
7- One feature per directory: `.scratch/<feature-slug>/`8- The spec is `.scratch/<feature-slug>/spec.md`9- Implementation issues are one file per ticket at `.scratch/<feature-slug>/issues/<NN>-<slug>.md`, numbered from `01` — never a single combined tickets file10- Triage state is recorded as a `Status:` line near the top of each issue file (see `triage-labels.md` for the role strings)11- Comments and conversation history append to the bottom of the file under a `## Comments` heading12
13## When a skill says "publish to the issue tracker"14
15Create a new file under `.scratch/<feature-slug>/` (creating the directory if needed).16
17## When a skill says "fetch the relevant ticket"18
19Read the file at the referenced path. The user will normally pass the path or the issue number directly.20
21## Wayfinding operations22
23Used by `/wayfinder`. The **map** is a file with one **child** file per ticket.24
25- **Map**: `.scratch/<effort>/map.md` — the Notes / Decisions-so-far / Fog body.26- **Child ticket**: `.scratch/<effort>/issues/NN-<slug>.md`, numbered from `01`, with the question in the body. A `Type:` line records the ticket type (`research`/`prototype`/`grilling`/`task`); a `Status:` line records `claimed`/`resolved`.27- **Blocking**: a `Blocked by: NN, NN` line near the top. A ticket is unblocked when every file it lists is `resolved`.28- **Frontier**: scan `.scratch/<effort>/issues/` for files that are open, unblocked, and unclaimed; first by number wins.29- **Claim**: set `Status: claimed` and save before any work.30- **Resolve**: append the answer under an `## Answer` heading, set `Status: resolved`, then append a context pointer (gist + link) to the map's Decisions-so-far in `map.md`.1# Triage Labels2
3The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker.4
5| Label in mattpocock/skills | Label in our tracker | Meaning |6| --------------------------- | --------------------- | ----------------------------------------- |7| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |8| `needs-info` | `needs-info` | Waiting on reporter for more information |9| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |10| `ready-for-human` | `ready-for-human` | Requires human implementation |11| `wontfix` | `wontfix` | Will not be actioned |1213When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.14
15Edit the right-hand column to match whatever vocabulary you actually use.1# Fixtures2
3Hand-assembled from a live, logged-out capture against real Instagram pages on 2026-08-12 (via Chrome/Playwright, direct navigation to permalink URLs — plain `curl` gets an anti-bot challenge shell, not real data).4
5**Correction (same day, later pass):** an earlier version of this README and `post-carousel.json` described posts as using a different, "legacy" `shortcode`/`edge_media_*` schema, distinct from reels' `xig_polaris_media` schema. That was captured via *client-side SPA link-clicks* within an already-loaded page — a different code path than what the Actor's fetcher actually uses (direct navigation to the permalink URL). Under direct navigation, verified with a structured `JSON.parse` + recursive search (not raw substring matching, which was misleading due to other items' data being embedded nearby on the same page) — **both posts and reels use the same `xig_polaris_media` schema**, differentiated by `media_type` (`2` = video/reel, `8` = carousel, presumably `1` = single image) and `product_type` (`"clips"` = reel, `"carousel_container"` = carousel, presumably something else for a single-image/video feed post). `post-carousel.json` below reflects this corrected, verified shape.6
7**Confidence levels, per fixture:**8
9- `reel-single.json` — field names/values (`code`, `like_count`, `comment_count`, `taken_at`, `media_type: 2`, `product_type: "clips"`, `caption.text`, `video_versions[].type`, `__typename: XIGPolarisVideoMedia`) directly verified from a real public reel (`instagram.com/reel/DbvqPafyqY-/`), confirmed present under `data.xig_polaris_media.if_not_gated_logged_out` via structured search. `view_count`, `play_count`, and `duration` were checked for and confirmed **absent** from this response entirely.10- `post-carousel.json` — field names/values (`code`, `like_count: 286443`, `comment_count: 11480`, `taken_at: 1785437998`, `media_type: 8`, `product_type: "carousel_container"`, `caption.text`, `carousel_media_count: 8`, `carousel_media[0].__typename: XIGPolarisImageMedia`, `display_uri`, `image_versions2.candidates`) directly verified via structured search on a real public carousel post (`instagram.com/instagram/p/DbbY9pdm6Q2/`).11- `not-found-error.json` — the real generic GraphQL error envelope captured from a rejected logged-out query (`{"errors":[...],"extensions":{"is_final":true}}`), observed during earlier client-side-navigation testing. It's the real *envelope shape* Instagram uses for rejected queries, not a verbatim capture of a "post not found" response specifically. The Actor's actual not-found detection instead relies on the verified DOM signal (page title containing `"isn't available"`) rather than this envelope, since that's what direct navigation to a deleted post was confirmed to produce.12
13All media URLs are replaced with `https://example-cdn.test/...` placeholders — the real ones are signed and expire.{ "errors": [ { "message": "Unauthorized logged out query.", "severity": "CRITICAL", "code": 1675002 } ], "extensions": { "is_final": true }}{ "data": { "xig_polaris_media": { "__typename": "XIGPolarisCarouselMedia", "pk": "3952862887472243766", "code": "DbbY9pdm6Q2", "if_not_gated_logged_out": { "__typename": "XIGPolarisCarouselMedia", "user": { "username": "instagram", "pk": "25025320" }, "caption": { "text": "Meet the artist behind “Miss Mango” 🥭🎶\n\n@prodlaila writes, sings and produces her own music from her bedroom studio." }, "like_count": 286443, "comment_count": 11480, "taken_at": 1785437998, "media_type": 8, "product_type": "carousel_container", "carousel_media_count": 8, "display_uri": "https://example-cdn.test/placeholder-carousel-cover.jpg", "image_versions2": { "candidates": [{ "url": "https://example-cdn.test/placeholder-carousel-cover.jpg" }], "additional_candidates": {} }, "carousel_media": [ { "__typename": "XIGPolarisImageMedia", "code": "DbbY9pdm6Q2", "media_type": 1, "display_uri": "https://example-cdn.test/placeholder-carousel-child-1.jpg" } ] } } }}{ "data": { "xig_polaris_media": { "__typename": "XIGPolarisVideoMedia", "pk": "3958568374678234680", "code": "DbvqPafyqY-", "if_not_gated_logged_out": { "__typename": "XIGPolarisVideoMedia", "user": { "username": "instagram", "pk": "25025320", "full_name": "Instagram", "is_verified": true, "is_private": false, "profile_pic_url": "https://example-cdn.test/placeholder-profile.jpg" }, "caption": { "text": "From 🤝 to 🌊\n\n@mark.flow.22 finds waterfalls, mountains and hidden spots you’ve probably never seen. \n\n“I hope my videos inspire people to explore more, spend time in nature and create their own adventures. Even a short trip outdoors can become an unforgettable experience.”\n" }, "like_count": 1250208, "comment_count": 19174, "taken_at": 1786118430, "created_at": 1786118429, "media_type": 2, "product_type": "clips", "has_audio": true, "image_versions2": { "additional_candidates": { "first_frame": { "url": "https://example-cdn.test/placeholder-first-frame.jpg" } } }, "video_versions": [ { "type": 101, "url": "https://example-cdn.test/placeholder-video.mp4" } ] } } }}1/** A promise a test can resolve/reject on its own schedule, to control timing in concurrency tests. */2export function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (error: unknown) => void } {3 let resolve!: (value: T) => void;4 let reject!: (error: unknown) => void;5 const promise = new Promise<T>((res, rej) => {6 resolve = res;7 reject = rej;8 });9 return { promise, resolve, reject };10}1import type { BrowserLauncher } from '../../src/instagram/browserFetcher.js';2
3/** A launcher stub that records its call args then throws, short-circuiting the caller4 * right after the browser-launch step — enough to assert proxy wiring without a real browser. */5export function stubLauncher(calls: unknown[]): BrowserLauncher {6 return async (options) => {7 calls.push(options);8 throw new Error('stop here — only checking launch args');9 };10}1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { describe, expect, it } from 'vitest';4import { mapXigPolarisMedia, UnrecognizedMediaShapeError } from '../../src/instagram/mappers/xigPolarisMedia.js';5
6function loadFixture(name: string): unknown {7 const path = fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url));8 return JSON.parse(readFileSync(path, 'utf8'));9}10
11describe('mapXigPolarisMedia', () => {12 it('maps a real captured reel response to the shared output shape', () => {13 expect(mapXigPolarisMedia(loadFixture('reel-single.json'))).toEqual({14 shortcode: 'DbvqPafyqY-',15 ownerUsername: 'instagram',16 contentType: 'reel',17 caption:18 'From 🤝 to 🌊\n\n@mark.flow.22 finds waterfalls, mountains and hidden spots you’ve probably never seen. \n\n“I hope my videos inspire people to explore more, spend time in nature and create their own adventures. Even a short trip outdoors can become an unforgettable experience.”\n',19 timestamp: 1786118430,20 likeCount: 1250208,21 commentCount: 19174,22 mediaType: 'video',23 mediaUrl: 'https://example-cdn.test/placeholder-video.mp4',24 });25 });26
27 it('maps a real captured carousel post response to the shared output shape', () => {28 expect(mapXigPolarisMedia(loadFixture('post-carousel.json'))).toEqual({29 shortcode: 'DbbY9pdm6Q2',30 ownerUsername: 'instagram',31 contentType: 'post',32 caption: 'Meet the artist behind “Miss Mango” 🥭🎶\n\n@prodlaila writes, sings and produces her own music from her bedroom studio.',33 timestamp: 1785437998,34 likeCount: 286443,35 commentCount: 11480,36 mediaType: 'carousel',37 mediaUrl: 'https://example-cdn.test/placeholder-carousel-cover.jpg',38 });39 });40
41 it('throws UnrecognizedMediaShapeError when the response has no xig_polaris_media.code', () => {42 expect(() => mapXigPolarisMedia({ data: {} })).toThrow(UnrecognizedMediaShapeError);43 });44});1import type { Browser } from 'playwright';2import { describe, expect, it } from 'vitest';3import { BrowserItemFetcher, type BrowserLauncher } from '../../src/instagram/browserFetcher.js';4import { deferred } from '../helpers/deferred.js';5import { stubLauncher } from '../helpers/stubLauncher.js';6
7/** A minimal fake satisfying the Playwright Browser/Page surface fetchRawItem actually calls. */8function fakePlaywrightBrowser(): Browser {9 const page = {10 goto: async () => undefined,11 evaluate: async () => ({ code: 'x', if_not_gated_logged_out: {} }),12 close: async () => undefined,13 };14 return {15 newPage: async () => page,16 close: async () => undefined,17 } as unknown as Browser;18}19
20describe('BrowserItemFetcher proxy wiring', () => {21 it('launches with the given proxy option when one is provided', async () => {22 const calls: unknown[] = [];23 const fetcher = new BrowserItemFetcher({24 proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' },25 launcher: stubLauncher(calls),26 });27
28 await expect(fetcher.fetchRawItem('https://www.instagram.com/p/ABC123/')).rejects.toThrow('stop here');29
30 expect(calls).toEqual([{ proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' } }]);31 });32
33 it('launches with no proxy option when none is provided', async () => {34 const calls: unknown[] = [];35 const fetcher = new BrowserItemFetcher({ launcher: stubLauncher(calls) });36
37 await expect(fetcher.fetchRawItem('https://www.instagram.com/p/ABC123/')).rejects.toThrow('stop here');38
39 expect(calls).toEqual([undefined]);40 });41});42
43describe('BrowserItemFetcher pool size', () => {44 it('bounds concurrent browser launches to poolSize, reusing a checked-in browser for a queued call', async () => {45 const launchCalls: unknown[] = [];46 const firstLaunchGate = deferred<void>();47
48 const launcher: BrowserLauncher = async (options) => {49 launchCalls.push(options);50 if (launchCalls.length === 1) await firstLaunchGate.promise;51 return fakePlaywrightBrowser();52 };53
54 const fetcher = new BrowserItemFetcher({ launcher, poolSize: 1 });55
56 const task1 = fetcher.fetchRawItem('https://www.instagram.com/p/AAA111/');57 await Promise.resolve();58 const task2 = fetcher.fetchRawItem('https://www.instagram.com/p/BBB222/');59 await Promise.resolve();60 await Promise.resolve();61
62 // poolSize 1: task2's checkout must queue rather than trigger its own launch.63 expect(launchCalls).toHaveLength(1);64
65 firstLaunchGate.resolve();66 await expect(task1).resolves.toBeDefined();67 await expect(task2).resolves.toBeDefined();68
69 // task2 reused the browser task1 checked back in — still only one launch total.70 expect(launchCalls).toHaveLength(1);71 });72});1import { describe, expect, it, vi } from 'vitest';2import { BrowserPool, type Closeable } from '../../src/instagram/browserPool.js';3import { deferred } from '../helpers/deferred.js';4
5function fakeBrowser(): Closeable & { close: ReturnType<typeof vi.fn> } {6 return { close: vi.fn().mockResolvedValue(undefined) };7}8
9describe('BrowserPool', () => {10 it('reuses a checked-in browser instead of launching a new one', async () => {11 const browser = fakeBrowser();12 const launch = vi.fn().mockResolvedValue(browser);13 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 2 });14
15 await pool.withBrowser(async () => 'first');16 await pool.withBrowser(async () => 'second');17
18 expect(launch).toHaveBeenCalledTimes(1);19 });20
21 it('bounds concurrency to `size`, queuing further checkouts until one frees up', async () => {22 const browsers = [fakeBrowser(), fakeBrowser()];23 let launchCount = 0;24 const launch = vi.fn().mockImplementation(async () => browsers[launchCount++]);25 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 2 });26
27 const first = deferred<void>();28 const second = deferred<void>();29 const thirdStarted = vi.fn();30
31 const task1 = pool.withBrowser(async () => {32 await first.promise;33 });34 const task2 = pool.withBrowser(async () => {35 await second.promise;36 });37 const task3 = pool.withBrowser(async () => {38 thirdStarted();39 });40
41 await Promise.resolve();42 await Promise.resolve();43 expect(launch).toHaveBeenCalledTimes(2);44 expect(thirdStarted).not.toHaveBeenCalled();45
46 first.resolve();47 await task1;48 await task3;49
50 expect(thirdStarted).toHaveBeenCalled();51 expect(launch).toHaveBeenCalledTimes(2);52
53 second.resolve();54 await task2;55 });56
57 it('retires a browser that fails with a retirable error, launching a fresh one next time', async () => {58 const browsers = [fakeBrowser(), fakeBrowser()];59 let launchCount = 0;60 const launch = vi.fn().mockImplementation(async () => browsers[launchCount++]);61 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: (error) => error === 'blocked', size: 1 });62
63 await expect(64 pool.withBrowser(async () => {65 throw 'blocked';66 }),67 ).rejects.toBe('blocked');68
69 expect(browsers[0].close).toHaveBeenCalledTimes(1);70
71 await pool.withBrowser(async () => 'ok');72
73 expect(launch).toHaveBeenCalledTimes(2);74 expect(browsers[1].close).not.toHaveBeenCalled();75 });76
77 it('does not retire a browser for a non-retirable error, and returns it to the pool for reuse', async () => {78 const browser = fakeBrowser();79 const launch = vi.fn().mockResolvedValue(browser);80 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });81
82 await expect(83 pool.withBrowser(async () => {84 throw new Error('not a block');85 }),86 ).rejects.toThrow('not a block');87
88 expect(browser.close).not.toHaveBeenCalled();89
90 await pool.withBrowser(async () => 'ok');91
92 expect(launch).toHaveBeenCalledTimes(1);93 });94
95 it('closes all idle browsers on close()', async () => {96 const browser = fakeBrowser();97 const launch = vi.fn().mockResolvedValue(browser);98 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });99
100 await pool.withBrowser(async () => 'ok');101 await pool.close();102
103 expect(browser.close).toHaveBeenCalledTimes(1);104 });105
106 it('does not leak a pool slot when a launch fails', async () => {107 const browser = fakeBrowser();108 const launch = vi109 .fn()110 .mockRejectedValueOnce(new Error('launch failed'))111 .mockResolvedValueOnce(browser);112 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });113
114 await expect(pool.withBrowser(async () => 'unreachable')).rejects.toThrow('launch failed');115
116 // If the failed launch had permanently consumed the pool's one slot, this117 // checkout would hang forever waiting for a browser that never arrives.118 await expect(pool.withBrowser(async () => 'ok')).resolves.toBe('ok');119 expect(launch).toHaveBeenCalledTimes(2);120 });121
122 it('closes a browser immediately, without re-queueing it, when checked in after close()', async () => {123 const browser = fakeBrowser();124 const launch = vi.fn().mockResolvedValue(browser);125 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });126
127 const inFlight = deferred<string>();128 const task = pool.withBrowser(async () => inFlight.promise);129
130 await pool.close();131 inFlight.resolve('done');132 await task;133
134 expect(browser.close).toHaveBeenCalledTimes(1);135 });136
137 it('rejects a checkout still queued for a slot when the pool closes, instead of hanging forever', async () => {138 const browser = fakeBrowser();139 const launch = vi.fn().mockResolvedValue(browser);140 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });141
142 const holdsTheOnlySlot = deferred<string>();143 void pool.withBrowser(async () => holdsTheOnlySlot.promise);144 await Promise.resolve();145
146 const queued = pool.withBrowser(async () => 'unreachable');147
148 await pool.close();149
150 await expect(queued).rejects.toThrow(/closed/i);151 holdsTheOnlySlot.resolve('done');152 });153
154 it('refuses a new checkout once the pool is closed', async () => {155 const browser = fakeBrowser();156 const launch = vi.fn().mockResolvedValue(browser);157 const pool = new BrowserPool<Closeable>({ launch, shouldRetire: () => false, size: 1 });158
159 await pool.close();160
161 await expect(pool.withBrowser(async () => 'unreachable')).rejects.toThrow(/closed/i);162 expect(launch).not.toHaveBeenCalled();163 });164});1import { describe, expect, it } from 'vitest';2import { BrowserProfileFetcher } from '../../src/instagram/browserProfileFetcher.js';3import { stubLauncher } from '../helpers/stubLauncher.js';4
5describe('BrowserProfileFetcher proxy wiring', () => {6 it('launches with the given proxy option when one is provided', async () => {7 const calls: unknown[] = [];8 const fetcher = new BrowserProfileFetcher({9 proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' },10 launcher: stubLauncher(calls),11 });12
13 await expect(fetcher.fetchProfileListing('nasa')).rejects.toThrow('stop here');14
15 expect(calls).toEqual([{ proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' } }]);16 });17
18 it('launches with no proxy option when none is provided', async () => {19 const calls: unknown[] = [];20 const fetcher = new BrowserProfileFetcher({ launcher: stubLauncher(calls) });21
22 await expect(fetcher.fetchProfileListing('nasa')).rejects.toThrow('stop here');23
24 expect(calls).toEqual([undefined]);25 });26});1import { describe, expect, it } from 'vitest';2import { isChallengeShell } from '../../src/instagram/challengeShell.js';3
4describe('isChallengeShell', () => {5 it("detects Instagram's anti-bot challenge shell captured via a live plain-HTTP request", () => {6 expect(isChallengeShell({ title: 'Instagram', bodyText: '' })).toBe(true);7 });8
9 it('does not flag a real, successfully rendered post/profile page', () => {10 expect(11 isChallengeShell({12 title: 'NASA (@nasa) • Instagram photos and videos',13 bodyText: 'Log in | Sign up | nasa | 104m followers',14 }),15 ).toBe(false);16 });17
18 it('does not flag the genuine "isn\'t available" not-found page', () => {19 expect(20 isChallengeShell({21 title: "Page isn't available • Instagram",22 bodyText: "Sorry, this page isn't available. The link you followed may be broken.",23 }),24 ).toBe(false);25 });26
27 it('tolerates surrounding whitespace', () => {28 expect(isChallengeShell({ title: ' Instagram ', bodyText: ' ' })).toBe(true);29 });30});1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { describe, expect, it } from 'vitest';4import { normalizeRawItem, UnrecognizedItemShapeError } from '../../src/instagram/normalize.js';5
6function loadFixture(name: string): unknown {7 const path = fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url));8 return JSON.parse(readFileSync(path, 'utf8'));9}10
11describe('normalizeRawItem', () => {12 it('normalizes a real reel response', () => {13 const item = normalizeRawItem(loadFixture('reel-single.json'));14 expect(item.contentType).toBe('reel');15 expect(item.shortcode).toBe('DbvqPafyqY-');16 });17
18 it('normalizes a real carousel post response', () => {19 const item = normalizeRawItem(loadFixture('post-carousel.json'));20 expect(item.contentType).toBe('post');21 expect(item.shortcode).toBe('DbbY9pdm6Q2');22 });23
24 it('throws UnrecognizedItemShapeError for a response with no xig_polaris_media.code', () => {25 expect(() => normalizeRawItem({ errors: [{ message: 'nope' }] })).toThrow(UnrecognizedItemShapeError);26 });27});1import { describe, expect, it } from 'vitest';2import { toPlaywrightProxy } from '../../src/instagram/proxy.js';3
4describe('toPlaywrightProxy', () => {5 it('maps hostname/port/username/password to a Playwright server URL and credentials', () => {6 const result = toPlaywrightProxy({7 hostname: 'proxy.example.com',8 port: 8000,9 username: 'user',10 password: 'pass',11 });12
13 expect(result).toEqual({14 server: 'http://proxy.example.com:8000',15 username: 'user',16 password: 'pass',17 });18 });19
20 it('omits username/password when the proxy has no credentials', () => {21 const result = toPlaywrightProxy({22 hostname: 'proxy.example.com',23 port: 8000,24 password: '',25 });26
27 expect(result).toEqual({ server: 'http://proxy.example.com:8000' });28 });29
30 it('omits credentials when password is empty even if a username is present', () => {31 // password is the field Apify's ProxyInfo always populates for a real proxy; an empty32 // password means no genuine credentials, regardless of username, so we don't send Playwright33 // into a doomed auth attempt against the proxy.34 const result = toPlaywrightProxy({35 hostname: 'proxy.example.com',36 port: 8000,37 username: 'user',38 password: '',39 });40
41 expect(result).toEqual({ server: 'http://proxy.example.com:8000' });42 });43
44 it('stringifies a numeric-string port as-is', () => {45 const result = toPlaywrightProxy({46 hostname: 'proxy.example.com',47 port: '8000',48 password: '',49 });50
51 expect(result.server).toBe('http://proxy.example.com:8000');52 });53});1import { errors, type Browser, type Page } from 'playwright';2import { BrowserPool } from './browserPool.js';3import { defaultLauncher, type BrowserLauncher } from './browserLauncher.js';4import { BlockedRequestError, isChallengeShell, readPageContent } from './challengeShell.js';5import { ItemNotFoundError, type ItemFetcher } from './fetcher.js';6import type { PlaywrightProxy } from './proxy.js';7import { InvalidInputError } from '../input.js';8
9export type { BrowserLauncher };10
11const PAGE_LOAD_TIMEOUT_MS = 20_000;12
13/**14 * Fetches a single post/reel's raw data by loading its permalink in a real15 * browser and reading the data Instagram embeds server-side in the initial16 * page (a `<script type="application/json">` block), rather than waiting on17 * a client-side XHR — direct navigation was confirmed (2026-08-12 live18 * testing) to embed the data this way and not reliably issue a matching XHR19 * at all. Plain HTTP requests get an anti-bot challenge shell instead of20 * real data, which is why this needs a real browser in the first place — see21 * ticket 02's Corrections note.22 *23 * There is no unit-testable seam inside this class for the page-reading24 * logic itself; `ItemFetcher` is the seam, and that part is exercised by25 * manual/live verification rather than the automated test suite. The26 * browser-launch step is injectable (see `launcher` below) specifically so27 * ticket 04's proxy wiring can be asserted by a test without a real browser28 * or real proxy — `BrowserPool`'s own checkout/retirement logic is29 * independently unit-tested (`browserPool.test.ts`).30 *31 * A bounded `BrowserPool` (ticket 07) replaces the original one-launch-32 * per-call design: browsers are reused across calls instead of relaunched33 * per item, and a `BlockedRequestError` retires just the affected instance34 * rather than aborting the whole run. Callers that construct this fetcher35 * (currently `main.ts`) own its lifecycle and must call `close()` once36 * they're done with it, on both the success and error paths — nothing37 * inside this class closes its own pool automatically.38 *39 * Uses raw Playwright rather than Crawlee's PlaywrightCrawler — a single40 * permalink fetch needs no request queue or routing.41 */42export class BrowserItemFetcher implements ItemFetcher {43 private readonly pool: BrowserPool<Browser>;44
45 constructor(options: { proxy?: PlaywrightProxy; launcher?: BrowserLauncher; poolSize?: number } = {}) {46 const proxy = options.proxy;47 const launcher = options.launcher ?? defaultLauncher;48 this.pool = new BrowserPool<Browser>({49 launch: () => launcher(proxy ? { proxy } : undefined),50 shouldRetire: (error) => error instanceof BlockedRequestError,51 size: options.poolSize,52 });53 }54
55 async fetchRawItem(url: string): Promise<unknown> {56 const shortcode = extractShortcode(url);57 return this.pool.withBrowser(async (browser) => {58 const page = await browser.newPage();59 try {60 return await fetchViaEmbeddedScripts(page, url, shortcode);61 } finally {62 await page.close().catch(() => {});63 }64 });65 }66
67 async close(): Promise<void> {68 await this.pool.close();69 }70}71
72function extractShortcode(url: string): string {73 const segments = new URL(url).pathname.split('/').filter(Boolean);74 const marker = segments.findIndex((segment) => segment === 'p' || segment === 'reel');75 const shortcode = marker === -1 ? undefined : segments[marker + 1];76 if (!shortcode) {77 throw new InvalidInputError(`postOrReelUrl must contain a "/p/" or "/reel/" segment: ${url}`);78 }79 return shortcode;80}81
82async function fetchViaEmbeddedScripts(page: Page, url: string, shortcode: string): Promise<unknown> {83 try {84 await page.goto(url, { waitUntil: 'networkidle', timeout: PAGE_LOAD_TIMEOUT_MS });85 } catch (error) {86 // Instagram's long-lived connections can keep networkidle from firing —87 // that's expected and we fall through to check whatever loaded. Any88 // other navigation failure (DNS, connection reset, proxy failure) is a89 // real infrastructure problem and must not be mistaken for a missing item.90 if (!(error instanceof errors.TimeoutError)) throw error;91 }92
93 const media = await page.evaluate(findEmbeddedMedia, shortcode);94
95 if (media) {96 return { data: { xig_polaris_media: media } };97 }98
99 // Embed scan came up empty — check title/body only now, so a real page100 // that just hasn't finished rendering (still carrying the SPA's101 // placeholder title/empty body) never gets misclassified while its102 // embedded JSON was findable all along.103 const content = await readPageContent(page);104 if (isChallengeShell(content)) {105 throw new BlockedRequestError(`Instagram served an anti-bot challenge shell for ${url}`);106 }107 if (content.title.includes("isn't available") && content.bodyText.includes('link may be broken')) {108 throw new ItemNotFoundError(`Instagram reports this item is not available: ${url}`);109 }110
111 throw new ItemNotFoundError(`Could not find media data embedded on the page for ${url}`);112}113
114/**115 * Runs inside the browser page (via `page.evaluate`). Scans every embedded116 * JSON script block for an object shaped like a `xig_polaris_media` node117 * matching the target shortcode.118 */119function findEmbeddedMedia(targetCode: string): Record<string, unknown> | null {120 function findMatch(value: unknown, depth: number): Record<string, unknown> | null {121 if (depth > 60 || value === null || typeof value !== 'object') return null;122 if (Array.isArray(value)) {123 for (const item of value) {124 const found = findMatch(item, depth + 1);125 if (found) return found;126 }127 return null;128 }129 const record = value as Record<string, unknown>;130 if (record.code === targetCode && record.if_not_gated_logged_out) return record;131 for (const key of Object.keys(record)) {132 const found = findMatch(record[key], depth + 1);133 if (found) return found;134 }135 return null;136 }137
138 const scripts = Array.from(document.querySelectorAll('script[type="application/json"]'));139 for (const script of scripts) {140 let parsed: unknown;141 try {142 parsed = JSON.parse(script.textContent || '');143 } catch {144 continue;145 }146 const match = findMatch(parsed, 0);147 if (match) return match;148 }149 return null;150}1import { chromium, type Browser } from 'playwright';2import type { PlaywrightProxy } from './proxy.js';3
4export type BrowserLauncher = (options?: { proxy?: PlaywrightProxy }) => Promise<Browser>;5
6// channel: 'chrome' launches the system Google Chrome that Apify's7// apify/actor-node-playwright-chrome base image pre-installs (at8// APIFY_CHROME_EXECUTABLE_PATH), instead of Playwright's own bundled,9// version-pinned Chromium build. The base image sets10// PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 specifically so a plain `npm install`11// of a newer `playwright` package never re-downloads browsers — so the12// unmanaged default channel can end up pointing at a Chromium revision13// that was never installed. The system Chrome channel sidesteps that14// version-matching entirely.15export const defaultLauncher: BrowserLauncher = (options) => chromium.launch({ ...options, channel: 'chrome' });1export interface Closeable {2 close(): Promise<void>;3}4
5export interface BrowserPoolOptions<T> {6 launch: () => Promise<T>;7 shouldRetire: (error: unknown) => boolean;8 size?: number;9}10
11const DEFAULT_SIZE = 3;12
13interface Waiter<T> {14 resolve: (value: T) => void;15 reject: (error: unknown) => void;16}17
18/**19 * A small, bounded pool of reused, lazily-launched resources (in practice,20 * Chromium instances). `withBrowser` checks out an idle instance — or21 * lazily launches one if the pool hasn't reached `size` yet — runs `fn`22 * against it, and returns it to the pool afterward for reuse. When `fn`23 * throws an error `shouldRetire` flags, that instance is closed and24 * discarded instead of returned, so the next checkout lazily relaunches a25 * clean replacement rather than reusing a now-compromised session.26 *27 * Generic over the resource type and decoupled from what makes an error28 * retirement-worthy (`shouldRetire` is caller-supplied) so this stays29 * plain pooling logic, independently testable with a stub launcher and no30 * real browser or Instagram-specific error type.31 */32export class BrowserPool<T extends Closeable> {33 private readonly launch: () => Promise<T>;34 private readonly shouldRetire: (error: unknown) => boolean;35 private readonly size: number;36 private readonly idle: T[] = [];37 private readonly waiters: Waiter<T>[] = [];38 private launched = 0;39 private closed = false;40
41 constructor(options: BrowserPoolOptions<T>) {42 this.launch = options.launch;43 this.shouldRetire = options.shouldRetire;44 this.size = options.size ?? DEFAULT_SIZE;45 }46
47 async withBrowser<R>(fn: (browser: T) => Promise<R>): Promise<R> {48 const browser = await this.checkout();49 let result: R;50 try {51 result = await fn(browser);52 } catch (error) {53 if (this.shouldRetire(error)) {54 await this.retire(browser);55 } else {56 this.checkin(browser);57 }58 throw error;59 }60 this.checkin(browser);61 return result;62 }63
64 async close(): Promise<void> {65 this.closed = true;66 const toClose = this.idle.splice(0, this.idle.length);67 await Promise.all(toClose.map((browser) => browser.close().catch(() => {})));68 this.launched = 0;69
70 // Anything still queued for a slot would otherwise hang forever — no71 // browser is ever coming, since checkout() below now refuses new work.72 const stuck = this.waiters.splice(0, this.waiters.length);73 for (const waiter of stuck) {74 waiter.reject(new Error('BrowserPool closed while a checkout was still queued.'));75 }76 }77
78 private async checkout(): Promise<T> {79 if (this.closed) {80 throw new Error('BrowserPool is closed.');81 }82
83 const idle = this.idle.pop();84 if (idle) return idle;85
86 if (this.launched < this.size) {87 return this.launchAndAccount();88 }89
90 return new Promise<T>((resolve, reject) => {91 this.waiters.push({ resolve, reject });92 });93 }94
95 private checkin(browser: T): void {96 if (this.closed) {97 void browser.close().catch(() => {});98 return;99 }100 const waiter = this.waiters.shift();101 if (waiter) {102 waiter.resolve(browser);103 return;104 }105 this.idle.push(browser);106 }107
108 private async retire(browser: T): Promise<void> {109 await browser.close().catch(() => {});110 this.launched -= 1;111 if (this.closed) return;112
113 const waiter = this.waiters.shift();114 if (!waiter) return;115
116 try {117 const fresh = await this.launchAndAccount();118 waiter.resolve(fresh);119 } catch (error) {120 waiter.reject(error);121 }122 }123
124 /** Reserves a pool slot and launches into it, releasing the slot again if the launch fails. */125 private async launchAndAccount(): Promise<T> {126 this.launched += 1;127 try {128 return await this.launch();129 } catch (error) {130 this.launched -= 1;131 throw error;132 }133 }134}1import { errors, type Page } from 'playwright';2import { defaultLauncher, type BrowserLauncher } from './browserLauncher.js';3import { BlockedRequestError, isChallengeShell, readPageContent } from './challengeShell.js';4import { PrivateProfileError, ProfileNotFoundError, type ProfileFetcher, type ProfileListing } from './profileFetcher.js';5import type { PlaywrightProxy } from './proxy.js';6import type { ContentType } from './types.js';7
8const PAGE_LOAD_TIMEOUT_MS = 20_000;9// Matches BrowserItemFetcher's findEmbeddedMedia depth cap (browserFetcher.ts) — kept in10// sync manually since both scan the same bootstrap structure but can't share code across11// the page.evaluate() serialization boundary (see findEmbeddedProfileAndTimeline's doc comment).12const MAX_EMBEDDED_SCRIPT_SCAN_DEPTH = 60;13
14/**15 * Discovers a public profile's posts/reels by reading the16 * `xig_user_by_username.polaris_ordered_timeline_connection` Instagram17 * embeds server-side on the profile page — the same embedded-script18 * technique `BrowserItemFetcher` uses for single items (see its doc19 * comment). Only the initial embed is read; a live Playwright probe20 * (2026-08-17) confirmed that requesting a further page while logged out21 * gets Instagram's backend to respond with an explicit "Unauthorized22 * logged out query" GraphQL error, not more data, so `has_next_page: true`23 * on the initial embed is treated directly as `limitReached` rather than24 * attempting (and always failing) a real next-page fetch. See ticket 03's25 * Corrections note.26 *27 * This connection's nodes only carry enough to build a permalink28 * (`code`/`product_type`) — full item detail (caption, timestamp, likes,29 * media URL) is fetched per item by the caller via the existing30 * `ItemFetcher` pipeline, not by this class.31 *32 * No unit-testable seam inside this class for the actual fetch logic;33 * `ProfileFetcher` is the seam, exercised by manual/live verification like34 * `BrowserItemFetcher`. The browser-launch step is injectable (see35 * `launcher` below) so ticket 04's proxy wiring can be asserted by a test36 * without a real browser or real proxy — same as `BrowserItemFetcher`.37 */38export class BrowserProfileFetcher implements ProfileFetcher {39 private readonly proxy?: PlaywrightProxy;40 private readonly launcher: BrowserLauncher;41
42 constructor(options: { proxy?: PlaywrightProxy; launcher?: BrowserLauncher } = {}) {43 this.proxy = options.proxy;44 this.launcher = options.launcher ?? defaultLauncher;45 }46
47 async fetchProfileListing(username: string): Promise<ProfileListing> {48 const browser = await this.launcher(this.proxy ? { proxy: this.proxy } : undefined);49
50 try {51 const page = await browser.newPage();52 return await fetchListingFromProfilePage(page, username);53 } finally {54 await browser.close();55 }56 }57}58
59async function fetchListingFromProfilePage(page: Page, username: string): Promise<ProfileListing> {60 const url = `https://www.instagram.com/${encodeURIComponent(username)}/`;61
62 try {63 await page.goto(url, { waitUntil: 'networkidle', timeout: PAGE_LOAD_TIMEOUT_MS });64 } catch (error) {65 if (!(error instanceof errors.TimeoutError)) throw error;66 }67
68 // Instagram usernames are case-insensitive; the embedded node carries the69 // canonical (lowercase) form regardless of how the input was cased.70 const found = await page.evaluate(findEmbeddedProfileAndTimeline, {71 targetUsername: username.toLowerCase(),72 maxDepth: MAX_EMBEDDED_SCRIPT_SCAN_DEPTH,73 });74
75 if (!found.profile) {76 // Embed scan came up empty — check title/body only now, so a real page77 // that just hasn't finished rendering never gets misclassified while78 // its embedded profile data was findable all along (same reasoning as79 // BrowserItemFetcher's identical reordering).80 const content = await readPageContent(page);81 if (isChallengeShell(content)) {82 throw new BlockedRequestError(`Instagram served an anti-bot challenge shell for ${username}`);83 }84 throw new ProfileNotFoundError(`Could not find profile data embedded on the page for ${username}`);85 }86 if (found.profile.isPrivate) {87 throw new PrivateProfileError(`This profile is private: ${username}`);88 }89 if (!found.connection) {90 throw new ProfileNotFoundError(`Could not find a posts/reels connection embedded on the page for ${username}`);91 }92
93 return {94 items: found.connection.edges.map(({ code, contentType }) => ({ shortcode: code, contentType })),95 limitReached: found.connection.hasNextPage,96 };97}98
99interface EmbeddedScanArgs {100 targetUsername: string;101 maxDepth: number;102}103
104interface EmbeddedScanResult {105 profile: { pk: string; isPrivate: boolean } | null;106 connection: { hasNextPage: boolean; edges: Array<{ code: string; contentType: ContentType }> } | null;107}108
109/**110 * Runs inside the browser page (via `page.evaluate`). Instagram splits the111 * profile's `is_private` flag and its `polaris_ordered_timeline_connection`112 * across two different embedded script blocks (confirmed via live capture —113 * they never appear on the same node), so this scans twice: once to find the114 * profile node matching the requested username, once more to find the115 * connection node sharing that profile's `pk` — matching by `pk` (rather116 * than returning the first connection-shaped object found anywhere on the117 * page) guards against accidentally picking up an unrelated connection, e.g.118 * from a "suggested accounts" rail.119 */120function findEmbeddedProfileAndTimeline(args: EmbeddedScanArgs): EmbeddedScanResult {121 function scan<T>(matcher: (record: Record<string, unknown>) => T | null): T | null {122 function findMatch(value: unknown, depth: number): T | null {123 if (depth > args.maxDepth || value === null || typeof value !== 'object') return null;124 if (Array.isArray(value)) {125 for (const item of value) {126 const found = findMatch(item, depth + 1);127 if (found) return found;128 }129 return null;130 }131 const record = value as Record<string, unknown>;132 const direct = matcher(record);133 if (direct) return direct;134 for (const key of Object.keys(record)) {135 const found = findMatch(record[key], depth + 1);136 if (found) return found;137 }138 return null;139 }140
141 const scripts = Array.from(document.querySelectorAll('script[type="application/json"]'));142 for (const script of scripts) {143 let parsed: unknown;144 try {145 parsed = JSON.parse(script.textContent || '');146 } catch {147 continue;148 }149 const match = findMatch(parsed, 0);150 if (match) return match;151 }152 return null;153 }154
155 const profile = scan((record) => {156 if (typeof record.username !== 'string' || typeof record.is_private !== 'boolean' || typeof record.pk !== 'string') {157 return null;158 }159 if (record.username.toLowerCase() !== args.targetUsername) return null;160 return { pk: record.pk, isPrivate: record.is_private };161 });162
163 if (!profile) return { profile: null, connection: null };164
165 const connection = scan((record) => {166 if (record.pk !== profile.pk) return null;167 const conn = record.polaris_ordered_timeline_connection as168 | { edges?: Array<{ node?: Record<string, unknown> }>; page_info?: { has_next_page?: unknown } }169 | undefined;170 if (!conn || !Array.isArray(conn.edges)) return null;171 const edges = conn.edges172 .filter((edge): edge is { node: Record<string, unknown> } => typeof edge.node?.code === 'string')173 .map((edge) => ({174 code: edge.node.code as string,175 // Duplicated from mapXigPolarisMedia.ts's identical contentType derivation — this176 // runs inside page.evaluate() and can't import across that serialization boundary.177 // Keep both in sync if Instagram adds a new product_type.178 contentType: (edge.node.product_type === 'clips' ? 'reel' : 'post') as ContentType,179 }));180 return { edges, hasNextPage: conn.page_info?.has_next_page === true };181 });182
183 return { profile, connection };184}1import type { Page } from 'playwright';2
3export interface PageContent {4 title: string;5 bodyText: string;6}7
8/**9 * Reads a page's title/visible body text for challenge-shell detection.10 * Callers must only consult this after an embed scan has already come up11 * empty — the SPA's placeholder title/empty body can transiently match a12 * real, still-hydrating page, so checking eagerly would misclassify a13 * page that just hasn't finished rendering yet as blocked.14 */15export async function readPageContent(page: Page): Promise<PageContent> {16 const title = await page.title().catch(() => '');17 const bodyText = await page18 .locator('body')19 .innerText()20 .catch(() => '');21 return { title, bodyText };22}23
24/**25 * Detects Instagram's anti-bot challenge shell: a bare, unrendered SPA page26 * with no server-embedded content, served instead of the real permalink27 * embed the Actor's fetchers read. Captured live 2026-08-26 via plain28 * `curl` against a known-good post and profile permalink — title exactly29 * "Instagram", empty visible body once script/style content is excluded —30 * consistent with the 2026-08-12 finding that plain HTTP never gets real31 * data (spec.md's Scraping technique decision).32 *33 * Not independently reproduced against a *rate-limited real browser*34 * session: 310 live requests (80 sequential + 230 parallel) against a35 * Playwright/Chrome session did not trigger a block to compare the shell36 * against, so this is best-effort/confirmed only for the plain-HTTP case —37 * same distinction as ticket 03's private-profile path and ticket 04's38 * proxy-wiring verification.39 */40export function isChallengeShell({ title, bodyText }: PageContent): boolean {41 return title.trim() === 'Instagram' && bodyText.trim() === '';42}43
44export class BlockedRequestError extends Error {}1export class ItemNotFoundError extends Error {}2
3export interface ItemFetcher {4 fetchRawItem(url: string): Promise<unknown>;5}1import { mapXigPolarisMedia, UnrecognizedMediaShapeError } from './mappers/xigPolarisMedia.js';2import type { ScrapedItem } from './types.js';3
4export { UnrecognizedMediaShapeError as UnrecognizedItemShapeError };5
6export function normalizeRawItem(raw: unknown): ScrapedItem {7 return mapXigPolarisMedia(raw);8}1import type { ContentType } from './types.js';2
3export class PrivateProfileError extends Error {}4export class ProfileNotFoundError extends Error {}5
6export interface ProfileListingItem {7 shortcode: string;8 contentType: ContentType;9}10
11export interface ProfileListing {12 items: ProfileListingItem[];13 limitReached: boolean;14}15
16export interface ProfileFetcher {17 fetchProfileListing(username: string): Promise<ProfileListing>;18}1export interface PlaywrightProxy {2 server: string;3 username?: string;4 password?: string;5}6
7interface ProxyInfoLike {8 hostname: string;9 port: number | string;10 username?: string;11 password: string;12}13
14export function toPlaywrightProxy(proxyInfo: ProxyInfoLike): PlaywrightProxy {15 const server = `http://${proxyInfo.hostname}:${proxyInfo.port}`;16
17 if (!proxyInfo.password) {18 return { server };19 }20
21 return { server, username: proxyInfo.username, password: proxyInfo.password };22}1export type ContentType = 'post' | 'reel';2export type MediaType = 'image' | 'video' | 'carousel';3
4export interface ScrapedItem {5 shortcode: string;6 ownerUsername: string | null;7 contentType: ContentType;8 caption: string | null;9 timestamp: number;10 likeCount: number | null;11 commentCount: number | null;12 mediaType: MediaType;13 mediaUrl: string | null;14}1# 01 — Scaffold the Actor project2
3**What to build:** A running, empty-but-valid Apify Actor project (Node.js + Crawlee + Apify SDK) that boots, accepts a profile-username-or-post/reel-URL input, and can push to a local Dataset under Apify SDK's local storage emulation. No real scraping logic yet — this ticket exists to make the pipeline and test harness real so every later ticket can build and test against it rather than bootstrapping from scratch.4
5**Blocked by:** None — can start immediately6
7**Status:** ready-for-agent8
9- [ ] Actor project initializes via the Apify SDK (`Actor.init()`/`Actor.main()`) and exits cleanly on an empty run10- [ ] INPUT_SCHEMA declares a `username` field and a `postOrReelUrl` field (at least one required)11- [ ] A test harness exists that runs the Actor entrypoint against Apify SDK's local storage emulation and can assert on local Dataset contents, with no real Apify Cloud run and no real network call to Instagram12- [ ] Crawlee is wired in with a placeholder request handler (no real Instagram calls yet) so the hybrid direct-API/browser-automation-fallback routing point exists for ticket 02 to fill in13- [ ] README stub and `actor.json` exist with placeholder metadata (title, description) — full pricing/disclaimer content is deferred to ticket 0514
15## Comments1# 02 — Single post/reel URL lookup2
3**What to build:** Given a post or reel URL as input, the Actor fetches that single item via a browser-driven request (Playwright, loading the permalink directly and reading the data Instagram embeds server-side), normalizes it into the shared output schema, pushes it to the Dataset, and emits the corresponding pay-per-event charge. This is the first real end-to-end scrape, and it establishes the request layer, normalization shape, and charging pattern that tickets 03 and 04 reuse.4
5**Blocked by:** 01 — Scaffold the Actor project6
7**Status:** ready-for-agent8
9- [ ] Given a valid public post URL, the Dataset receives one item matching the shared output schema (shortcode, owner username, content type = post, caption, timestamp, like count, comment count, media type, media URL)10- [ ] Given a valid public reel URL, the Dataset item matches the shared output schema (content type = reel) — **no view count or duration required in v1**11- [ ] Fetching is browser-driven (Playwright) as the default and only path for v1; direct-HTTP is out of scope for this ticket12- [ ] A pay-per-event charge is emitted once per successfully scraped item, tagged by content type13- [ ] Given a URL pointing at a deleted/nonexistent post or reel, the run completes with a clear error/empty-with-reason result rather than a silent empty Dataset or an unhandled crash14- [ ] Tests use the confirmed seam (Actor input → real pipeline → Dataset output) with the fetcher injected/stubbed via the real captured fixtures — no live network calls in the test suite15
16## Comments17
18**2026-08-12 — Corrections from live capture**, superseding the original AC2/AC3 wording (the checkboxes above already reflect these):19
20- **Fetch strategy is inverted from the original plan.** Plain HTTP requests (no browser) get Instagram's anti-bot challenge shell, not real data — confirmed via multiple direct attempts. A real browser session succeeds. So **browser-driven fetching (Playwright) is the primary path for v1**, not a fallback. A lighter direct-HTTP path (replicating the session cookies/CSRF token a browser sends) is a possible *future* optimization if that request recipe is captured later — it is explicitly not part of this ticket's scope.21- **`view_count`, `play_count`, and `duration` are dropped from required reel output.** Confirmed absent from the anonymous single-item reel query — not an extraction gap, the fields aren't present in the response at all.22- **One schema, not two.** An earlier pass through this correction claimed posts and reels use two structurally different schemas. Further verification (structured `JSON.parse` + recursive search, not raw substring matching, which was misleading due to other items' data sitting nearby on the same page) showed that's wrong for the fetch strategy actually used: on **direct navigation** to a permalink (what the browser-driven fetcher does), both posts and reels come back as the same `xig_polaris_media` shape, differentiated by `media_type`/`product_type` fields — not by two schemas. One mapper handles both. (The two-schema observation was real but came from a different code path — client-side SPA link-clicks within an already-loaded page — that the Actor doesn't use.)23- **Live-verified end-to-end** against real URLs (a real reel, a real carousel post, and a real deleted post) after the above correction — fetch → normalize produced correct output for all three, including a genuinely playable media URL and a correctly-thrown not-found error.24- Verified fixtures (real field names/values, captured live) are at `test/fixtures/` — see `test/fixtures/README.md` for exactly what's confirmed vs. best-effort in each one.25
26**2026-08-12 — Code review fixes applied:**27
28- Docker base image switched to `apify/actor-node-playwright-chrome:20` (both build stages) — the plain `apify/actor-node:20` image has no Chromium binary or system libraries, so the deployed Actor could not have launched a browser at all.29- Not-found detection requires both the page title (`"isn't available"`) and body text (`"link may be broken"`) to match, rather than title alone — a real post whose own caption happens to contain the phrase "isn't available" would otherwise false-positive as deleted.30- `page.goto()` failures are now only silently tolerated when they're a Playwright `TimeoutError`; any other navigation failure (DNS, connection reset, proxy failure) now propagates instead of being misreported as "item not found."31- The mapper now requires `if_not_gated_logged_out` to be present with a numeric `taken_at` before mapping, rather than defaulting missing data to `{}`/`null`/`0` and pushing a near-empty item as if it were a successful scrape.32- `extractShortcode()` now throws `InvalidInputError` (consistent with the rest of input validation) instead of a bare `Error`.1# 03 — Profile posts + reels scraping2
3**What to build:** Given a public profile username, the Actor scrapes that profile's posts and reels (paginating as needed) and pushes each as a Dataset item, reusing the output schema, request layer, and charging pattern established in ticket 02.4
5**Blocked by:** 02 — Single post/reel URL lookup6
7**Status:** ready-for-agent8
9- [ ] Given a public username with both posts and reels, the Dataset receives one item per post and per reel, each correctly tagged by content type and matching the shared schema (reusing ticket 02's per-item fetch/normalize pipeline)10- [ ] Pagination continues until the profile's available public posts/reels are exhausted, **or until the anonymous-access pagination limit is hit** — in the latter case the run still completes successfully with whatever was collected, and reports a clear status message that results may be incomplete, rather than silently truncating11- [ ] A pay-per-event charge is emitted once per successfully scraped item, same pattern as ticket 0212- [ ] Given a private profile, the run completes with a clear error/empty-with-reason result rather than a silent empty Dataset13- [ ] Given a public profile with zero posts/reels, the run completes with an explicit empty result distinguishable from an error14- [ ] Tests use the same confirmed seam as ticket 02, with canned network responses covering the profile-listing endpoint(s)15
16## Comments17
18**2026-08-17 — Corrections from live capture**, superseding the original AC1/AC2 wording (the checkboxes above already reflect these):19
20- **The profile-listing connection is a different, lighter shape than the single-item permalink shape ticket 02 built against.** Live capture against `instagram.com/nasa` (logged out) found the listing lives at `xig_user_by_username.polaris_ordered_timeline_connection` (cursor-paginated via `edges`/`page_info.end_cursor`/`page_info.has_next_page`), with posts and reels combined in one connection and differentiated by `product_type` (e.g. `"clips"` for reels). But each node only carries `pk`, `code`, `caption`, `display_uri`, `media_type`, `product_type`, `user{username}` — no `taken_at`, `like_count`, `comment_count`, or playable `video_versions`. It cannot fill the shared output schema on its own. **Decision:** use this connection only to discover `(shortcode, contentType)` pairs, then fetch each item's full detail by reusing ticket 02's `ItemFetcher`/`mapXigPolarisMedia` pipeline unchanged against its permalink — no new mapper needed.21- **Anonymous (logged-out) pagination is capped at the initial embed — there is no further page to fetch at all.** A first pass (browser UI scrolling) looked like it was loading a genuine second page before hitting a "Log in to continue watching" wall around ~20-24 items. A follow-up Playwright probe (same runtime the real fetcher uses) instrumented `page.on('response')` for `/api/graphql` calls while scrolling: none of the captured responses ever contained more `polaris_ordered_timeline_connection` data, and two came back as an explicit `{"errors":[{"message":"Unauthorized logged out query.","code":1675002}]}` — confirming the backend itself rejects a real next-page fetch while logged out, rather than the UI just choosing to nag. What looked like "one more page" in the browser was the already-embedded connection's items being progressively rendered, not new data. The initial embed reliably contained exactly 12 edges with `page_info.has_next_page: true` for a profile with many more items. **Implementation implication:** `BrowserProfileFetcher` only ever reads the single initial embed — no scrolling or pagination-request handling needed — and treats `has_next_page: true` on that embed directly as `limitReached: true`, since attempting a real next page is known to fail. This is a simpler implementation than originally planned, not a bigger one.22- **Decision (user-confirmed):** stop at the wall and report partial results with a clear reason (`actor.setStatusMessage`), rather than erroring the whole run or silently truncating. Profiles with fewer items than the initial embed provides are unaffected and scrape to completion as originally specified.23- The private-profile path (`is_private: true` on the embedded user node) is confirmed structurally — the field and its meaning are documented by Instagram's own client behavior — but wasn't live-verified end-to-end against a known private account (none was available to test against). Flagged here at the same confidence level ticket 02's fixtures README used for best-effort vs. confirmed paths.24- `is_private` is present directly on the profile's bootstrap user node (`xig_user_by_username.is_private`) — private-profile detection needs no extra live-verification beyond what's already captured.1# 04 — Optional bring-your-own proxy configuration2
3**What to build:** An end user can supply their own proxy configuration as Actor input; when present, all outbound requests (both the direct API path and the browser-automation fallback) route through it. When absent, the Actor runs proxy-less exactly as it does in ticket 02/03's baseline. No bundled/paid proxy is ever used by default — this keeps proxy cost, if any, entirely on the end user.4
5**Blocked by:** 02 — Single post/reel URL lookup6
7**Status:** ready-for-agent8
9- [ ] INPUT_SCHEMA gains an optional proxy configuration field (following Apify's standard proxy configuration input convention)10- [ ] When a proxy configuration is supplied, requests in a test run are verifiably routed through it (asserted via the browser-launch call receiving the expected proxy option)11- [ ] When no proxy configuration is supplied, behavior is unchanged from ticket 02/03's baseline12- [ ] README/documentation notes this as an explicit, known limitation: omitting proxy configuration means a higher block/ban rate is possible13
14## Comments15
16**2026-08-17 — Correction, superseding the original AC1 wording:** "both the direct API path and the browser-automation fallback" referred to an architecture ticket 02 abandoned (direct-HTTP requests get Instagram's anti-bot challenge shell, not real data — see ticket 02's Corrections note). There is only one request path in this project: browser-driven via Playwright. The checkbox above already reflects this — proxy wiring applies to `chromium.launch()`'s `proxy` option in `BrowserItemFetcher`/`BrowserProfileFetcher`, nothing else. AC2's wording was also adjusted: there's no mocked network layer in this project's test suite (per ticket 02/03 precedent, the browser-driven fetchers have no unit-testable network seam) — instead, both fetcher classes gain an injectable browser-launcher function (defaulting to the real `chromium.launch`), and a test asserts the launcher is called with the correct `proxy` option. This proves the wiring is correct without needing a real proxy or real network calls; it does not live-verify actual traffic routing through a live proxy, since no test proxy was available (same best-effort/confirmed distinction used for ticket 03's private-profile path).1# 05 — Store listing readiness2
3**What to build:** The Actor is ready to publish to the Apify Store: pay-per-event pricing configured per content-type event, a README documenting inputs/outputs/limitations/maintenance stance, and a ToS/liability disclaimer. This is the final v1 ticket — after it, the Actor can actually be listed and sold.4
5**Blocked by:** 02 — Single post/reel URL lookup, 03 — Profile posts + reels scraping, 04 — Optional bring-your-own proxy configuration6
7**Status:** ready-for-agent8
9- [ ] Pay-per-event pricing is configured in the Actor's pricing metadata for each event emitted by tickets 02 and 03, priced below the lowest current comparable competitor identified during scoping10- [ ] README documents: required/optional inputs, output schema, the anonymous-only v1 auth model, the optional BYO-proxy input, and the best-effort ("ship it, revisit occasionally") maintenance stance11- [ ] A standard ToS/liability disclaimer (lawful/research use only, used at runner's own risk, no ToS-compliance guarantee) is present on the listing12- [ ] `actor.json`/store metadata accurately reflects only the v1 feature set (no references to Stories/Highlights, transcripts, or follower stats, which are out of scope for this spec)13
14## Comments15
16**2026-08-26**: Verified current state before starting the agent:17
18- **AC1 wording corrected.** The spec's original phrasing ("post scraped," "reel scraped," "URL lookup") implied three chargeable events. The actual implementation (`src/actor.ts::pushScrapedItem`) only ever emits `post-scraped` or `reel-scraped` via `actor.pushData(item, `${item.contentType}-scraped`)` — a single-URL lookup charges whichever of those two matches the looked-up item's content type; there is no separate "URL lookup" event. AC1 should read: configure Apify Store pricing for the two events `post-scraped` and `reel-scraped`, each priced below the lowest current comparable competitor.19- **AC1 and AC3 are partly outside what a coding ticket can deliver.** Traced through `node_modules/apify` — pay-per-event pricing is not a repo file; it's configured on the Apify Console's Publication/Monetization tab at actor-version-publish time, keyed by the event-name strings the code already calls (`post-scraped`/`reel-scraped`). Likewise, a ToS/liability disclaimer "present on the listing" is a Console-side Store listing concern; the README does get pulled in as the Store listing description, so drafting the disclaimer text there covers the content, but actually attaching pricing numbers and confirming the listing is a manual publish-time step (Console UI), not something `/implement`'s TDD/code-review loop can do inside this git repo. **Scope for `/implement`:** draft the disclaimer text in the README, and document the two event names clearly enough (e.g. in the README's pricing note) that whoever publishes can fill in the Console pricing form correctly. The actual Console pricing entry + publish action is a separate manual step afterward (candidate for the `wizard` skill when the user is ready to actually publish) — not part of this ticket's automated work.20- **AC4 already satisfied.** Grepped `.actor/actor.json`, `.actor/input_schema.json`, and `README.md` for "stories", "highlights", "transcript", "follower" — no matches. No stale out-of-scope references exist to remove.21- **AC2 (README pass) is the ticket's main real deliverable.** Confirmed the README is badly stale: still says "Status: scaffolding only (ticket 01). No scraping logic exists yet," despite tickets 02–04 being fully implemented and committed. Needs a full rewrite covering required/optional inputs (`username` XOR `postOrReelUrl`, optional `proxyConfiguration`), the actual output schema (shortcode/id, owner username, content type, caption, timestamp, like/comment count, media type, media URL(s)), the anonymous-only v1 auth model, the BYO-proxy input (already partially documented), and the best-effort maintenance stance.1# 06 — Distinguish blocked requests from genuinely missing content2
3**What to build:** `BrowserItemFetcher` and `BrowserProfileFetcher` currently treat "Instagram served an anti-bot challenge shell" and "this post/profile is genuinely gone" as the same outcome — both fall through to the same not-found path. Add explicit detection of the challenge-shell case (distinct markers from the genuine "isn't available" / private-profile pages already handled) and surface it as a distinct error type from `ItemNotFoundError`/`ProfileNotFoundError`, so callers — and every reliability ticket that follows this one — can tell "retry this" from "this doesn't exist."4
5**Why this comes first:** every v2 reliability change (proxy, session reuse, fingerprinting — ticket 07) is otherwise unmeasurable: without this, a drop in effective block rate and a rise in "content not found" look identical in the dataset output. This also directly implements spec.md's v2 story 27.6
7**Blocked by:** none (builds on 02/03's existing fetchers)8
9**Status:** ready-for-agent10
11- [ ] A new error type (e.g. `BlockedRequestError`) is thrown by both `BrowserItemFetcher.fetchRawItem` and `BrowserProfileFetcher.fetchProfileListing` when the loaded page matches Instagram's anti-bot challenge shell, distinct from `ItemNotFoundError`/`ProfileNotFoundError`/`PrivateProfileError`12- [ ] The challenge-shell markers are captured from a live probe (title/body text or DOM structure unique to the challenge page, not shared with the genuine "isn't available" page) and documented in a code comment with the capture date, matching this project's existing live-verification convention (see `browserFetcher.ts`'s and ticket 03's doc comments)13- [ ] Existing genuinely-missing-content and private-profile paths are unaffected — their tests (from tickets 02/03) still pass unchanged14- [ ] A test exercises the new blocked-detection branch using a canned challenge-shell HTML fixture, asserting `BlockedRequestError` (not `ItemNotFoundError`) is thrown15- [ ] A regression test confirms a `BlockedRequestError` mid-profile-scrape propagates out of `processRun` rather than being silently swallowed (exact retry behavior is ticket 07's concern, this ticket only needs the distinction to propagate correctly)16
17## Comments18
19**2026-08-26:** AC5's original wording ("`scrapeProfile` is updated so a `BlockedRequestError`... does not get silently swallowed") assumed a code change would be needed. Checked `actor.ts::scrapeProfile`'s current catch block:20
21```ts22if (!(error instanceof ItemNotFoundError) && !(error instanceof UnrecognizedItemShapeError)) throw error;23```24
25This already re-throws anything that isn't one of those two named types — a new `BlockedRequestError` type would propagate correctly with zero changes to `actor.ts`, both from the per-item loop and from the unwrapped `profileFetcher.fetchProfileListing()` call before it (same reasoning as ticket 05's finding that `PrivateProfileError` already propagates there). Reworded the AC to ask for a regression test locking this in, rather than a code change that isn't actually needed — avoids the implementing agent adding a redundant special-case.26
27**2026-08-26 — AC2's live-probe result, narrower than written:** attempted to trigger Instagram's anti-bot challenge shell against a real Playwright/Chrome browser session to capture its exact markers — 80 sequential page loads across 10 well-known public profiles, then 230 more fired in parallel bursts (310 requests total). None triggered a block; Instagram's rate-limiting threshold for this request pattern is higher than what's reasonable to keep probing for (further escalation means hammering a third party's servers harder, with real IP-reputation cost to this project's own dev/runtime environment).28
29Fell back to the *other* already-documented anti-bot response this project has real evidence for: plain HTTP requests (no browser), which ticket 02's Corrections note and `spec.md`'s 2026-08-12 finding both already established get an anti-bot challenge shell instead of real data. Captured via `curl` against a known-good post permalink (`instagram.com/p/DcRPo4JuQKe/`, the item the Actor's own first live Store run successfully scraped) and a known-good profile permalink (`instagram.com/nasa/`): both return a bare, unrendered SPA shell — `<title>Instagram</title>` exactly, and zero visible body text once `<script>`/`<style>` content is excluded. Confirmed disjoint from the genuine "isn't available" markers already in use (`title` containing `"isn't available"`, body containing `"link may be broken"`) and from any real rendered page's title/body.30
31**What this means for AC2 as shipped:** the markers are real and live-captured, but only confirmed for the plain-HTTP case — not independently verified against what a *rate-limited real browser* session renders, since one couldn't be triggered. `BrowserItemFetcher`/`BrowserProfileFetcher` both execute JS via a real browser, so there's a real (unresolved) chance the production code path could hit a differently-shaped block page that this detection wouldn't catch. Shipping this as best-effort/confirmed-for-the-known-case detection rather than blocking the ticket on reproducing a browser-side block — same best-effort/confirmed distinction used for ticket 03's private-profile path and ticket 04's proxy-wiring verification. Revisit if ticket 07's real-world runs surface a block that this detection misses.1# 07 — Browser reuse, bounded concurrency, and blocked-session retirement2
3**What to build:** Replace `actor.ts`'s current one-Chromium-launch-per-item, strictly-sequential loop (`scrapeProfile`) with a small worker pool sharing a bounded number of browser instances/pages, retiring and relaunching only on a detected block (ticket 06's `BlockedRequestError`) rather than per item.4
5**Blocked by:** 06 — Distinguish blocked requests from genuinely missing content6
7**Status:** ready-for-agent8
9- [ ] Profile scraping uses a bounded pool of concurrent browser contexts (config value, not hardcoded to Instagram's rate limits blindly — pick a conservative default and make it tunable) instead of sequential per-item launch/close; a test asserts concurrency is bounded (e.g. by asserting the launcher is invoked at most N times concurrently, reusing the injectable-launcher seam from ticket 04)10- [ ] A `BlockedRequestError` on one item in a profile scrape triggers session/context retirement (drop that context, get a fresh one) rather than aborting the whole run or reusing a now-flagged context11- [ ] `BrowserItemFetcher`'s lifecycle changes from launch-and-close-per-call to a lazily-created shared browser with an explicit `close()`, so retirement is expressible ("drop the shared browser instance; the next call relaunches") — the `fetchRawItem(url): Promise<unknown>` signature stays byte-identical, satisfying the caller-visible `ItemFetcher` contract ticket 02 depends on12- [ ] Whatever now owns the fetcher's lifecycle (likely `main.ts`, since it constructs the fetcher and currently never closes anything) calls `close()` after `processRun` completes, on both the success and error paths13- [ ] Existing single-item lookup path (ticket 02) behavior is unchanged for the caller-visible contract (same `ItemFetcher` interface, same output shape)14
15## Comments16
17**2026-08-28:** Split this ticket. It originally bundled three changes (default proxy, browser reuse/concurrency/retirement, fingerprint injection) under one "reliability hardening" umbrella, but they're only narratively related, not implementation-coupled — except that retirement (the old AC4) is genuinely unimplementable without a context pool existing first (there's nothing long-lived to retire today), so reuse+concurrency+retirement had to stay one ticket. Proxy default is now ticket 08, fingerprint injection is ticket 09 — both are independently shippable and don't need this ticket's pool to exist first.18
19Also added the `BrowserItemFetcher` lifecycle change and the "who calls `close()`" touchpoint, neither of which the original ticket mentioned — without them, AC1/AC2 (pool + retirement) have no concrete implementation path given the existing launch-per-call design.20
21**2026-08-28 — Implementation notes:**22
23- **Pool design chosen: a genuine pool of up to `poolSize` reused browser *instances*** (`BrowserPool<Browser>` in `src/instagram/browserPool.ts`), not one shared browser with multiple concurrent pages. The ticket's own "What to build" line ("bounded number of browser **instances**/pages") and AC1's test hint ("launcher invoked at most N times **concurrently**") both point at multiple concurrent launcher calls, which only a real multi-instance pool produces. `BrowserPool` is generic and decoupled from Instagram specifics: it takes a plain `launch()`/`shouldRetire(error)` pair rather than importing `BlockedRequestError` itself, so pool logic (checkout/reuse/retirement/bounded-concurrency/close, including two easy-to-miss bugs — a failed launch leaking a permanently-reserved slot, and a waiter left pending forever if its replacement launch also fails) is unit-tested with a stub `launch` function and no real browser or Instagram-specific error type (`test/instagram/browserPool.test.ts`, 7 tests).24- **Dataset item order is no longer listing order.** Switching `scrapeProfile` from a sequential loop to `Promise.all`-fanned-out concurrent fetches means items land in the Dataset in completion order, not the profile's own order. This is a deliberate, accepted tradeoff — a profile timeline is newest-first, so it's a minor, user-visible regression not covered by any AC — documented in `actor.ts::scrapeProfile`'s doc comment rather than left implicit. Push still happens inside each item's own task (not collected and pushed at the end), since collect-then-push would silently break ticket 03 AC2's user-confirmed partial-results guarantee: an infra failure on a later item must not discard items already successfully pushed.25- **`BrowserProfileFetcher` (the once-per-run listing fetch) was deliberately left un-pooled** — ticket 07's ACs only cover the per-item loop, and a `BlockedRequestError` from the listing fetch itself still aborts the run (nothing to retire into; no partial per-item results exist yet at that point). Confirmed via a test (`actor.test.ts`) that this differs from a per-item block, which is now caught-and-skipped rather than propagated (superseding ticket 06's test of the same name, which correctly asserted propagation *before* retirement infrastructure existed to make skip-and-continue viable).26
27**2026-08-28 — Code review fixes:** the first pass used `Promise.all` in `scrapeProfile`, which rejects as soon as the first unrecoverable error hits without waiting for sibling item-tasks still mid-fetch or still queued for a pool slot — those stragglers would then race against `main.ts`'s `finally { await fetcher.close(); }`, and anything still queued for a slot would hang forever once the pool closed under it (`BrowserPool.close()` only drained idle browsers, never rejected queued waiters). Fixed on both sides: `scrapeProfile` now uses `Promise.allSettled` and only re-throws the first genuine failure after every task has settled, so `close()` never runs while the pool is still in use; and `BrowserPool.close()` now also rejects any queued waiters (with tests for both this and a checkout-after-close guard), as defense-in-depth for the class's own contract independent of how the current single caller happens to use it.1# 08 — Default Apify Proxy on2
3**What to build:** Flip the Actor's proxy behavior from opt-in (proxy-less unless the user supplies their own) to defaulting Apify Proxy (RESIDENTIAL group) on for any run that doesn't explicitly override it, per spec.md's v2 pivot away from the zero-spend constraint. BYO proxy remains a supported override (ticket 04's wiring is unchanged; only the default changes).4
5**Blocked by:** none (independent of ticket 07's pool work)6
7**Status:** closed — not viable, see 2026-09-10 Comments entry8
9- [ ] `input_schema.json`'s `proxyConfiguration` property gets a `default` value turning Apify Proxy on (e.g. `{"useApifyProxy": true, "apifyProxyGroups": ["RESIDENTIAL"]}`) — **not** `prefill`, which is UI-only and never reaches the Actor via the API, CLI, scheduler, or any run that omits the key entirely (confirmed against Apify's input-schema spec: prefill "does not affect the Actor functionality and API"). A run triggered via the API with the input key entirely absent must still route through Apify Proxy.10- [ ] `main.ts`'s existing guard (`input?.proxyConfiguration ? await actor.createProxyConfiguration(...) : undefined`) is re-examined against the new `default`: since the platform populates `input.proxyConfiguration` from the schema's `default` when the caller omits it, the guard's existing truthiness check should already do the right thing once `default` (not `prefill`) is set — but this must be verified, not assumed, since that guard's comment currently documents a zero-spend invariant the v2 spec pivot has superseded. Update the comment regardless so it doesn't keep asserting a constraint that's no longer true.11- [ ] `README.md`'s input section and "Known limitation: no bundled proxy" section are rewritten — proxy is no longer "runs proxy-less by default at no extra cost"; it's now on by default and billed to whoever runs the Actor, with BYO/opt-out still available12- [ ] `input_schema.json`'s proxy field description text is updated to match13
14## Comments15
16**2026-08-28:** Split out of the original combined "reliability hardening" ticket (now ticket 07, which keeps only browser reuse/concurrency/retirement) — this is independently shippable and has no dependency on ticket 07's pool work.17
18**Do not `apify push` this to the live Store listing until this ticket and pricing (a later ticket, number not yet assigned) land together.** The Actor is already live and published; its current README explicitly states it "never uses a paid proxy by default." Shipping default-on proxy without updating both the README and the per-event pricing (to absorb the new proxy cost) leaves the live listing's claims false and the daily autotest run incurring a real, unpriced cost.19
20**2026-08-31 — Implemented, with code review fixes:**21
22- `input_schema.json`'s `proxyConfiguration` now carries `default: {useApifyProxy: true, apifyProxyGroups: ["RESIDENTIAL"]}` (kept `prefill` too, matching value, for the Console UI). `main.ts`'s guard comment rewritten to document the flipped intent rather than the superseded zero-spend one.23- **Code review caught a real crash risk**: the original implementation had no error handling around `createProxyConfiguration`. Traced the SDK source (`node_modules/apify/dist/proxy_configuration.js`'s `_checkAccess`): on the real platform, a failed proxy-access check *throws*, not warns — so any caller whose account can't use the RESIDENTIAL group (e.g. lacks the required paid plan tier) would have the *entire run hard-fail* with the proxy default on, where v1 would have quietly succeeded proxy-less. Fixed with a try/catch around proxy setup: on failure, log a status message explaining why and continue proxy-less, rather than crash. This preserves "the Actor still works" as the floor regardless of the caller's proxy access.24- **Code review also caught an overclaim in my own doc comment.** I'd written "confirmed against Apify's input-schema spec" for the claim that a schema `default` reaches the Actor at runtime for an omitted key — true for the general spec text, and now also live-verified for one property (`username`, via the 2026-08-31 autotest-input fix: a real triggered run with the key omitted correctly received `"nasa"`, proven by the run getting past validation and attempting to scrape it). But that's a plain string property; `proxyConfiguration` is object-typed with the special `"editor": "proxy"` rendering, and whether the platform merges `default` the same way for that specific shape has **not** been independently confirmed. Softened the comment to say so explicitly, and flagged it as something to check on this ticket's first live run (once pricing lands and it actually gets pushed) — if the assumption is wrong, the failure mode is silent (stays proxy-less, matching v1 behavior) rather than a crash, per the try/catch fix above.25- README's Pricing section didn't mention the new proxy cost despite the Proxy section describing it as separate/billed — added a one-line cross-reference so a reader who only reads the Pricing section isn't misled, without inventing specific proxy-cost numbers (those belong to the later pricing ticket).26
27**2026-09-01 — Live incident: RESIDENTIAL tunnel failures.** First live run after this ticket shipped failed with `page.goto: net::ERR_TUNNEL_CONNECTION_FAILED`, a Chromium-level proxy-tunnel CONNECT failure distinct from ticket 06's anti-bot block detection, occurring *after* the RESIDENTIAL access-check try/catch (added above) had already succeeded. Traced `proxy.ts` against the Apify SDK's `proxy_configuration.js` and confirmed this project's hostname/port/credential wiring matches the SDK's documented contract exactly — ruled out our own code.28
29**2026-09-01 — Switched RESIDENTIAL → default datacenter pool.** Hypothesized a RESIDENTIAL-specific access limitation; switched `apifyProxyGroups: ["RESIDENTIAL"]` to omitting the groups key entirely (datacenter pool, also cheaper: $1/IP vs $8/GB). Re-tested live: **identical failure**, same error, same URL pattern. This ruled out RESIDENTIAL specifically — the problem is proxy-vs-no-proxy, not which group.30
31**2026-09-02 — Reverted default to proxy-less.** The Actor was failing on every run with proxy on (worse than the pre-ticket-08 baseline, which at least sometimes succeeded), so reverted `useApifyProxy` to `false` by default to restore a working live Actor while the tunnel failure was diagnosed offline. This is the config still in place as of ticket closure.32
33**2026-09-02 — Local diagnosis: our code and `channel:'chrome'` are not the cause.** Built a throwaway local HTTP CONNECT proxy with Basic auth (no `mitmproxy`/`squid` available) and tested three scenarios against it with real Playwright/Chrome:34- `channel: 'chrome'` + authenticated proxy → **succeeded**. Full 407-challenge → retry-with-credentials → 200-tunnel-established flow, real page load, dozens of resources through the tunnel.35- `channel: 'chrome'` + no credentials against an auth-required proxy → failed with `ERR_INVALID_AUTH_CREDENTIALS` — a **different** error from the live failure, ruling out a simple "missing credentials" explanation.36- Bundled Chromium: couldn't test locally (browser revision not installed on the dev machine either — unrelated to this investigation).37
38This conclusively ruled out a general Chrome+Playwright+authenticated-proxy incompatibility: the mechanism works correctly in principle. Whatever's failing is specific to the live Apify platform environment or Apify Proxy's actual server behavior, neither of which could be reproduced locally.39
40**2026-09-10 — Diagnostic re-test: third distinct failure, confirms this isn't fixable at the code level.** Added temporary `requestfailed`/`response` event logging (commit `2ddab13`, since removed) and triggered one live run with `proxyConfiguration: {useApifyProxy: true}` explicitly overridden via custom input (default stayed proxy-less throughout, so this didn't affect the autotest). Result: a **third** distinct error, `net::ERR_HTTP_RESPONSE_CODE_FAILURE` — different again from both prior failures — and **none of the diagnostic listeners fired at all**, meaning the failure occurred before Playwright ever received a usable `response` object. A real HTTP-level block (e.g. a 403 page) would have produced a normal `response` event with some status code; this didn't, consistent with a garbled/malformed response coming back through the tunnel rather than a clean rejection.41
42**Conclusion — closing this ticket as not viable, not "needs more debugging."** Three different failure signatures, across two different Apify Proxy groups (RESIDENTIAL and datacenter), none reproducible outside the live platform, with our own proxy-wiring code and Playwright's Chrome-proxy-auth handling both independently verified correct. The most plausible explanation: Instagram (Meta) actively detects and disrupts traffic from Apify's shared proxy IP pools specifically — a common defense among large platforms against widely-used proxy-as-a-service providers, whose IP ranges are easy to catalog given how many customers share them. This is not something fixable by retry logic, fingerprinting, or further code changes on our side; it's a property of the shared IP pool itself for this specific target site.43
44**Decision**: proxy-less stays the permanent default (already the case since the 2026-09-02 revert). BYO proxy (ticket 04) remains available and unaffected — a *non*-shared, non-Apify proxy was never tested and might behave differently, since the hypothesis is specifically about Apify's shared pool being recognized, not proxies in general. Ticket 10 (pricing reassessment for proxy cost) is now moot as scoped and should be closed or rescoped, since there's no proxy cost to absorb into default pricing anymore.1# 09 — Fingerprint injection2
3**What to build:** Wire `fingerprint-generator`/`fingerprint-injector` into the browser-driven fetch path so launched sessions carry a generated, consistent browser fingerprint instead of Playwright/Chrome's bare default, reducing fingerprint-based blocking.4
5**Blocked by:** none (independent of tickets 07 and 08)6
7**Status:** ready-for-agent8
9- [ ] `fingerprint-generator` and `fingerprint-injector` are added as **explicit direct dependencies** in `package.json`, not left as transitive Crawlee dependencies — the production Docker image runs `npm install --omit=dev --omit=optional`, and relying on a transitive resolution is the same class of version-drift risk that caused the real Playwright browser-binary bug fixed after this Actor's first live run (`5e68291`)10- [ ] Fingerprint injection attaches to the Playwright **`BrowserContext`**, not the browser-launch step: `fingerprint-injector`'s API (`attachFingerprintToPlaywright(browserContext, fingerprint)` / `newInjectedContext(browser, options)`) operates on a context, not launch options. `browserLauncher.ts`'s current `BrowserLauncher = (options?) => Promise<Browser>` seam is the wrong place to wire this in — injection belongs wherever a `BrowserContext`/page gets created from the launched browser11- [ ] A test asserts a launched session carries an injected fingerprint via the injectable seam (not live verification against Instagram) — the exact assertion shape depends on where context creation ends up living after ticket 07's pool work, so coordinate with whichever of 07/08/09 ships first12- [ ] Existing single-item lookup path (ticket 02) and profile-listing path (ticket 03) behavior are unchanged for their caller-visible contracts13
14## Comments15
16**2026-08-28:** Split out of the original combined "reliability hardening" ticket (now ticket 07) — independently shippable, no dependency on ticket 07's pool work or ticket 08's proxy default.17
18The original ticket's AC assumed the launcher (`BrowserLauncher`, which returns a `Browser`) was the right injection seam, asserting "the launcher is called with fingerprint-bearing context options." Checked `fingerprint-injector`'s actual type signatures (`node_modules/fingerprint-injector/fingerprint-injector.d.ts`): injection is a `BrowserContext`-level operation, not a launch-option. Reworded the AC accordingly rather than let an agent build against a seam that doesn't exist.1# 10 — Reassess per-event pricing to cover Apify Proxy cost2
3**What to build:** Now that ticket 08 has shipped and defaults Apify Proxy (RESIDENTIAL) on for every run, the current live per-event pricing (`post-scraped` and `reel-scraped`, both $0.001, confirmed via the Actor's own live pricing config) predates that cost entirely — it was set purely to undercut the cheapest comparable competitor under v1's zero-spend, proxy-less design. Reassess and update both events' prices so they cover the new proxy cost with reasonable margin, per spec.md's Monetization decision ("per-event pricing... to absorb it rather than assuming zero infra cost").4
5**Blocked by:** 08 — Default Apify Proxy on (must be live before real usage data exists to measure against)6
7**Status:** closed — moot, see 2026-09-10 Comments entry8
9- [ ] **Measure real proxy bandwidth per run, don't guess it.** Apify's own proxy pricing is billed per GB of traffic (confirmed via Apify's docs: "Pricing is based on data traffic... displayed on your proxy usage dashboard in Apify Console"), and no bandwidth-per-item figure exists anywhere in this project yet. Check Console → Proxy → usage dashboard against the runs already executed since ticket 08 shipped (2026-09-02 onward) to get a real GB-per-run figure for both a single-item lookup and a profile scrape. A plain `curl` fetch of one post permalink was 617KB (captured during ticket 06's investigation) — that is a floor, not an estimate, since a real browser session pulls down JS bundles/CSS/images/fonts on top of the base document, which a `curl` never sees.10- [ ] Using the measured GB/run figure and Apify's current RESIDENTIAL rate (confirmed: $8/GB on Free/Starter plans, $7.5/GB Scale, $7/GB Business — assume the Free-tier $8/GB rate, since that's what applies to whoever runs this Actor unless they're on a paid Apify plan), compute a per-event price for `post-scraped` and `reel-scraped` that covers proxy cost with reasonable margin — not just breaks even11- [ ] Console pricing is updated to the new numbers (same manual, Console-UI publish-time step as ticket 05's original pricing entry — not something committable to this repo; a candidate for the `wizard` skill when ready to actually change live prices)12- [ ] `README.md`'s Pricing section is updated with the new prices and, if useful, a one-line note on why they changed (proxy cost now included, not a competitor-driven repricing)13- [ ] `.scratch/instagram-scraper/spec.md`'s "Further Notes" pricing-is-a-launch-time-decision note is updated or superseded to reflect that this repricing already happened, so a future pass doesn't re-litigate the same reasoning14
15## Comments16
17**2026-09-02:** Scoped in response to a live incident: the Actor was flagged "Under maintenance" after 3 consecutive daily-autotest failures (intermittent anti-bot blocking on the proxy-less v1 default), which motivated pushing ticket 08 immediately rather than waiting for pricing to be ready first (the original plan in ticket 08's Comments). Proxy is now live and billed on every run, including the free daily autotest, with no corresponding price change yet — this ticket closes that gap.18
19Deliberately did not propose specific new prices here. Every number in this project so far (the original $0.001/event undercut, the RESIDENTIAL default) was set from a live-verified fact, not a guess — this ticket's first AC exists so the actual price gets computed from measured bandwidth, not an estimate invented at scoping time.20
21**2026-09-10 — Closed, moot.** Ticket 08 (the thing this ticket exists to price for) was itself closed as not viable: Apify's shared proxy pools (both RESIDENTIAL and the default datacenter pool) hit three distinct connection-failure modes against Instagram specifically, none fixable at the code level — see ticket 08's Comments for the full investigation. Proxy-less is the permanent default again, with no proxy cost to absorb into per-event pricing. Current live pricing ($0.001/event for both `post-scraped` and `reel-scraped`) stands unchanged. Revisit this ticket only if a future BYO-proxy-driven or non-Apify-proxy default path reopens the cost question.1import type { ContentType, MediaType, ScrapedItem } from '../types.js';2
3export class UnrecognizedMediaShapeError extends Error {}4
5interface RawMediaDetails {6 user?: { username?: unknown };7 caption?: { text?: unknown };8 like_count?: unknown;9 comment_count?: unknown;10 taken_at?: unknown;11 media_type?: unknown;12 product_type?: unknown;13 display_uri?: unknown;14 image_versions2?: { candidates?: Array<{ url?: unknown }> };15 video_versions?: Array<{ url?: unknown }>;16}17
18interface RawMediaResponse {19 data?: {20 xig_polaris_media?: {21 code?: unknown;22 if_not_gated_logged_out?: RawMediaDetails;23 };24 };25}26
27/**28 * Maps Instagram's `xig_polaris_media` response shape — used for both posts29 * and reels on direct permalink navigation, differentiated by `media_type`/30 * `product_type` rather than by two separate schemas — to the shared output31 * shape. See `test/fixtures/README.md` for what's verified vs. best-effort.32 */33export function mapXigPolarisMedia(raw: unknown): ScrapedItem {34 const media = (raw as RawMediaResponse)?.data?.xig_polaris_media;35 if (!media || typeof media.code !== 'string') {36 throw new UnrecognizedMediaShapeError('Response did not contain a recognizable xig_polaris_media.code.');37 }38
39 const details = media.if_not_gated_logged_out;40 if (!details || typeof details !== 'object' || typeof details.taken_at !== 'number') {41 throw new UnrecognizedMediaShapeError(42 'Response was missing if_not_gated_logged_out (or its taken_at field) — refusing to push a near-empty item.',43 );44 }45
46 const mediaType = mapMediaType(details.media_type);47 const contentType: ContentType = details.product_type === 'clips' ? 'reel' : 'post';48
49 return {50 shortcode: media.code,51 ownerUsername: typeof details.user?.username === 'string' ? details.user.username : null,52 contentType,53 caption: typeof details.caption?.text === 'string' ? details.caption.text : null,54 timestamp: details.taken_at,55 likeCount: typeof details.like_count === 'number' ? details.like_count : null,56 commentCount: typeof details.comment_count === 'number' ? details.comment_count : null,57 mediaType,58 mediaUrl: extractMediaUrl(details, mediaType),59 };60}61
62function mapMediaType(rawMediaType: unknown): MediaType {63 if (rawMediaType === 2) return 'video';64 if (rawMediaType === 8) return 'carousel';65 return 'image';66}67
68/**69 * Returns a single representative URL — the cover image for a carousel, not70 * every slide. `carousel_media[]` (verified present, e.g. 8 entries on the71 * captured fixture) is intentionally not surfaced: the ticket's output72 * schema specifies one `mediaUrl` per item, singular.73 */74function extractMediaUrl(details: RawMediaDetails, mediaType: MediaType): string | null {75 if (mediaType === 'video') {76 const url = details.video_versions?.[0]?.url;77 return typeof url === 'string' ? url : null;78 }79 if (typeof details.display_uri === 'string') return details.display_uri;80 const candidateUrl = details.image_versions2?.candidates?.[0]?.url;81 return typeof candidateUrl === 'string' ? candidateUrl : null;82}