Instagram Post & Reel Scraper
Pricing
from $1.00 / 1,000 posts
Scrapes public Instagram profile posts and reels, or looks up a single post/reel by URL. No login required.
Pricing
from $1.00 / 1,000 posts
Rating
0.0
(0)
Developer
Saketh Balijepalli
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
3 days ago
Last modified
Categories
Share
node_modulesdiststorage.git.scratch.envtestcoveragenode_modules/dist/storage/apify_storage/.env1## Agent skills2
3### Issue tracker4
5Issues and specs live as local markdown files under `.scratch/<feature-slug>/`. See `docs/agents/issue-tracker.md`.6
7### Triage labels8
9Default five canonical roles (needs-triage, needs-info, ready-for-agent, ready-for-human, wontfix). See `docs/agents/triage-labels.md`.10
11### Domain docs12
13Single-context — `CONTEXT.md` + `docs/adr/` at the repo root. See `docs/agents/domain.md`.{ "name": "instagram-apify-scraper", "version": "0.1.0", "type": "module", "private": true, "description": "Apify Actor that scrapes public Instagram posts and reels.", "main": "dist/main.js", "scripts": { "build": "tsc", "start": "npm run build && node dist/main.js", "typecheck": "tsc --noEmit -p tsconfig.typecheck.json", "test": "vitest run" }, "dependencies": { "apify": "^3.2.6", "crawlee": "^3.11.5", "playwright": "^1.47.0" }, "devDependencies": { "@crawlee/memory-storage": "^3.11.5", "@types/node": "^20.14.0", "typescript": "^5.5.4", "vitest": "^2.0.5" }}{ "compilerOptions": { "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", "lib": ["ES2022"], "outDir": "dist", "rootDir": "src", "strict": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "declaration": false, "sourceMap": false }, "include": ["src/**/*.ts"], "exclude": ["node_modules", "dist", "test", "storage"]}{ "extends": "./tsconfig.json", "compilerOptions": { "rootDir": "." }, "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["node_modules", "dist"]}FROM apify/actor-node-playwright-chrome:20 AS builder
COPY package*.json ./RUN npm install --include=dev --audit=false
COPY . ./RUN npm run build
FROM apify/actor-node-playwright-chrome:20
COPY package*.json ./RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional --audit=false \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
COPY /home/myuser/dist ./distCOPY . ./
CMD ["node", "dist/main.js"]{ "actorSpecification": 1, "name": "instagram-post-reel-scraper", "title": "Instagram Post & Reel Scraper", "description": "Scrapes public Instagram profile posts and reels, or looks up a single post/reel by URL. No login required.", "version": "0.1", "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": "./dataset_schema.json" }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "shortcode": { "type": "string", "description": "Instagram's short code for the item, taken from its URL." }, "ownerUsername": { "type": "string", "description": "Null if Instagram didn't include it in the response." }, "contentType": { "type": "string", "enum": ["post", "reel"], "description": "Content type" }, "caption": { "type": "string", "description": "Caption" }, "timestamp": { "type": "number", "description": "Unix timestamp, in seconds, of when the item was posted." }, "likeCount": { "type": "number", "description": "Like count" }, "commentCount": { "type": "number", "description": "Comment count" }, "mediaType": { "type": "string", "enum": ["image", "video", "carousel"], "description": "Media type" }, "mediaUrl": { "type": "string", "description": "A single representative URL: the video file for videos, or the cover/first image for images and carousels." } }, "required": ["shortcode", "contentType", "timestamp", "mediaType"] }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "shortcode", "ownerUsername", "contentType", "caption", "timestamp", "likeCount", "commentCount", "mediaType", "mediaUrl" ] }, "display": { "component": "table", "properties": { "shortcode": { "label": "Shortcode", "format": "text" }, "ownerUsername": { "label": "Owner", "format": "text" }, "contentType": { "label": "Type", "format": "text" }, "caption": { "label": "Caption", "format": "text" }, "timestamp": { "label": "Posted at", "format": "number" }, "likeCount": { "label": "Likes", "format": "number" }, "commentCount": { "label": "Comments", "format": "number" }, "mediaType": { "label": "Media type", "format": "text" }, "mediaUrl": { "label": "Media", "format": "link" } } } } }}{ "title": "Instagram Post & Reel Scraper input", "type": "object", "schemaVersion": 1, "properties": { "username": { "title": "Profile username", "type": "string", "editor": "textfield", "description": "Public Instagram username to scrape posts and reels from. Provide this or postOrReelUrl." }, "postOrReelUrl": { "title": "Post or reel URL", "type": "string", "editor": "textfield", "description": "Direct URL to a single public Instagram post or reel to look up. Provide this or username." }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "editor": "proxy", "description": "Optional. Route requests through your own proxy (or Apify Proxy, at your own cost) to reduce the chance of being blocked. Left empty, the Actor runs proxy-less by default at no extra cost.", "prefill": { "useApifyProxy": false } } }, "required": []}{ "actorOutputSchemaVersion": 1, "title": "Instagram Post & Reel Scraper output", "properties": { "scrapedItems": { "type": "string", "title": "Scraped posts & reels", "template": "{{links.apiDefaultDatasetUrl}}/items?view=overview" } }}1import type { Actor } from 'apify';2import { validateInput, type ActorInput } from './input.js';3import { ItemNotFoundError, type ItemFetcher } from './instagram/fetcher.js';4import { normalizeRawItem, UnrecognizedItemShapeError } from './instagram/normalize.js';5import type { ProfileFetcher } from './instagram/profileFetcher.js';6import type { ContentType, ScrapedItem } from './instagram/types.js';7
8/**9 * Reads and validates the run's input, then fetches, normalizes, and pushes10 * item(s): a single item for `postOrReelUrl`, or every item discoverable on11 * a profile for `username` (which internally reuses the same per-item12 * fetch/normalize/charge pipeline once shortcodes are known). Takes an13 * already-`init()`-ed Actor instance and injected fetchers so it can be14 * exercised against an isolated in-memory storage instance and canned15 * responses in tests.16 *17 * When both inputs are given, `postOrReelUrl` takes priority and `username`18 * is ignored — see ticket 02 code review finding #7.19 */20export async function processRun(actor: Actor, fetcher: ItemFetcher, profileFetcher: ProfileFetcher): Promise<void> {21 const input = await actor.getInput<ActorInput>();22 validateInput(input);23
24 if (input.postOrReelUrl) {25 await scrapeSingleItem(actor, fetcher, input.postOrReelUrl);26 return;27 }28
29 if (input.username) {30 await scrapeProfile(actor, fetcher, profileFetcher, input.username);31 }32}33
34async function scrapeSingleItem(actor: Actor, fetcher: ItemFetcher, url: string): Promise<void> {35 const raw = await fetcher.fetchRawItem(url);36 const item = normalizeRawItem(raw);37 await pushScrapedItem(actor, item);38}39
40/**41 * Fetches each discovered item strictly sequentially — one Chromium launch42 * per item via `fetcher`, same as a single-item run. Known v1 limitation:43 * for large profiles this means tens of minutes of sequential browser44 * overhead where a small bounded-concurrency change could help; deferred45 * per the project's phased "ship, revisit occasionally" plan rather than46 * built now.47 */48async function scrapeProfile(49 actor: Actor,50 fetcher: ItemFetcher,51 profileFetcher: ProfileFetcher,52 username: string,53): Promise<void> {54 const listing = await profileFetcher.fetchProfileListing(username);55
56 for (const { shortcode, contentType } of listing.items) {57 try {58 const raw = await fetcher.fetchRawItem(permalinkFor(shortcode, contentType));59 const item = normalizeRawItem(raw);60 await pushScrapedItem(actor, item);61 } catch (error) {62 // An item can become unavailable or unparseable between listing capture and63 // detail fetch — skip it and keep scraping the rest of the profile rather64 // than aborting the whole run (AC2: partial results, not silent/total failure).65 // Anything else (infra failures, unexpected shapes elsewhere) still propagates.66 if (!(error instanceof ItemNotFoundError) && !(error instanceof UnrecognizedItemShapeError)) throw error;67 }68 }69
70 if (listing.limitReached) {71 await actor.setStatusMessage(72 "Reached Instagram's anonymous-access pagination limit before this profile's items were exhausted — results may be incomplete.",73 { isStatusMessageTerminal: true },74 );75 }76}77
78async function pushScrapedItem(actor: Actor, item: ScrapedItem): Promise<void> {79 // 'post-scraped' / 'reel-scraped' must match the event names registered80 // in the Actor's pay-per-event pricing config (ticket 05) — until that's81 // configured, pushData still succeeds but charges $0 (Apify SDK no-ops82 // charging for unregistered event names rather than throwing).83 await actor.pushData(item, `${item.contentType}-scraped`);84}85
86function permalinkFor(shortcode: string, contentType: ContentType): string {87 const segment = contentType === 'reel' ? 'reel' : 'p';88 return `https://www.instagram.com/${segment}/${shortcode}/`;89}1import type { ProxyConfigurationOptions } from 'apify';2
3export interface ActorInput {4 username?: string;5 postOrReelUrl?: string;6 proxyConfiguration?: ProxyConfigurationOptions & { useApifyProxy?: boolean };7}8
9export class InvalidInputError extends Error {}10
11export function validateInput(input: ActorInput | null): asserts input is ActorInput {12 if (!input || (!input.username && !input.postOrReelUrl)) {13 throw new InvalidInputError(14 'Provide either "username" (to scrape a profile) or "postOrReelUrl" (to look up a single post or reel).',15 );16 }17}1import { Actor } from 'apify';2import { processRun } from './actor.js';3import type { ActorInput } from './input.js';4import { BrowserItemFetcher } from './instagram/browserFetcher.js';5import { BrowserProfileFetcher } from './instagram/browserProfileFetcher.js';6import { toPlaywrightProxy, type PlaywrightProxy } from './instagram/proxy.js';7
8const actor = new Actor();9
10try {11 await actor.init();12
13 // Actor.createProxyConfiguration(undefined) does NOT mean "no proxy" — its default14 // parameter is {}, which falls through to real, auto-mode (billable) Apify Proxy on15 // the platform, where APIFY_TOKEN is always present. Only an explicit, present16 // proxyConfiguration input (including the Input Schema editor's {useApifyProxy:false}17 // prefill, which the SDK does special-case to undefined) may be passed through — this18 // guard is what actually keeps a run with no proxyConfiguration key zero-spend.19 const input = await actor.getInput<ActorInput>();20 const proxyConfiguration = input?.proxyConfiguration21 ? await actor.createProxyConfiguration(input.proxyConfiguration)22 : undefined;23 const proxyInfo = await proxyConfiguration?.newProxyInfo();24 const proxy: PlaywrightProxy | undefined = proxyInfo ? toPlaywrightProxy(proxyInfo) : undefined;25
26 const fetcher = new BrowserItemFetcher({ proxy });27 const profileFetcher = new BrowserProfileFetcher({ proxy });28
29 await processRun(actor, fetcher, profileFetcher);30 await actor.exit();31} catch (error) {32 const message = error instanceof Error ? error.message : String(error);33 await actor.exit({ statusMessage: message, exitCode: 1 });34}1import { Router, type BasicCrawlingContext, type RouterHandler } from 'crawlee';2
3/**4 * Placeholder proving Crawlee's Router wires up and compiles. `BasicCrawlingContext`5 * can't host a Playwright handler, so this is not itself the hybrid direct-API/6 * browser-automation routing point the spec describes — ticket 02 designs that7 * (most likely as two crawlers, one per context type, behind a shared dispatch8 * function) and will replace this module accordingly.9 *10 * Returns `RouterHandler`, not the base `Router` type — `RouterHandler` carries11 * the call signature that makes the result usable as a crawler's `requestHandler`12 * (or directly callable, as tests do).13 */14export function createScraperRouter(): RouterHandler<BasicCrawlingContext> {15 const router = Router.create<BasicCrawlingContext>();16
17 router.addDefaultHandler(async () => {18 // Intentionally a no-op — no scraping logic exists yet.19 });20
21 return router;22}1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { MemoryStorage } from '@crawlee/memory-storage';4import { Actor } from 'apify';5import { describe, expect, it, vi } from 'vitest';6import { processRun } from '../src/actor.js';7import { ItemNotFoundError, type ItemFetcher } from '../src/instagram/fetcher.js';8import { PrivateProfileError, ProfileNotFoundError, type ProfileFetcher } from '../src/instagram/profileFetcher.js';9import { InvalidInputError } from '../src/input.js';10
11async function createTestActor(): Promise<Actor> {12 const actor = new Actor({13 storageClient: new MemoryStorage({ persistStorage: false }),14 });15 await actor.init();16 return actor;17}18
19function loadFixture(name: string): unknown {20 const path = fileURLToPath(new URL(`./fixtures/${name}`, import.meta.url));21 return JSON.parse(readFileSync(path, 'utf8'));22}23
24function stubFetcher(fetchRawItem: ItemFetcher['fetchRawItem']): ItemFetcher {25 return { fetchRawItem };26}27
28function stubProfileFetcher(fetchProfileListing: ProfileFetcher['fetchProfileListing']): ProfileFetcher {29 return { fetchProfileListing };30}31
32const neverCalledFetcher = stubFetcher(async () => {33 throw new Error('fetcher should not be called for this input');34});35
36const neverCalledProfileFetcher = stubProfileFetcher(async () => {37 throw new Error('profileFetcher should not be called for this input');38});39
40describe('local Dataset storage harness', () => {41 it('can observe an item pushed to the Dataset', async () => {42 const actor = await createTestActor();43 const dataset = await actor.openDataset();44
45 await dataset.pushData({ shortcode: 'ABC123' });46
47 const { items } = await dataset.getData();48 expect(items).toEqual([{ shortcode: 'ABC123' }]);49 });50});51
52describe('processRun', () => {53 it('rejects with a clear error when neither username nor postOrReelUrl is provided', async () => {54 const actor = await createTestActor();55 const keyValueStore = await actor.openKeyValueStore();56 await keyValueStore.setValue('INPUT', {});57
58 await expect(processRun(actor, neverCalledFetcher, neverCalledProfileFetcher)).rejects.toThrow(InvalidInputError);59 });60
61 it('pushes a normalized reel item and charges the reel event, given a postOrReelUrl', async () => {62 const actor = await createTestActor();63 const keyValueStore = await actor.openKeyValueStore();64 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/reel/DbvqPafyqY-/' });65 const fetcher = stubFetcher(async () => loadFixture('reel-single.json'));66 const chargeSpy = vi.spyOn(actor, 'pushData');67
68 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).resolves.toBeUndefined();69
70 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbvqPafyqY-' }), 'reel-scraped');71 const dataset = await actor.openDataset();72 const { items } = await dataset.getData();73 expect(items).toEqual([expect.objectContaining({ shortcode: 'DbvqPafyqY-', contentType: 'reel' })]);74 });75
76 it('pushes a normalized post item and charges the post event, given a postOrReelUrl', async () => {77 const actor = await createTestActor();78 const keyValueStore = await actor.openKeyValueStore();79 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/p/DbbY9pdm6Q2/' });80 const fetcher = stubFetcher(async () => loadFixture('post-carousel.json'));81 const chargeSpy = vi.spyOn(actor, 'pushData');82
83 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).resolves.toBeUndefined();84
85 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }), 'post-scraped');86 const dataset = await actor.openDataset();87 const { items } = await dataset.getData();88 expect(items).toEqual([expect.objectContaining({ shortcode: 'DbbY9pdm6Q2', contentType: 'post' })]);89 });90
91 it('rejects with a clear error and leaves the Dataset empty when the item is not found', async () => {92 const actor = await createTestActor();93 const keyValueStore = await actor.openKeyValueStore();94 await keyValueStore.setValue('INPUT', { postOrReelUrl: 'https://www.instagram.com/p/deleted000/' });95 const fetcher = stubFetcher(async () => {96 throw new ItemNotFoundError('This post or reel is not available.');97 });98
99 await expect(processRun(actor, fetcher, neverCalledProfileFetcher)).rejects.toThrow(ItemNotFoundError);100
101 const dataset = await actor.openDataset();102 const { items } = await dataset.getData();103 expect(items).toHaveLength(0);104 });105
106 it('rejects with PrivateProfileError and pushes nothing, given a private profile', async () => {107 const actor = await createTestActor();108 const keyValueStore = await actor.openKeyValueStore();109 await keyValueStore.setValue('INPUT', { username: 'a-private-account' });110 const profileFetcher = stubProfileFetcher(async () => {111 throw new PrivateProfileError('This profile is private.');112 });113
114 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).rejects.toThrow(PrivateProfileError);115
116 const dataset = await actor.openDataset();117 const { items } = await dataset.getData();118 expect(items).toHaveLength(0);119 });120
121 it('resolves with an explicit empty Dataset, given a public profile with zero items', async () => {122 const actor = await createTestActor();123 const keyValueStore = await actor.openKeyValueStore();124 await keyValueStore.setValue('INPUT', { username: 'an-empty-account' });125 const profileFetcher = stubProfileFetcher(async () => ({ items: [], limitReached: false }));126
127 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).resolves.toBeUndefined();128
129 const dataset = await actor.openDataset();130 const { items } = await dataset.getData();131 expect(items).toHaveLength(0);132 });133
134 it('fetches, normalizes, and pushes each discovered item at the correct permalink, given a populated profile', async () => {135 const actor = await createTestActor();136 const keyValueStore = await actor.openKeyValueStore();137 await keyValueStore.setValue('INPUT', { username: 'nasa' });138 const profileFetcher = stubProfileFetcher(async () => ({139 items: [140 { shortcode: 'DbvqPafyqY-', contentType: 'reel' },141 { shortcode: 'DbbY9pdm6Q2', contentType: 'post' },142 ],143 limitReached: false,144 }));145 const requestedUrls: string[] = [];146 const fetcher = stubFetcher(async (url) => {147 requestedUrls.push(url);148 return url.includes('/reel/') ? loadFixture('reel-single.json') : loadFixture('post-carousel.json');149 });150 const chargeSpy = vi.spyOn(actor, 'pushData');151
152 await expect(processRun(actor, fetcher, profileFetcher)).resolves.toBeUndefined();153
154 expect(requestedUrls).toEqual([155 'https://www.instagram.com/reel/DbvqPafyqY-/',156 'https://www.instagram.com/p/DbbY9pdm6Q2/',157 ]);158 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbvqPafyqY-' }), 'reel-scraped');159 expect(chargeSpy).toHaveBeenCalledWith(expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }), 'post-scraped');160 const dataset = await actor.openDataset();161 const { items } = await dataset.getData();162 expect(items).toHaveLength(2);163 });164
165 it('rejects with ProfileNotFoundError and pushes nothing, given an unrecognized profile', async () => {166 const actor = await createTestActor();167 const keyValueStore = await actor.openKeyValueStore();168 await keyValueStore.setValue('INPUT', { username: 'this-should-not-exist' });169 const profileFetcher = stubProfileFetcher(async () => {170 throw new ProfileNotFoundError('Could not find profile data.');171 });172
173 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).rejects.toThrow(ProfileNotFoundError);174
175 const dataset = await actor.openDataset();176 const { items } = await dataset.getData();177 expect(items).toHaveLength(0);178 });179
180 it('skips an item that fails with a content-level error and keeps scraping the rest of the profile', async () => {181 const actor = await createTestActor();182 const keyValueStore = await actor.openKeyValueStore();183 await keyValueStore.setValue('INPUT', { username: 'nasa' });184 const profileFetcher = stubProfileFetcher(async () => ({185 items: [186 { shortcode: 'DbvqPafyqY-', contentType: 'reel' },187 { shortcode: 'deleted000', contentType: 'post' },188 { shortcode: 'DbbY9pdm6Q2', contentType: 'post' },189 ],190 limitReached: false,191 }));192 const fetcher = stubFetcher(async (url) => {193 if (url.includes('deleted000')) throw new ItemNotFoundError('This post or reel is not available.');194 return url.includes('/reel/') ? loadFixture('reel-single.json') : loadFixture('post-carousel.json');195 });196
197 await expect(processRun(actor, fetcher, profileFetcher)).resolves.toBeUndefined();198
199 const dataset = await actor.openDataset();200 const { items } = await dataset.getData();201 expect(items).toEqual([202 expect.objectContaining({ shortcode: 'DbvqPafyqY-' }),203 expect.objectContaining({ shortcode: 'DbbY9pdm6Q2' }),204 ]);205 });206
207 it('reports a clear partial-results status message when the anonymous pagination limit is hit', async () => {208 const actor = await createTestActor();209 const keyValueStore = await actor.openKeyValueStore();210 await keyValueStore.setValue('INPUT', { username: 'a-very-active-account' });211 const profileFetcher = stubProfileFetcher(async () => ({ items: [], limitReached: true }));212 const statusSpy = vi.spyOn(actor, 'setStatusMessage').mockResolvedValue({} as never);213
214 await expect(processRun(actor, neverCalledFetcher, profileFetcher)).resolves.toBeUndefined();215
216 expect(statusSpy).toHaveBeenCalledWith(expect.stringMatching(/incomplete|limit/i), expect.anything());217 });218});1import { describe, expect, it } from 'vitest';2import { InvalidInputError, validateInput } from '../src/input.js';3
4describe('validateInput', () => {5 it('throws InvalidInputError when neither username nor postOrReelUrl is provided', () => {6 expect(() => validateInput({})).toThrow(InvalidInputError);7 expect(() => validateInput(null)).toThrow(InvalidInputError);8 });9
10 it('does not throw when a username is provided', () => {11 expect(() => validateInput({ username: 'natgeo' })).not.toThrow();12 });13
14 it('does not throw when a postOrReelUrl is provided', () => {15 expect(() =>16 validateInput({ postOrReelUrl: 'https://www.instagram.com/p/ABC123/' }),17 ).not.toThrow();18 });19});1import { describe, expect, it } from 'vitest';2import { createScraperRouter } from '../src/router.js';3
4describe('createScraperRouter', () => {5 it('registers a default handler that runs without throwing', async () => {6 const router = createScraperRouter();7 const defaultHandler = router.getHandler();8
9 // @ts-expect-error -- empty context stub; the placeholder handler ignores it entirely10 await expect(defaultHandler({})).resolves.toBeUndefined();11 });12});1#!/usr/bin/env bash2#3# A wizard — walks a human through a manual procedure step by step.4# Generated by the /wizard skill.5#6# Everything above the "STAGES" marker is the wizard library: do not hand-edit7# it. Author the per-step stages below the marker.8
9set -euo pipefail10
11# ──────────────────────────────────────────────────────────────────────────12# Wizard library — delightful, consistent UX. Identical across every wizard.13# ──────────────────────────────────────────────────────────────────────────14
15if [[ -t 1 ]] && command -v tput >/dev/null 2>&1 && [[ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]]; then16 BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)17 BLUE=$(tput setaf 4); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); RED=$(tput setaf 1)18else19 BOLD=""; DIM=""; RESET=""; BLUE=""; GREEN=""; YELLOW=""; RED=""20fi21
22# Author sets this at the top of the stages section.23TOTAL_STAGES=024
25_STAGE_INDEX=026ENV_FILE="${ENV_FILE:-.env}"27WRITTEN_ENV=() # KEYs written to ENV_FILE this run28WRITTEN_SECRET=() # secret NAMEs set this run29SKIPPED=() # things we couldn't do (e.g. gh missing)30
31# _clear — wipe the terminal so only the current step is on screen. No-op when32# output isn't a terminal, so piped logs stay readable.33_clear() {34 [[ -t 1 ]] || return 035 if command -v tput >/dev/null 2>&1; then tput clear; else printf '\033[2J\033[3J\033[H'; fi36}37
38# banner "Title" — opening frame: what this wizard does.39banner() {40 _clear41 printf '\n%s%s %s%s\n' "$BOLD" "$BLUE" "$1" "$RESET"42 printf '%s %s stages%s\n\n' "$DIM" "$TOTAL_STAGES" "$RESET"43 printf '%s You drive the browser; this wizard tells you exactly what to do and\n' "$DIM"44 printf ' captures the values you copy back. Stop any time with Ctrl-C and re-run\n'45 printf ' later — it remembers values already saved.%s\n' "$RESET"46 pause "Ready to start?"47}48
49# stage "Name" — clear the screen, then announce a stage and show progress.50# Clearing keeps only the current step on screen.51stage() {52 _clear53 _STAGE_INDEX=$((_STAGE_INDEX + 1))54 printf '\n%s%s▸ Stage %s/%s · %s%s\n' \55 "$BOLD" "$BLUE" "$_STAGE_INDEX" "$TOTAL_STAGES" "$1" "$RESET"56}57
58# say "..." — a plain instruction line.59say() { printf ' %s\n' "$1"; }60# step "..." — a numbered-feeling action the human takes in the browser.61step() { printf ' %s•%s %s\n' "$BLUE" "$RESET" "$1"; }62note() { printf ' %s%s%s\n' "$DIM" "$1" "$RESET"; }63warn() { printf ' %s⚠ %s%s\n' "$YELLOW" "$1" "$RESET"; }64
65# open_url URL — open in the human's browser, cross-platform incl. WSL.66open_url() {67 local url="$1"68 printf ' %s↗ opening%s %s\n' "$GREEN" "$RESET" "$url"69 { if command -v wslview >/dev/null 2>&1; then wslview "$url"70 elif command -v explorer.exe >/dev/null 2>&1; then explorer.exe "$url"71 elif command -v xdg-open >/dev/null 2>&1; then xdg-open "$url"72 elif command -v open >/dev/null 2>&1; then open "$url"73 else warn "couldn't open a browser — visit it manually: $url"; fi74 } >/dev/null 2>&1 || warn "couldn't open a browser — visit it manually: $url"75}76
77# pause "msg" — wait for the human to confirm they've done the manual part.78pause() {79 printf ' %s%s%s ' "$DIM" "${1:-Press Enter to continue}" "$RESET"80 read -r _ || true81}82
83# confirm "question" — y/N gate; returns success on yes.84confirm() {85 local reply=""86 printf ' %s? %s [y/N] ' "$YELLOW" "$1"87 read -r reply || true88 [[ "$reply" =~ ^[Yy] ]]89}90
91# _existing KEY — current value of KEY in ENV_FILE, if any.92_existing() {93 [[ -f "$ENV_FILE" ]] || return 194 local line; line=$(grep -E "^${1}=" "$ENV_FILE" | tail -n1) || return 195 printf '%s' "${line#*=}"96}97
98# ask KEY "Prompt" — read a value into $KEY. Offers the existing .env value as99# a default on re-runs (Enter keeps it). Visible input (non-secret).100ask() {101 local key="$1" prompt="$2" current input102 current=$(_existing "$key" || true)103 if [[ -n "$current" ]]; then104 printf ' %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"105 else106 printf ' %s%s%s ' "$BOLD" "$prompt" "$RESET"107 fi108 read -r input || true109 [[ -z "$input" && -n "$current" ]] && input="$current"110 printf -v "$key" '%s' "$input"111}112
113# ask_secret KEY "Prompt" — like ask, but input is hidden.114ask_secret() {115 local key="$1" prompt="$2" current input116 current=$(_existing "$key" || true)117 if [[ -n "$current" ]]; then118 printf ' %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"119 else120 printf ' %s%s%s ' "$BOLD" "$prompt" "$RESET"121 fi122 read -rs input || true123 printf '\n'124 [[ -z "$input" && -n "$current" ]] && input="$current"125 printf -v "$key" '%s' "$input"126}127
128# write_env KEY VALUE — upsert KEY=VALUE into ENV_FILE (creates it; replaces129# any existing line). Idempotent.130write_env() {131 local key="$1" value="$2" tmp132 touch "$ENV_FILE"133 tmp=$(mktemp)134 grep -vE "^${key}=" "$ENV_FILE" > "$tmp" || true135 printf '%s=%s\n' "$key" "$value" >> "$tmp"136 mv "$tmp" "$ENV_FILE"137 WRITTEN_ENV+=("$key")138 printf ' %s✓ wrote%s %s → %s\n' "$GREEN" "$RESET" "$key" "$ENV_FILE"139}140
141# set_secret NAME VALUE — set a GitHub Actions repo secret via gh. Falls back142# to a warning (and records it) if gh is unavailable or unauthenticated.143set_secret() {144 local name="$1" value="$2"145 if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then146 if printf '%s' "$value" | gh secret set "$name" >/dev/null 2>&1; then147 WRITTEN_SECRET+=("$name")148 printf ' %s✓ set%s GitHub secret %s\n' "$GREEN" "$RESET" "$name"149 return150 fi151 fi152 SKIPPED+=("GitHub secret $name (set it manually: gh secret set $name)")153 warn "skipped GitHub secret $name — gh not ready; set it later"154}155
156# set_var NAME VALUE — set a GitHub Actions repo variable (non-secret).157set_var() {158 local name="$1" value="$2"159 if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then160 if gh variable set "$name" --body "$value" >/dev/null 2>&1; then161 printf ' %s✓ set%s GitHub variable %s\n' "$GREEN" "$RESET" "$name"162 return163 fi164 fi165 SKIPPED+=("GitHub variable $name")166 warn "skipped GitHub variable $name — gh not ready; set it later"167}168
169# finish — clear, then a closing summary of everything configured.170finish() {171 _clear172 printf '\n%s%s ✓ Setup complete%s\n' "$BOLD" "$GREEN" "$RESET"173 (( ${#WRITTEN_ENV[@]} )) && note "wrote ${#WRITTEN_ENV[@]} value(s) to $ENV_FILE: ${WRITTEN_ENV[*]}"174 (( ${#WRITTEN_SECRET[@]} )) && note "set ${#WRITTEN_SECRET[@]} GitHub secret(s): ${WRITTEN_SECRET[*]}"175 if (( ${#SKIPPED[@]} )); then176 printf '\n'; warn "still to do by hand:"177 for s in "${SKIPPED[@]}"; do note " - $s"; done178 fi179 printf '\n'180}181
182# ──────────────────────────────────────────────────────────────────────────183# STAGES — author this section. One stage() per step the human takes.184# Replace the example below. Set TOTAL_STAGES to match the stages you write.185# ──────────────────────────────────────────────────────────────────────────186
187TOTAL_STAGES=9188
189REPO_ROOT="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel 2>/dev/null || echo "$HOME/instagram-apify-scraper")"190
191banner "Publish instagram-post-reel-scraper to the Apify Store"192
193# ── Stage 1: Apify CLI install + login ─────────────────────────────────────194stage "Apify CLI — install & log in"195say "Nothing has pushed this Actor to Apify's platform yet — no CLI, no linked project."196if command -v apify >/dev/null 2>&1; then197 say "apify CLI already installed ($(apify --version 2>/dev/null))."198else199 if confirm "apify CLI not found — install it now via 'npm install -g apify-cli'?"; then200 npm install -g apify-cli201 else202 warn "can't continue without the CLI — install it manually, then re-run this wizard."203 exit 1204 fi205fi206step "Run 'apify login' below. It opens a browser to authenticate, or you can paste an API token"207step "from Console → Settings → Integrations → API token."208( cd "$REPO_ROOT" && apify login )209pause "Logged in? Press Enter to continue."210
211# ── Stage 2: Push the Actor ─────────────────────────────────────────────────212stage "Push the Actor to your account"213say "This uploads the current code as an Actor on your account. It is NOT public or listed yet —"214say "just present on the platform so the Console has something to configure."215step "If this is the first push, the CLI will ask whether to create a new Actor — say yes and"216step "accept the name from .actor/actor.json (instagram-post-reel-scraper)."217( cd "$REPO_ROOT" && apify push )218pause "Push finished successfully? Press Enter to continue."219
220# ── Stage 3: Competitor pricing research ────────────────────────────────────221stage "Research competitor pricing"222say "The spec requires pricing below the lowest current comparable competitor — this has to be"223say "checked live, not assumed from memory."224open_url "https://apify.com/store?search=instagram+scraper"225step "Skim the pay-per-event Instagram scrapers listed. Note the lowest per-event price you see"226step "among actors comparable to this one (single post/reel or profile post/reel scraping)."227ask LOWEST_COMPETITOR_PRICE "Lowest comparable competitor price you found (e.g. \$0.01 per item):"228note "Stage 4 will ask you to price both events below this."229
230# ── Stage 4: Configure pay-per-event pricing ────────────────────────────────231stage "Configure pay-per-event pricing"232open_url "https://console.apify.com/actors"233step "Open the Actor you just pushed (instagram-post-reel-scraper) → Publication tab → Monetization."234step "Add TWO events, matching these names exactly (the code charges these via Actor.pushData(item, eventName)):"235say " • name: post-scraped — title: e.g. \"Post scraped\" — description: one post pushed to the Dataset"236say " • name: reel-scraped — title: e.g. \"Reel scraped\" — description: one reel pushed to the Dataset"237step "Price BOTH below the competitor price from Stage 3 ($LOWEST_COMPETITOR_PRICE)."238ask POST_SCRAPED_PRICE "Price you set for post-scraped:"239ask REEL_SCRAPED_PRICE "Price you set for reel-scraped:"240pause "Both events saved in Console? Press Enter to continue."241
242# ── Stage 5: Display information ────────────────────────────────────────────243stage "Display information"244say "Still on the Publication tab:"245step "Add an Actor logo (any square image works for now — can be swapped later)."246step "Confirm the description pulls in cleanly and reads well."247step "Confirm the README renders correctly as the Store detail page — it now carries the"248step "Pricing, Maintenance, and Disclaimer sections written for ticket 05. Check the"249step "Disclaimer section is visible and unmodified by any Console-side formatting."250pause "Display Information section marked complete? Press Enter to continue."251
252# ── Stage 6: Sample output & schemas ────────────────────────────────────────253stage "Sample output & schemas"254note "Apify's docs don't spell out exact fields here — follow whatever the Publication tab's"255note "\"Sample Output\" and \"Schemas\" sections prompt you for on the page itself."256step "Sample Output: if not auto-populated, run the Actor once (single postOrReelUrl input against"257step "a real public post) and use that run's dataset item as the sample."258step "Schemas: accept the auto-generated output schema unless the Console flags something missing."259pause "Both sections marked complete? Press Enter to continue."260
261# ── Stage 7: Actor permissions ──────────────────────────────────────────────262stage "Actor permissions"263note "Again, no fixed field list from the docs — follow the Console page's own prompts."264step "Set whatever permission level the Publication tab asks for (this determines what a user"265step "running the Actor from the Store can see/access) — the default/minimal option is fine for v1."266pause "Actor Permissions section marked complete? Press Enter to continue."267
268# ── Stage 8: Publish ─────────────────────────────────────────────────────────269stage "Publish on Store"270warn "This is the irreversible step — the Actor becomes publicly listed and runnable by anyone."271say "All Publication tab sections should now show as complete (Display Information, Monetization,"272say "Sample Output, Schemas, Actor Permissions)."273if confirm "All sections complete and ready to go public — click 'Publish on Store' now?"; then274 step "Click 'Publish on Store' in the Console."275 pause "Published? Press Enter to continue."276else277 warn "stopping before publish — re-run this wizard (or just do Stage 8 by hand) when ready."278 SKIPPED+=("Publish on Store — deferred, run was stopped before the irreversible step")279fi280
281# ── Stage 9: Verify ──────────────────────────────────────────────────────────282stage "Verify the listing is live"283open_url "https://apify.com/store?search=instagram+post+reel+scraper"284step "Confirm your Actor appears in the Store search results with the two priced events showing."285pause "Confirmed live? Press Enter to continue."286
287finish288note "Recorded this run: competitor floor $LOWEST_COMPETITOR_PRICE, post-scraped $POST_SCRAPED_PRICE, reel-scraped $REEL_SCRAPED_PRICE."289# ──────────────────────────────────────────────────────────────────────────1Status: ready-for-agent2
3# Instagram Post/Reel Scraper — Apify Actor4
5## Problem Statement6
7Anyone who wants structured data out of public Instagram content — marketers pulling a competitor's post performance, researchers sampling reels, analysts tracking a brand's account — has to either build their own scraper (fragile, time-consuming, breaks constantly against Instagram's anti-bot measures) or pay one of the existing Apify Store scrapers, which are already commoditized around the same core feature set (profile posts, reels, hashtags, comments) at broadly similar prices.8
9Separately, the developer publishing this Actor wants to enter that market and generate revenue from it, but cannot take on any ongoing personal financial cost — not for proxies, not for third-party APIs, not for storage — at launch or at any point later.10
11## Solution12
13A v1 Apify Actor that scrapes public Instagram profile posts and reels, plus single post/reel URL lookups, with no login required, and publishes to the Apify Store under pay-per-event pricing set below the cheapest current competitor. The Actor returns structured metadata and direct media URLs (never downloads or stores media itself).14
15Every design choice pushes any real operating cost (proxying, authentication, future paid AI calls) onto the end user of the Actor via optional bring-your-own inputs, never onto the developer — so the product can be run and grown indefinitely at zero cost to its publisher.16
17Three further capabilities are planned as later phases, once v1 has validated demand: Stories/Highlights access (gated behind a user-supplied session cookie), reel transcription (self-hosted, open-source, no paid API), and follower/following counts and growth stats (not full list scraping, which carries too high a ban risk).18
19## User Stories20
211. As an Apify Store user, I want to scrape a public Instagram profile's posts by supplying just a username, so that I can get structured post data without logging in.222. As an Apify Store user, I want to scrape a public Instagram profile's reels by supplying just a username, so that I can get structured reel data without logging in.233. As an Apify Store user, I want to look up a single post or reel by URL, so that I don't have to scrape an entire profile when I only need one item.244. As an Apify Store user, I want the output to include direct media URLs rather than downloaded files, so that I can fetch or skip the media myself without paying for Actor storage I don't need.255. As an Apify Store user, I want the returned dataset fields (caption, timestamp, like count, comment count, media type, owner username, shortcode) to be consistent across posts and reels, so that I can process both content types with the same downstream code.266. As an Apify Store user, I want reel-specific fields (e.g. view count, duration) included when scraping reels, so that I don't lose data specific to that content type.277. As an Apify Store user, I want to run the Actor without providing any Instagram login credentials, so that I'm not exposing my own account to a ban risk just to get public data.288. As an Apify Store user, I want to optionally supply my own proxy configuration, so that I can improve reliability at scale if I choose to, without the Actor forcing a paid proxy cost on me by default.299. As an Apify Store user, I want clear pricing per event/content type, so that I know exactly what a run will cost before I trigger it.3010. As an Apify Store user, I want the listing to disclose that scraping Instagram may conflict with its Terms of Service, so that I can make an informed decision about my own usage risk.3111. As an Apify Store user, I want the Actor to fail gracefully (clear error, not a silent empty result) when a requested profile is private or a post/reel no longer exists, so that I can distinguish "no data" from "broken run."3212. As the developer/publisher, I want the Actor priced below the cheapest comparable competitor at launch, so that I can win initial volume and reviews without having a differentiated feature set yet.3313. As the developer/publisher, I want zero infrastructure or third-party API costs at v1, so that running and hosting the Actor never costs me money regardless of usage volume.3414. As the developer/publisher, I want a pay-per-event monetization model on the Apify Store, so that I'm paid per unit of value delivered (per post/reel/lookup) rather than a flat rental fee I might not recoup.3515. As the developer/publisher, I want to be able to ship v1 with a narrow scope and add Stories/Highlights, transcripts, and follower stats later, so that I can validate demand before investing further unpaid effort.3616. As the developer/publisher, I want to treat maintenance as best-effort/ship-and-revisit rather than committing to an SLA, so that the project doesn't require an ongoing time commitment I can't guarantee.3717. As an Apify Store user (future, v1.1), I want to supply my own Instagram session cookie to unlock Stories/Highlights scraping, so that I can access that content without the Actor's developer ever handling my credentials.3818. As an Apify Store user (future, v1.2), I want reels to come with an automatically generated transcript, so that I can search/analyze reel content as text without manually transcribing audio.3919. As an Apify Store user (future, v1.2), I want follower/following counts and growth stats for a profile, so that I can track account growth without the Actor attempting a full (high-ban-risk) follower list scrape.4020. As the developer/publisher (future phases), I want any paid dependency introduced later (e.g. a transcription API) to be funded by the end user (bring-your-own API key) rather than by me, so that the zero-spend constraint holds as the product grows.41
42## Implementation Decisions43
44- **Runtime/SDK**: Node.js using Crawlee + the Apify SDK — Apify's own primary stack, with the deepest first-party docs/examples and the stack most incumbent Instagram scrapers on the Store are built on.45- **Scraping technique**: browser-driven (Playwright), loading each permalink directly and reading the data Instagram embeds server-side in the page rather than waiting on a client-side XHR (direct navigation doesn't reliably trigger one). **Corrected 2026-08-12 from an original hybrid direct-API-first design** — live capture confirmed plain HTTP requests get Instagram's anti-bot challenge shell, not real data, while the identical request succeeds from within a real browser session; no replicable direct-HTTP request recipe (session cookies/CSRF token) was captured. A lighter direct-HTTP path remains a possible future optimization, not part of v1.46- **Auth (v1)**: fully anonymous — no login, no session cookie required or accepted for the v1 feature set (profile posts, profile reels, single URL lookup). All v1 data must be reachable from Instagram's public web surface without authentication.47- **Auth (v1.1, Stories/Highlights only)**: optional `sessionCookie` Actor input, supplied by the end user for their own Instagram session. The Actor must never request, store, log, or persist this value beyond the lifetime of the run that uses it, and must never attempt to acquire a session via a login flow itself.48- **Proxy**: no bundled/paid proxy (e.g. no default use of Apify Proxy's paid tiers). An optional proxy-configuration Actor input lets the end user supply their own proxy; absent that input, the Actor runs proxy-less and accepts a higher block/ban rate as a known, documented limitation.49- **Media handling**: dataset items contain direct URLs to images/video, never downloaded binaries. No use of Apify Key-Value Store (or any storage) for media content, to keep storage cost at zero regardless of scale.50- **Output schema (v1, shared shape across posts and reels)**: shortcode/id, owner username, content type (post|reel), caption, timestamp, like count, comment count, media type (image|video|carousel), media URL(s). **Corrected 2026-08-12**: view count and duration dropped for reels — confirmed absent from the anonymous single-item reel query, not just unextracted. Posts and reels arrive via the **same** `xig_polaris_media` source schema on direct navigation (an earlier pass mistakenly concluded two separate schemas, based on data captured via a different, unused code path — client-side SPA link-clicks); one mapper derives `contentType`/`mediaType` from the `media_type`/`product_type` fields rather than dispatching between two schemas.51- **Input schema (v1)**: accepts either a profile username (to scrape that profile's posts+reels) or a direct post/reel URL (single-item lookup); optional proxy configuration input as described above.52- **Monetization**: pay-per-event pricing on the Apify Store, priced per distinct content-type event (e.g. "post scraped," "reel scraped," "URL lookup"), set below the lowest current comparable competitor price at launch.53- **Legal**: the Store listing carries a standard disclaimer (lawful/research use only, used at the runner's own risk, no guarantee of compliance with Instagram's Terms of Service).54- **Phasing** (drives ticket sequencing in `to-tickets`, not built as part of this spec's v1 slice):55 - v1: profile posts, profile reels, single post/reel URL lookup, anonymous only.56 - v1.1: Stories/Highlights via optional user-supplied session cookie.57 - v1.2: reel transcription via a self-hosted, open-source speech-to-text model running inside the Actor (no external paid API, so no BYO-key input is needed for this feature specifically — the zero-spend property holds for both developer and end user here).58 - v1.2 (or alongside): follower/following counts and growth stats only — explicit non-goal is full follower/following list scraping.59- **Zero-spend principle** (applies to every phase, not just v1): any feature that would otherwise require the developer to pay for external infrastructure (proxies, paid APIs, storage) must instead be structured so that cost sits with the end user (bring-your-own key/cookie/proxy) or is eliminated entirely (self-hosting, URLs-not-files). This principle should be treated as a standing constraint when scoping future tickets, not a one-time decision.60
61## Testing Decisions62
63- **Seam (single, highest-level)**: `Actor input → real pipeline (request routing, scraping, normalization, dataset push) → Apify Dataset output`. The only thing stubbed is the outbound HTTP layer to Instagram — canned GraphQL/JSON responses stand in for the network call. Everything else in the pipeline runs for real.64- **Harness**: Apify SDK's local storage emulation (`Actor.init()` against a local/in-memory dataset) — no real Apify Cloud run and no real network call to Instagram in tests.65- **What a good test asserts**: the shape and content of the resulting Dataset item(s) for a given canned input+response pair — never internal function calls or implementation details of the scraper.66- **Coverage required through this one seam**: profile-posts path, profile-reels path, single-URL-lookup path, and edge cases — private/inaccessible profile, deleted or missing post/reel, and an empty (no-posts) profile — each as a distinct canned network response.67- **Prior art**: none — this is a greenfield repo, so the first tests against this seam establish the pattern that subsequent tickets (v1.1, v1.2) should extend rather than replace.68
69## Out of Scope70
71- Login-based scraping, session/cookie handling, and Stories/Highlights access — deferred to v1.1, not part of this spec's buildable slice.72- Reel transcription — deferred to v1.2.73- Follower/following counts and growth stats — deferred to v1.2.74- Full follower/following list scraping — out of scope indefinitely; rejected due to Instagram's aggressive rate-limiting/ban risk on that specific action.75- Hashtag or search-based discovery — never adopted during scoping; not part of any planned phase.76- Downloading or storing media binaries — permanently out of scope by design (URLs only).77- Any bundled/paid proxy service — permanently out of scope by design (BYO proxy only).78- Any developer-funded third-party API or infrastructure cost, at any phase.79
80## Further Notes81
82- Pricing is a launch-time decision, not a permanent one: undercutting competitors is the only viable lever for v1 because it has no feature differentiation yet; once v1.1/v1.2 ship, pricing should be reassessed since Stories/transcripts/follower-stats give the Actor differentiated value to charge a premium for.83- Maintenance stance is explicitly best-effort ("ship it, revisit occasionally") — the Actor's Store listing and any user-facing documentation should set this expectation rather than implying an SLA, since Instagram changes its rendering/API frequently enough that breakage should be expected between fixes.84- This spec and its phasing came out of a structured design-tree interview (the `/grilling` session) rather than a pre-existing codebase — there is no prior code, ADRs, or domain glossary to reconcile against yet. `docs/agents/domain.md`'s "proceed silently if absent" rule applies as-is.1# Domain Docs2
3How the engineering skills should consume this repo's domain documentation when exploring the codebase.4
5## Before exploring, read these6
7- **`CONTEXT.md`** at the repo root, or8- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.9- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.10
11If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The `/domain-modeling` skill (reached via `/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily when terms or decisions actually get resolved.12
13## File structure14
15Single-context repo (most repos):16
17```18/19├── CONTEXT.md20├── docs/adr/21│ ├── 0001-event-sourced-orders.md22│ └── 0002-postgres-for-write-model.md23└── src/24```25
26## Use the glossary's vocabulary27
28When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.29
30If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`).31
32## Flag ADR conflicts33
34If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:35
36> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_1# Issue tracker: Local Markdown2
3Issues and specs for this repo live as markdown files in `.scratch/`.4
5## Conventions6
7- One feature per directory: `.scratch/<feature-slug>/`8- The spec is `.scratch/<feature-slug>/spec.md`9- Implementation issues are one file per ticket at `.scratch/<feature-slug>/issues/<NN>-<slug>.md`, numbered from `01` — never a single combined tickets file10- Triage state is recorded as a `Status:` line near the top of each issue file (see `triage-labels.md` for the role strings)11- Comments and conversation history append to the bottom of the file under a `## Comments` heading12
13## When a skill says "publish to the issue tracker"14
15Create a new file under `.scratch/<feature-slug>/` (creating the directory if needed).16
17## When a skill says "fetch the relevant ticket"18
19Read the file at the referenced path. The user will normally pass the path or the issue number directly.20
21## Wayfinding operations22
23Used by `/wayfinder`. The **map** is a file with one **child** file per ticket.24
25- **Map**: `.scratch/<effort>/map.md` — the Notes / Decisions-so-far / Fog body.26- **Child ticket**: `.scratch/<effort>/issues/NN-<slug>.md`, numbered from `01`, with the question in the body. A `Type:` line records the ticket type (`research`/`prototype`/`grilling`/`task`); a `Status:` line records `claimed`/`resolved`.27- **Blocking**: a `Blocked by: NN, NN` line near the top. A ticket is unblocked when every file it lists is `resolved`.28- **Frontier**: scan `.scratch/<effort>/issues/` for files that are open, unblocked, and unclaimed; first by number wins.29- **Claim**: set `Status: claimed` and save before any work.30- **Resolve**: append the answer under an `## Answer` heading, set `Status: resolved`, then append a context pointer (gist + link) to the map's Decisions-so-far in `map.md`.1# Triage Labels2
3The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker.4
5| Label in mattpocock/skills | Label in our tracker | Meaning |6| --------------------------- | --------------------- | ----------------------------------------- |7| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |8| `needs-info` | `needs-info` | Waiting on reporter for more information |9| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |10| `ready-for-human` | `ready-for-human` | Requires human implementation |11| `wontfix` | `wontfix` | Will not be actioned |1213When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.14
15Edit the right-hand column to match whatever vocabulary you actually use.1import { errors, type Page } from 'playwright';2import { defaultLauncher, type BrowserLauncher } from './browserLauncher.js';3import { ItemNotFoundError, type ItemFetcher } from './fetcher.js';4import type { PlaywrightProxy } from './proxy.js';5import { InvalidInputError } from '../input.js';6
7export type { BrowserLauncher };8
9const PAGE_LOAD_TIMEOUT_MS = 20_000;10
11/**12 * Fetches a single post/reel's raw data by loading its permalink in a real13 * browser and reading the data Instagram embeds server-side in the initial14 * page (a `<script type="application/json">` block), rather than waiting on15 * a client-side XHR — direct navigation was confirmed (2026-08-12 live16 * testing) to embed the data this way and not reliably issue a matching XHR17 * at all. Plain HTTP requests get an anti-bot challenge shell instead of18 * real data, which is why this needs a real browser in the first place — see19 * ticket 02's Corrections note.20 *21 * There is no unit-testable seam inside this class for the actual fetch22 * logic; `ItemFetcher` is the seam, and this implementation is exercised by23 * manual/live verification rather than the automated test suite. The24 * browser-launch step is the one exception (see `launcher` below) — it's25 * injectable specifically so ticket 04's proxy wiring can be asserted by a26 * test without a real browser or real proxy.27 *28 * Uses raw Playwright rather than Crawlee's PlaywrightCrawler — a single29 * permalink fetch needs no request queue or routing.30 */31export class BrowserItemFetcher implements ItemFetcher {32 private readonly proxy?: PlaywrightProxy;33 private readonly launcher: BrowserLauncher;34
35 constructor(options: { proxy?: PlaywrightProxy; launcher?: BrowserLauncher } = {}) {36 this.proxy = options.proxy;37 this.launcher = options.launcher ?? defaultLauncher;38 }39
40 async fetchRawItem(url: string): Promise<unknown> {41 const shortcode = extractShortcode(url);42 const browser = await this.launcher(this.proxy ? { proxy: this.proxy } : undefined);43
44 try {45 const page = await browser.newPage();46 return await fetchViaEmbeddedScripts(page, url, shortcode);47 } finally {48 await browser.close();49 }50 }51}52
53function extractShortcode(url: string): string {54 const segments = new URL(url).pathname.split('/').filter(Boolean);55 const marker = segments.findIndex((segment) => segment === 'p' || segment === 'reel');56 const shortcode = marker === -1 ? undefined : segments[marker + 1];57 if (!shortcode) {58 throw new InvalidInputError(`postOrReelUrl must contain a "/p/" or "/reel/" segment: ${url}`);59 }60 return shortcode;61}62
63async function fetchViaEmbeddedScripts(page: Page, url: string, shortcode: string): Promise<unknown> {64 try {65 await page.goto(url, { waitUntil: 'networkidle', timeout: PAGE_LOAD_TIMEOUT_MS });66 } catch (error) {67 // Instagram's long-lived connections can keep networkidle from firing —68 // that's expected and we fall through to check whatever loaded. Any69 // other navigation failure (DNS, connection reset, proxy failure) is a70 // real infrastructure problem and must not be mistaken for a missing item.71 if (!(error instanceof errors.TimeoutError)) throw error;72 }73
74 const title = await page.title().catch(() => '');75 const bodyText = await page76 .locator('body')77 .innerText()78 .catch(() => '');79 if (title.includes("isn't available") && bodyText.includes('link may be broken')) {80 throw new ItemNotFoundError(`Instagram reports this item is not available: ${url}`);81 }82
83 const media = await page.evaluate(findEmbeddedMedia, shortcode);84
85 if (!media) {86 throw new ItemNotFoundError(`Could not find media data embedded on the page for ${url}`);87 }88
89 return { data: { xig_polaris_media: media } };90}91
92/**93 * Runs inside the browser page (via `page.evaluate`). Scans every embedded94 * JSON script block for an object shaped like a `xig_polaris_media` node95 * matching the target shortcode.96 */97function findEmbeddedMedia(targetCode: string): Record<string, unknown> | null {98 function findMatch(value: unknown, depth: number): Record<string, unknown> | null {99 if (depth > 60 || value === null || typeof value !== 'object') return null;100 if (Array.isArray(value)) {101 for (const item of value) {102 const found = findMatch(item, depth + 1);103 if (found) return found;104 }105 return null;106 }107 const record = value as Record<string, unknown>;108 if (record.code === targetCode && record.if_not_gated_logged_out) return record;109 for (const key of Object.keys(record)) {110 const found = findMatch(record[key], depth + 1);111 if (found) return found;112 }113 return null;114 }115
116 const scripts = Array.from(document.querySelectorAll('script[type="application/json"]'));117 for (const script of scripts) {118 let parsed: unknown;119 try {120 parsed = JSON.parse(script.textContent || '');121 } catch {122 continue;123 }124 const match = findMatch(parsed, 0);125 if (match) return match;126 }127 return null;128}1import { chromium, type Browser } from 'playwright';2import type { PlaywrightProxy } from './proxy.js';3
4export type BrowserLauncher = (options?: { proxy?: PlaywrightProxy }) => Promise<Browser>;5
6// channel: 'chrome' launches the system Google Chrome that Apify's7// apify/actor-node-playwright-chrome base image pre-installs (at8// APIFY_CHROME_EXECUTABLE_PATH), instead of Playwright's own bundled,9// version-pinned Chromium build. The base image sets10// PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 specifically so a plain `npm install`11// of a newer `playwright` package never re-downloads browsers — so the12// unmanaged default channel can end up pointing at a Chromium revision13// that was never installed. The system Chrome channel sidesteps that14// version-matching entirely.15export const defaultLauncher: BrowserLauncher = (options) => chromium.launch({ ...options, channel: 'chrome' });1import { errors, type Page } from 'playwright';2import { defaultLauncher, type BrowserLauncher } from './browserLauncher.js';3import { PrivateProfileError, ProfileNotFoundError, type ProfileFetcher, type ProfileListing } from './profileFetcher.js';4import type { PlaywrightProxy } from './proxy.js';5import type { ContentType } from './types.js';6
7const PAGE_LOAD_TIMEOUT_MS = 20_000;8// Matches BrowserItemFetcher's findEmbeddedMedia depth cap (browserFetcher.ts) — kept in9// sync manually since both scan the same bootstrap structure but can't share code across10// the page.evaluate() serialization boundary (see findEmbeddedProfileAndTimeline's doc comment).11const MAX_EMBEDDED_SCRIPT_SCAN_DEPTH = 60;12
13/**14 * Discovers a public profile's posts/reels by reading the15 * `xig_user_by_username.polaris_ordered_timeline_connection` Instagram16 * embeds server-side on the profile page — the same embedded-script17 * technique `BrowserItemFetcher` uses for single items (see its doc18 * comment). Only the initial embed is read; a live Playwright probe19 * (2026-08-17) confirmed that requesting a further page while logged out20 * gets Instagram's backend to respond with an explicit "Unauthorized21 * logged out query" GraphQL error, not more data, so `has_next_page: true`22 * on the initial embed is treated directly as `limitReached` rather than23 * attempting (and always failing) a real next-page fetch. See ticket 03's24 * Corrections note.25 *26 * This connection's nodes only carry enough to build a permalink27 * (`code`/`product_type`) — full item detail (caption, timestamp, likes,28 * media URL) is fetched per item by the caller via the existing29 * `ItemFetcher` pipeline, not by this class.30 *31 * No unit-testable seam inside this class for the actual fetch logic;32 * `ProfileFetcher` is the seam, exercised by manual/live verification like33 * `BrowserItemFetcher`. The browser-launch step is injectable (see34 * `launcher` below) so ticket 04's proxy wiring can be asserted by a test35 * without a real browser or real proxy — same as `BrowserItemFetcher`.36 */37export class BrowserProfileFetcher implements ProfileFetcher {38 private readonly proxy?: PlaywrightProxy;39 private readonly launcher: BrowserLauncher;40
41 constructor(options: { proxy?: PlaywrightProxy; launcher?: BrowserLauncher } = {}) {42 this.proxy = options.proxy;43 this.launcher = options.launcher ?? defaultLauncher;44 }45
46 async fetchProfileListing(username: string): Promise<ProfileListing> {47 const browser = await this.launcher(this.proxy ? { proxy: this.proxy } : undefined);48
49 try {50 const page = await browser.newPage();51 return await fetchListingFromProfilePage(page, username);52 } finally {53 await browser.close();54 }55 }56}57
58async function fetchListingFromProfilePage(page: Page, username: string): Promise<ProfileListing> {59 const url = `https://www.instagram.com/${encodeURIComponent(username)}/`;60
61 try {62 await page.goto(url, { waitUntil: 'networkidle', timeout: PAGE_LOAD_TIMEOUT_MS });63 } catch (error) {64 if (!(error instanceof errors.TimeoutError)) throw error;65 }66
67 // Instagram usernames are case-insensitive; the embedded node carries the68 // canonical (lowercase) form regardless of how the input was cased.69 const found = await page.evaluate(findEmbeddedProfileAndTimeline, {70 targetUsername: username.toLowerCase(),71 maxDepth: MAX_EMBEDDED_SCRIPT_SCAN_DEPTH,72 });73
74 if (!found.profile) {75 throw new ProfileNotFoundError(`Could not find profile data embedded on the page for ${username}`);76 }77 if (found.profile.isPrivate) {78 throw new PrivateProfileError(`This profile is private: ${username}`);79 }80 if (!found.connection) {81 throw new ProfileNotFoundError(`Could not find a posts/reels connection embedded on the page for ${username}`);82 }83
84 return {85 items: found.connection.edges.map(({ code, contentType }) => ({ shortcode: code, contentType })),86 limitReached: found.connection.hasNextPage,87 };88}89
90interface EmbeddedScanArgs {91 targetUsername: string;92 maxDepth: number;93}94
95interface EmbeddedScanResult {96 profile: { pk: string; isPrivate: boolean } | null;97 connection: { hasNextPage: boolean; edges: Array<{ code: string; contentType: ContentType }> } | null;98}99
100/**101 * Runs inside the browser page (via `page.evaluate`). Instagram splits the102 * profile's `is_private` flag and its `polaris_ordered_timeline_connection`103 * across two different embedded script blocks (confirmed via live capture —104 * they never appear on the same node), so this scans twice: once to find the105 * profile node matching the requested username, once more to find the106 * connection node sharing that profile's `pk` — matching by `pk` (rather107 * than returning the first connection-shaped object found anywhere on the108 * page) guards against accidentally picking up an unrelated connection, e.g.109 * from a "suggested accounts" rail.110 */111function findEmbeddedProfileAndTimeline(args: EmbeddedScanArgs): EmbeddedScanResult {112 function scan<T>(matcher: (record: Record<string, unknown>) => T | null): T | null {113 function findMatch(value: unknown, depth: number): T | null {114 if (depth > args.maxDepth || value === null || typeof value !== 'object') return null;115 if (Array.isArray(value)) {116 for (const item of value) {117 const found = findMatch(item, depth + 1);118 if (found) return found;119 }120 return null;121 }122 const record = value as Record<string, unknown>;123 const direct = matcher(record);124 if (direct) return direct;125 for (const key of Object.keys(record)) {126 const found = findMatch(record[key], depth + 1);127 if (found) return found;128 }129 return null;130 }131
132 const scripts = Array.from(document.querySelectorAll('script[type="application/json"]'));133 for (const script of scripts) {134 let parsed: unknown;135 try {136 parsed = JSON.parse(script.textContent || '');137 } catch {138 continue;139 }140 const match = findMatch(parsed, 0);141 if (match) return match;142 }143 return null;144 }145
146 const profile = scan((record) => {147 if (typeof record.username !== 'string' || typeof record.is_private !== 'boolean' || typeof record.pk !== 'string') {148 return null;149 }150 if (record.username.toLowerCase() !== args.targetUsername) return null;151 return { pk: record.pk, isPrivate: record.is_private };152 });153
154 if (!profile) return { profile: null, connection: null };155
156 const connection = scan((record) => {157 if (record.pk !== profile.pk) return null;158 const conn = record.polaris_ordered_timeline_connection as159 | { edges?: Array<{ node?: Record<string, unknown> }>; page_info?: { has_next_page?: unknown } }160 | undefined;161 if (!conn || !Array.isArray(conn.edges)) return null;162 const edges = conn.edges163 .filter((edge): edge is { node: Record<string, unknown> } => typeof edge.node?.code === 'string')164 .map((edge) => ({165 code: edge.node.code as string,166 // Duplicated from mapXigPolarisMedia.ts's identical contentType derivation — this167 // runs inside page.evaluate() and can't import across that serialization boundary.168 // Keep both in sync if Instagram adds a new product_type.169 contentType: (edge.node.product_type === 'clips' ? 'reel' : 'post') as ContentType,170 }));171 return { edges, hasNextPage: conn.page_info?.has_next_page === true };172 });173
174 return { profile, connection };175}1export class ItemNotFoundError extends Error {}2
3export interface ItemFetcher {4 fetchRawItem(url: string): Promise<unknown>;5}1import { mapXigPolarisMedia, UnrecognizedMediaShapeError } from './mappers/xigPolarisMedia.js';2import type { ScrapedItem } from './types.js';3
4export { UnrecognizedMediaShapeError as UnrecognizedItemShapeError };5
6export function normalizeRawItem(raw: unknown): ScrapedItem {7 return mapXigPolarisMedia(raw);8}1import type { ContentType } from './types.js';2
3export class PrivateProfileError extends Error {}4export class ProfileNotFoundError extends Error {}5
6export interface ProfileListingItem {7 shortcode: string;8 contentType: ContentType;9}10
11export interface ProfileListing {12 items: ProfileListingItem[];13 limitReached: boolean;14}15
16export interface ProfileFetcher {17 fetchProfileListing(username: string): Promise<ProfileListing>;18}1export interface PlaywrightProxy {2 server: string;3 username?: string;4 password?: string;5}6
7interface ProxyInfoLike {8 hostname: string;9 port: number | string;10 username?: string;11 password: string;12}13
14export function toPlaywrightProxy(proxyInfo: ProxyInfoLike): PlaywrightProxy {15 const server = `http://${proxyInfo.hostname}:${proxyInfo.port}`;16
17 if (!proxyInfo.password) {18 return { server };19 }20
21 return { server, username: proxyInfo.username, password: proxyInfo.password };22}1export type ContentType = 'post' | 'reel';2export type MediaType = 'image' | 'video' | 'carousel';3
4export interface ScrapedItem {5 shortcode: string;6 ownerUsername: string | null;7 contentType: ContentType;8 caption: string | null;9 timestamp: number;10 likeCount: number | null;11 commentCount: number | null;12 mediaType: MediaType;13 mediaUrl: string | null;14}1# Fixtures2
3Hand-assembled from a live, logged-out capture against real Instagram pages on 2026-08-12 (via Chrome/Playwright, direct navigation to permalink URLs — plain `curl` gets an anti-bot challenge shell, not real data).4
5**Correction (same day, later pass):** an earlier version of this README and `post-carousel.json` described posts as using a different, "legacy" `shortcode`/`edge_media_*` schema, distinct from reels' `xig_polaris_media` schema. That was captured via *client-side SPA link-clicks* within an already-loaded page — a different code path than what the Actor's fetcher actually uses (direct navigation to the permalink URL). Under direct navigation, verified with a structured `JSON.parse` + recursive search (not raw substring matching, which was misleading due to other items' data being embedded nearby on the same page) — **both posts and reels use the same `xig_polaris_media` schema**, differentiated by `media_type` (`2` = video/reel, `8` = carousel, presumably `1` = single image) and `product_type` (`"clips"` = reel, `"carousel_container"` = carousel, presumably something else for a single-image/video feed post). `post-carousel.json` below reflects this corrected, verified shape.6
7**Confidence levels, per fixture:**8
9- `reel-single.json` — field names/values (`code`, `like_count`, `comment_count`, `taken_at`, `media_type: 2`, `product_type: "clips"`, `caption.text`, `video_versions[].type`, `__typename: XIGPolarisVideoMedia`) directly verified from a real public reel (`instagram.com/reel/DbvqPafyqY-/`), confirmed present under `data.xig_polaris_media.if_not_gated_logged_out` via structured search. `view_count`, `play_count`, and `duration` were checked for and confirmed **absent** from this response entirely.10- `post-carousel.json` — field names/values (`code`, `like_count: 286443`, `comment_count: 11480`, `taken_at: 1785437998`, `media_type: 8`, `product_type: "carousel_container"`, `caption.text`, `carousel_media_count: 8`, `carousel_media[0].__typename: XIGPolarisImageMedia`, `display_uri`, `image_versions2.candidates`) directly verified via structured search on a real public carousel post (`instagram.com/instagram/p/DbbY9pdm6Q2/`).11- `not-found-error.json` — the real generic GraphQL error envelope captured from a rejected logged-out query (`{"errors":[...],"extensions":{"is_final":true}}`), observed during earlier client-side-navigation testing. It's the real *envelope shape* Instagram uses for rejected queries, not a verbatim capture of a "post not found" response specifically. The Actor's actual not-found detection instead relies on the verified DOM signal (page title containing `"isn't available"`) rather than this envelope, since that's what direct navigation to a deleted post was confirmed to produce.12
13All media URLs are replaced with `https://example-cdn.test/...` placeholders — the real ones are signed and expire.{ "errors": [ { "message": "Unauthorized logged out query.", "severity": "CRITICAL", "code": 1675002 } ], "extensions": { "is_final": true }}{ "data": { "xig_polaris_media": { "__typename": "XIGPolarisCarouselMedia", "pk": "3952862887472243766", "code": "DbbY9pdm6Q2", "if_not_gated_logged_out": { "__typename": "XIGPolarisCarouselMedia", "user": { "username": "instagram", "pk": "25025320" }, "caption": { "text": "Meet the artist behind “Miss Mango” 🥭🎶\n\n@prodlaila writes, sings and produces her own music from her bedroom studio." }, "like_count": 286443, "comment_count": 11480, "taken_at": 1785437998, "media_type": 8, "product_type": "carousel_container", "carousel_media_count": 8, "display_uri": "https://example-cdn.test/placeholder-carousel-cover.jpg", "image_versions2": { "candidates": [{ "url": "https://example-cdn.test/placeholder-carousel-cover.jpg" }], "additional_candidates": {} }, "carousel_media": [ { "__typename": "XIGPolarisImageMedia", "code": "DbbY9pdm6Q2", "media_type": 1, "display_uri": "https://example-cdn.test/placeholder-carousel-child-1.jpg" } ] } } }}{ "data": { "xig_polaris_media": { "__typename": "XIGPolarisVideoMedia", "pk": "3958568374678234680", "code": "DbvqPafyqY-", "if_not_gated_logged_out": { "__typename": "XIGPolarisVideoMedia", "user": { "username": "instagram", "pk": "25025320", "full_name": "Instagram", "is_verified": true, "is_private": false, "profile_pic_url": "https://example-cdn.test/placeholder-profile.jpg" }, "caption": { "text": "From 🤝 to 🌊\n\n@mark.flow.22 finds waterfalls, mountains and hidden spots you’ve probably never seen. \n\n“I hope my videos inspire people to explore more, spend time in nature and create their own adventures. Even a short trip outdoors can become an unforgettable experience.”\n" }, "like_count": 1250208, "comment_count": 19174, "taken_at": 1786118430, "created_at": 1786118429, "media_type": 2, "product_type": "clips", "has_audio": true, "image_versions2": { "additional_candidates": { "first_frame": { "url": "https://example-cdn.test/placeholder-first-frame.jpg" } } }, "video_versions": [ { "type": 101, "url": "https://example-cdn.test/placeholder-video.mp4" } ] } } }}1import type { BrowserLauncher } from '../../src/instagram/browserFetcher.js';2
3/** A launcher stub that records its call args then throws, short-circuiting the caller4 * right after the browser-launch step — enough to assert proxy wiring without a real browser. */5export function stubLauncher(calls: unknown[]): BrowserLauncher {6 return async (options) => {7 calls.push(options);8 throw new Error('stop here — only checking launch args');9 };10}1import { describe, expect, it } from 'vitest';2import { BrowserItemFetcher } from '../../src/instagram/browserFetcher.js';3import { stubLauncher } from '../helpers/stubLauncher.js';4
5describe('BrowserItemFetcher proxy wiring', () => {6 it('launches with the given proxy option when one is provided', async () => {7 const calls: unknown[] = [];8 const fetcher = new BrowserItemFetcher({9 proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' },10 launcher: stubLauncher(calls),11 });12
13 await expect(fetcher.fetchRawItem('https://www.instagram.com/p/ABC123/')).rejects.toThrow('stop here');14
15 expect(calls).toEqual([{ proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' } }]);16 });17
18 it('launches with no proxy option when none is provided', async () => {19 const calls: unknown[] = [];20 const fetcher = new BrowserItemFetcher({ launcher: stubLauncher(calls) });21
22 await expect(fetcher.fetchRawItem('https://www.instagram.com/p/ABC123/')).rejects.toThrow('stop here');23
24 expect(calls).toEqual([undefined]);25 });26});1import { describe, expect, it } from 'vitest';2import { BrowserProfileFetcher } from '../../src/instagram/browserProfileFetcher.js';3import { stubLauncher } from '../helpers/stubLauncher.js';4
5describe('BrowserProfileFetcher proxy wiring', () => {6 it('launches with the given proxy option when one is provided', async () => {7 const calls: unknown[] = [];8 const fetcher = new BrowserProfileFetcher({9 proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' },10 launcher: stubLauncher(calls),11 });12
13 await expect(fetcher.fetchProfileListing('nasa')).rejects.toThrow('stop here');14
15 expect(calls).toEqual([{ proxy: { server: 'http://proxy.example.com:8000', username: 'u', password: 'p' } }]);16 });17
18 it('launches with no proxy option when none is provided', async () => {19 const calls: unknown[] = [];20 const fetcher = new BrowserProfileFetcher({ launcher: stubLauncher(calls) });21
22 await expect(fetcher.fetchProfileListing('nasa')).rejects.toThrow('stop here');23
24 expect(calls).toEqual([undefined]);25 });26});1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { describe, expect, it } from 'vitest';4import { normalizeRawItem, UnrecognizedItemShapeError } from '../../src/instagram/normalize.js';5
6function loadFixture(name: string): unknown {7 const path = fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url));8 return JSON.parse(readFileSync(path, 'utf8'));9}10
11describe('normalizeRawItem', () => {12 it('normalizes a real reel response', () => {13 const item = normalizeRawItem(loadFixture('reel-single.json'));14 expect(item.contentType).toBe('reel');15 expect(item.shortcode).toBe('DbvqPafyqY-');16 });17
18 it('normalizes a real carousel post response', () => {19 const item = normalizeRawItem(loadFixture('post-carousel.json'));20 expect(item.contentType).toBe('post');21 expect(item.shortcode).toBe('DbbY9pdm6Q2');22 });23
24 it('throws UnrecognizedItemShapeError for a response with no xig_polaris_media.code', () => {25 expect(() => normalizeRawItem({ errors: [{ message: 'nope' }] })).toThrow(UnrecognizedItemShapeError);26 });27});1import { describe, expect, it } from 'vitest';2import { toPlaywrightProxy } from '../../src/instagram/proxy.js';3
4describe('toPlaywrightProxy', () => {5 it('maps hostname/port/username/password to a Playwright server URL and credentials', () => {6 const result = toPlaywrightProxy({7 hostname: 'proxy.example.com',8 port: 8000,9 username: 'user',10 password: 'pass',11 });12
13 expect(result).toEqual({14 server: 'http://proxy.example.com:8000',15 username: 'user',16 password: 'pass',17 });18 });19
20 it('omits username/password when the proxy has no credentials', () => {21 const result = toPlaywrightProxy({22 hostname: 'proxy.example.com',23 port: 8000,24 password: '',25 });26
27 expect(result).toEqual({ server: 'http://proxy.example.com:8000' });28 });29
30 it('omits credentials when password is empty even if a username is present', () => {31 // password is the field Apify's ProxyInfo always populates for a real proxy; an empty32 // password means no genuine credentials, regardless of username, so we don't send Playwright33 // into a doomed auth attempt against the proxy.34 const result = toPlaywrightProxy({35 hostname: 'proxy.example.com',36 port: 8000,37 username: 'user',38 password: '',39 });40
41 expect(result).toEqual({ server: 'http://proxy.example.com:8000' });42 });43
44 it('stringifies a numeric-string port as-is', () => {45 const result = toPlaywrightProxy({46 hostname: 'proxy.example.com',47 port: '8000',48 password: '',49 });50
51 expect(result.server).toBe('http://proxy.example.com:8000');52 });53});1import { readFileSync } from 'node:fs';2import { fileURLToPath } from 'node:url';3import { describe, expect, it } from 'vitest';4import { mapXigPolarisMedia, UnrecognizedMediaShapeError } from '../../src/instagram/mappers/xigPolarisMedia.js';5
6function loadFixture(name: string): unknown {7 const path = fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url));8 return JSON.parse(readFileSync(path, 'utf8'));9}10
11describe('mapXigPolarisMedia', () => {12 it('maps a real captured reel response to the shared output shape', () => {13 expect(mapXigPolarisMedia(loadFixture('reel-single.json'))).toEqual({14 shortcode: 'DbvqPafyqY-',15 ownerUsername: 'instagram',16 contentType: 'reel',17 caption:18 'From 🤝 to 🌊\n\n@mark.flow.22 finds waterfalls, mountains and hidden spots you’ve probably never seen. \n\n“I hope my videos inspire people to explore more, spend time in nature and create their own adventures. Even a short trip outdoors can become an unforgettable experience.”\n',19 timestamp: 1786118430,20 likeCount: 1250208,21 commentCount: 19174,22 mediaType: 'video',23 mediaUrl: 'https://example-cdn.test/placeholder-video.mp4',24 });25 });26
27 it('maps a real captured carousel post response to the shared output shape', () => {28 expect(mapXigPolarisMedia(loadFixture('post-carousel.json'))).toEqual({29 shortcode: 'DbbY9pdm6Q2',30 ownerUsername: 'instagram',31 contentType: 'post',32 caption: 'Meet the artist behind “Miss Mango” 🥭🎶\n\n@prodlaila writes, sings and produces her own music from her bedroom studio.',33 timestamp: 1785437998,34 likeCount: 286443,35 commentCount: 11480,36 mediaType: 'carousel',37 mediaUrl: 'https://example-cdn.test/placeholder-carousel-cover.jpg',38 });39 });40
41 it('throws UnrecognizedMediaShapeError when the response has no xig_polaris_media.code', () => {42 expect(() => mapXigPolarisMedia({ data: {} })).toThrow(UnrecognizedMediaShapeError);43 });44});1# 01 — Scaffold the Actor project2
3**What to build:** A running, empty-but-valid Apify Actor project (Node.js + Crawlee + Apify SDK) that boots, accepts a profile-username-or-post/reel-URL input, and can push to a local Dataset under Apify SDK's local storage emulation. No real scraping logic yet — this ticket exists to make the pipeline and test harness real so every later ticket can build and test against it rather than bootstrapping from scratch.4
5**Blocked by:** None — can start immediately6
7**Status:** ready-for-agent8
9- [ ] Actor project initializes via the Apify SDK (`Actor.init()`/`Actor.main()`) and exits cleanly on an empty run10- [ ] INPUT_SCHEMA declares a `username` field and a `postOrReelUrl` field (at least one required)11- [ ] A test harness exists that runs the Actor entrypoint against Apify SDK's local storage emulation and can assert on local Dataset contents, with no real Apify Cloud run and no real network call to Instagram12- [ ] Crawlee is wired in with a placeholder request handler (no real Instagram calls yet) so the hybrid direct-API/browser-automation-fallback routing point exists for ticket 02 to fill in13- [ ] README stub and `actor.json` exist with placeholder metadata (title, description) — full pricing/disclaimer content is deferred to ticket 0514
15## Comments1# 02 — Single post/reel URL lookup2
3**What to build:** Given a post or reel URL as input, the Actor fetches that single item via a browser-driven request (Playwright, loading the permalink directly and reading the data Instagram embeds server-side), normalizes it into the shared output schema, pushes it to the Dataset, and emits the corresponding pay-per-event charge. This is the first real end-to-end scrape, and it establishes the request layer, normalization shape, and charging pattern that tickets 03 and 04 reuse.4
5**Blocked by:** 01 — Scaffold the Actor project6
7**Status:** ready-for-agent8
9- [ ] Given a valid public post URL, the Dataset receives one item matching the shared output schema (shortcode, owner username, content type = post, caption, timestamp, like count, comment count, media type, media URL)10- [ ] Given a valid public reel URL, the Dataset item matches the shared output schema (content type = reel) — **no view count or duration required in v1**11- [ ] Fetching is browser-driven (Playwright) as the default and only path for v1; direct-HTTP is out of scope for this ticket12- [ ] A pay-per-event charge is emitted once per successfully scraped item, tagged by content type13- [ ] Given a URL pointing at a deleted/nonexistent post or reel, the run completes with a clear error/empty-with-reason result rather than a silent empty Dataset or an unhandled crash14- [ ] Tests use the confirmed seam (Actor input → real pipeline → Dataset output) with the fetcher injected/stubbed via the real captured fixtures — no live network calls in the test suite15
16## Comments17
18**2026-08-12 — Corrections from live capture**, superseding the original AC2/AC3 wording (the checkboxes above already reflect these):19
20- **Fetch strategy is inverted from the original plan.** Plain HTTP requests (no browser) get Instagram's anti-bot challenge shell, not real data — confirmed via multiple direct attempts. A real browser session succeeds. So **browser-driven fetching (Playwright) is the primary path for v1**, not a fallback. A lighter direct-HTTP path (replicating the session cookies/CSRF token a browser sends) is a possible *future* optimization if that request recipe is captured later — it is explicitly not part of this ticket's scope.21- **`view_count`, `play_count`, and `duration` are dropped from required reel output.** Confirmed absent from the anonymous single-item reel query — not an extraction gap, the fields aren't present in the response at all.22- **One schema, not two.** An earlier pass through this correction claimed posts and reels use two structurally different schemas. Further verification (structured `JSON.parse` + recursive search, not raw substring matching, which was misleading due to other items' data sitting nearby on the same page) showed that's wrong for the fetch strategy actually used: on **direct navigation** to a permalink (what the browser-driven fetcher does), both posts and reels come back as the same `xig_polaris_media` shape, differentiated by `media_type`/`product_type` fields — not by two schemas. One mapper handles both. (The two-schema observation was real but came from a different code path — client-side SPA link-clicks within an already-loaded page — that the Actor doesn't use.)23- **Live-verified end-to-end** against real URLs (a real reel, a real carousel post, and a real deleted post) after the above correction — fetch → normalize produced correct output for all three, including a genuinely playable media URL and a correctly-thrown not-found error.24- Verified fixtures (real field names/values, captured live) are at `test/fixtures/` — see `test/fixtures/README.md` for exactly what's confirmed vs. best-effort in each one.25
26**2026-08-12 — Code review fixes applied:**27
28- Docker base image switched to `apify/actor-node-playwright-chrome:20` (both build stages) — the plain `apify/actor-node:20` image has no Chromium binary or system libraries, so the deployed Actor could not have launched a browser at all.29- Not-found detection requires both the page title (`"isn't available"`) and body text (`"link may be broken"`) to match, rather than title alone — a real post whose own caption happens to contain the phrase "isn't available" would otherwise false-positive as deleted.30- `page.goto()` failures are now only silently tolerated when they're a Playwright `TimeoutError`; any other navigation failure (DNS, connection reset, proxy failure) now propagates instead of being misreported as "item not found."31- The mapper now requires `if_not_gated_logged_out` to be present with a numeric `taken_at` before mapping, rather than defaulting missing data to `{}`/`null`/`0` and pushing a near-empty item as if it were a successful scrape.32- `extractShortcode()` now throws `InvalidInputError` (consistent with the rest of input validation) instead of a bare `Error`.1# 03 — Profile posts + reels scraping2
3**What to build:** Given a public profile username, the Actor scrapes that profile's posts and reels (paginating as needed) and pushes each as a Dataset item, reusing the output schema, request layer, and charging pattern established in ticket 02.4
5**Blocked by:** 02 — Single post/reel URL lookup6
7**Status:** ready-for-agent8
9- [ ] Given a public username with both posts and reels, the Dataset receives one item per post and per reel, each correctly tagged by content type and matching the shared schema (reusing ticket 02's per-item fetch/normalize pipeline)10- [ ] Pagination continues until the profile's available public posts/reels are exhausted, **or until the anonymous-access pagination limit is hit** — in the latter case the run still completes successfully with whatever was collected, and reports a clear status message that results may be incomplete, rather than silently truncating11- [ ] A pay-per-event charge is emitted once per successfully scraped item, same pattern as ticket 0212- [ ] Given a private profile, the run completes with a clear error/empty-with-reason result rather than a silent empty Dataset13- [ ] Given a public profile with zero posts/reels, the run completes with an explicit empty result distinguishable from an error14- [ ] Tests use the same confirmed seam as ticket 02, with canned network responses covering the profile-listing endpoint(s)15
16## Comments17
18**2026-08-17 — Corrections from live capture**, superseding the original AC1/AC2 wording (the checkboxes above already reflect these):19
20- **The profile-listing connection is a different, lighter shape than the single-item permalink shape ticket 02 built against.** Live capture against `instagram.com/nasa` (logged out) found the listing lives at `xig_user_by_username.polaris_ordered_timeline_connection` (cursor-paginated via `edges`/`page_info.end_cursor`/`page_info.has_next_page`), with posts and reels combined in one connection and differentiated by `product_type` (e.g. `"clips"` for reels). But each node only carries `pk`, `code`, `caption`, `display_uri`, `media_type`, `product_type`, `user{username}` — no `taken_at`, `like_count`, `comment_count`, or playable `video_versions`. It cannot fill the shared output schema on its own. **Decision:** use this connection only to discover `(shortcode, contentType)` pairs, then fetch each item's full detail by reusing ticket 02's `ItemFetcher`/`mapXigPolarisMedia` pipeline unchanged against its permalink — no new mapper needed.21- **Anonymous (logged-out) pagination is capped at the initial embed — there is no further page to fetch at all.** A first pass (browser UI scrolling) looked like it was loading a genuine second page before hitting a "Log in to continue watching" wall around ~20-24 items. A follow-up Playwright probe (same runtime the real fetcher uses) instrumented `page.on('response')` for `/api/graphql` calls while scrolling: none of the captured responses ever contained more `polaris_ordered_timeline_connection` data, and two came back as an explicit `{"errors":[{"message":"Unauthorized logged out query.","code":1675002}]}` — confirming the backend itself rejects a real next-page fetch while logged out, rather than the UI just choosing to nag. What looked like "one more page" in the browser was the already-embedded connection's items being progressively rendered, not new data. The initial embed reliably contained exactly 12 edges with `page_info.has_next_page: true` for a profile with many more items. **Implementation implication:** `BrowserProfileFetcher` only ever reads the single initial embed — no scrolling or pagination-request handling needed — and treats `has_next_page: true` on that embed directly as `limitReached: true`, since attempting a real next page is known to fail. This is a simpler implementation than originally planned, not a bigger one.22- **Decision (user-confirmed):** stop at the wall and report partial results with a clear reason (`actor.setStatusMessage`), rather than erroring the whole run or silently truncating. Profiles with fewer items than the initial embed provides are unaffected and scrape to completion as originally specified.23- The private-profile path (`is_private: true` on the embedded user node) is confirmed structurally — the field and its meaning are documented by Instagram's own client behavior — but wasn't live-verified end-to-end against a known private account (none was available to test against). Flagged here at the same confidence level ticket 02's fixtures README used for best-effort vs. confirmed paths.24- `is_private` is present directly on the profile's bootstrap user node (`xig_user_by_username.is_private`) — private-profile detection needs no extra live-verification beyond what's already captured.1# 04 — Optional bring-your-own proxy configuration2
3**What to build:** An end user can supply their own proxy configuration as Actor input; when present, all outbound requests (both the direct API path and the browser-automation fallback) route through it. When absent, the Actor runs proxy-less exactly as it does in ticket 02/03's baseline. No bundled/paid proxy is ever used by default — this keeps proxy cost, if any, entirely on the end user.4
5**Blocked by:** 02 — Single post/reel URL lookup6
7**Status:** ready-for-agent8
9- [ ] INPUT_SCHEMA gains an optional proxy configuration field (following Apify's standard proxy configuration input convention)10- [ ] When a proxy configuration is supplied, requests in a test run are verifiably routed through it (asserted via the browser-launch call receiving the expected proxy option)11- [ ] When no proxy configuration is supplied, behavior is unchanged from ticket 02/03's baseline12- [ ] README/documentation notes this as an explicit, known limitation: omitting proxy configuration means a higher block/ban rate is possible13
14## Comments15
16**2026-08-17 — Correction, superseding the original AC1 wording:** "both the direct API path and the browser-automation fallback" referred to an architecture ticket 02 abandoned (direct-HTTP requests get Instagram's anti-bot challenge shell, not real data — see ticket 02's Corrections note). There is only one request path in this project: browser-driven via Playwright. The checkbox above already reflects this — proxy wiring applies to `chromium.launch()`'s `proxy` option in `BrowserItemFetcher`/`BrowserProfileFetcher`, nothing else. AC2's wording was also adjusted: there's no mocked network layer in this project's test suite (per ticket 02/03 precedent, the browser-driven fetchers have no unit-testable network seam) — instead, both fetcher classes gain an injectable browser-launcher function (defaulting to the real `chromium.launch`), and a test asserts the launcher is called with the correct `proxy` option. This proves the wiring is correct without needing a real proxy or real network calls; it does not live-verify actual traffic routing through a live proxy, since no test proxy was available (same best-effort/confirmed distinction used for ticket 03's private-profile path).1# 05 — Store listing readiness2
3**What to build:** The Actor is ready to publish to the Apify Store: pay-per-event pricing configured per content-type event, a README documenting inputs/outputs/limitations/maintenance stance, and a ToS/liability disclaimer. This is the final v1 ticket — after it, the Actor can actually be listed and sold.4
5**Blocked by:** 02 — Single post/reel URL lookup, 03 — Profile posts + reels scraping, 04 — Optional bring-your-own proxy configuration6
7**Status:** ready-for-agent8
9- [ ] Pay-per-event pricing is configured in the Actor's pricing metadata for each event emitted by tickets 02 and 03, priced below the lowest current comparable competitor identified during scoping10- [ ] README documents: required/optional inputs, output schema, the anonymous-only v1 auth model, the optional BYO-proxy input, and the best-effort ("ship it, revisit occasionally") maintenance stance11- [ ] A standard ToS/liability disclaimer (lawful/research use only, used at runner's own risk, no ToS-compliance guarantee) is present on the listing12- [ ] `actor.json`/store metadata accurately reflects only the v1 feature set (no references to Stories/Highlights, transcripts, or follower stats, which are out of scope for this spec)13
14## Comments15
16**2026-08-26**: Verified current state before starting the agent:17
18- **AC1 wording corrected.** The spec's original phrasing ("post scraped," "reel scraped," "URL lookup") implied three chargeable events. The actual implementation (`src/actor.ts::pushScrapedItem`) only ever emits `post-scraped` or `reel-scraped` via `actor.pushData(item, `${item.contentType}-scraped`)` — a single-URL lookup charges whichever of those two matches the looked-up item's content type; there is no separate "URL lookup" event. AC1 should read: configure Apify Store pricing for the two events `post-scraped` and `reel-scraped`, each priced below the lowest current comparable competitor.19- **AC1 and AC3 are partly outside what a coding ticket can deliver.** Traced through `node_modules/apify` — pay-per-event pricing is not a repo file; it's configured on the Apify Console's Publication/Monetization tab at actor-version-publish time, keyed by the event-name strings the code already calls (`post-scraped`/`reel-scraped`). Likewise, a ToS/liability disclaimer "present on the listing" is a Console-side Store listing concern; the README does get pulled in as the Store listing description, so drafting the disclaimer text there covers the content, but actually attaching pricing numbers and confirming the listing is a manual publish-time step (Console UI), not something `/implement`'s TDD/code-review loop can do inside this git repo. **Scope for `/implement`:** draft the disclaimer text in the README, and document the two event names clearly enough (e.g. in the README's pricing note) that whoever publishes can fill in the Console pricing form correctly. The actual Console pricing entry + publish action is a separate manual step afterward (candidate for the `wizard` skill when the user is ready to actually publish) — not part of this ticket's automated work.20- **AC4 already satisfied.** Grepped `.actor/actor.json`, `.actor/input_schema.json`, and `README.md` for "stories", "highlights", "transcript", "follower" — no matches. No stale out-of-scope references exist to remove.21- **AC2 (README pass) is the ticket's main real deliverable.** Confirmed the README is badly stale: still says "Status: scaffolding only (ticket 01). No scraping logic exists yet," despite tickets 02–04 being fully implemented and committed. Needs a full rewrite covering required/optional inputs (`username` XOR `postOrReelUrl`, optional `proxyConfiguration`), the actual output schema (shortcode/id, owner username, content type, caption, timestamp, like/comment count, media type, media URL(s)), the anonymous-only v1 auth model, the BYO-proxy input (already partially documented), and the best-effort maintenance stance.1import type { ContentType, MediaType, ScrapedItem } from '../types.js';2
3export class UnrecognizedMediaShapeError extends Error {}4
5interface RawMediaDetails {6 user?: { username?: unknown };7 caption?: { text?: unknown };8 like_count?: unknown;9 comment_count?: unknown;10 taken_at?: unknown;11 media_type?: unknown;12 product_type?: unknown;13 display_uri?: unknown;14 image_versions2?: { candidates?: Array<{ url?: unknown }> };15 video_versions?: Array<{ url?: unknown }>;16}17
18interface RawMediaResponse {19 data?: {20 xig_polaris_media?: {21 code?: unknown;22 if_not_gated_logged_out?: RawMediaDetails;23 };24 };25}26
27/**28 * Maps Instagram's `xig_polaris_media` response shape — used for both posts29 * and reels on direct permalink navigation, differentiated by `media_type`/30 * `product_type` rather than by two separate schemas — to the shared output31 * shape. See `test/fixtures/README.md` for what's verified vs. best-effort.32 */33export function mapXigPolarisMedia(raw: unknown): ScrapedItem {34 const media = (raw as RawMediaResponse)?.data?.xig_polaris_media;35 if (!media || typeof media.code !== 'string') {36 throw new UnrecognizedMediaShapeError('Response did not contain a recognizable xig_polaris_media.code.');37 }38
39 const details = media.if_not_gated_logged_out;40 if (!details || typeof details !== 'object' || typeof details.taken_at !== 'number') {41 throw new UnrecognizedMediaShapeError(42 'Response was missing if_not_gated_logged_out (or its taken_at field) — refusing to push a near-empty item.',43 );44 }45
46 const mediaType = mapMediaType(details.media_type);47 const contentType: ContentType = details.product_type === 'clips' ? 'reel' : 'post';48
49 return {50 shortcode: media.code,51 ownerUsername: typeof details.user?.username === 'string' ? details.user.username : null,52 contentType,53 caption: typeof details.caption?.text === 'string' ? details.caption.text : null,54 timestamp: details.taken_at,55 likeCount: typeof details.like_count === 'number' ? details.like_count : null,56 commentCount: typeof details.comment_count === 'number' ? details.comment_count : null,57 mediaType,58 mediaUrl: extractMediaUrl(details, mediaType),59 };60}61
62function mapMediaType(rawMediaType: unknown): MediaType {63 if (rawMediaType === 2) return 'video';64 if (rawMediaType === 8) return 'carousel';65 return 'image';66}67
68/**69 * Returns a single representative URL — the cover image for a carousel, not70 * every slide. `carousel_media[]` (verified present, e.g. 8 entries on the71 * captured fixture) is intentionally not surfaced: the ticket's output72 * schema specifies one `mediaUrl` per item, singular.73 */74function extractMediaUrl(details: RawMediaDetails, mediaType: MediaType): string | null {75 if (mediaType === 'video') {76 const url = details.video_versions?.[0]?.url;77 return typeof url === 'string' ? url : null;78 }79 if (typeof details.display_uri === 'string') return details.display_uri;80 const candidateUrl = details.image_versions2?.candidates?.[0]?.url;81 return typeof candidateUrl === 'string' ? candidateUrl : null;82}