Substack Scraper: Posts, Comments & Authors
Pricing
from $3.00 / 1,000 posts
Scrape any Substack publication: post archives, article text, comments, author profiles and subscriber signals. Search across newsletters and export structured data for research, monitoring and AI datasets. No browser. Output to CSV, JSON or Excel.
node_modulesstorageapify_storagecrawlee_storage.git.vscode.idea*.logdataset_*.jsonscripts# Dependenciesnode_modules
# Apify / Crawlee local storagestorageapify_storagecrawlee_storage
# Dataset exportsdataset_*.json
# Environment & secrets.env.env.*
# Logs & OS files*.log.DS_StoreThumbs.db{ "name": "substack-scraper", "version": "1.1.0", "type": "module", "description": "Apify actor that scrapes Substack publications, posts, authors and comments via the public JSON API.", "engines": { "node": ">=20" }, "dependencies": { "apify": "^3.7.2", "crawlee": "^3.17.0", "cheerio": "^1.2.0" }, "scripts": { "start": "node src/main.js", "test": "node --check src/main.js && node --check src/routes.js && node --check src/state.js && node scripts/check-classify.mjs" }, "author": "Callum", "license": "ISC"}{ "actorSpecification": 1, "name": "substack-scraper-posts-comments-authors", "title": "Substack Scraper", "description": "Scrape Substack newsletters at scale: full post archives with article text, comments, author profiles, and publication stats like subscriber counts. Works with any Substack URL or custom domain. Fast API-based scraping with no browser, pay per result. Export to CSV, JSON, Excel, or API.", "version": "1.2", "meta": { "templateId": "js-crawlee-cheerio" }, "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "minMemoryMbytes": 256, "maxMemoryMbytes": 1024, "defaultMemoryMbytes": 512, "storages": { "dataset": { "actorSpecification": 1, "fields": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "type": { "type": "string", "enum": ["post", "publication", "author", "comment"], "description": "Record discriminator" }, "id": { "type": ["integer", "string", "null"] }, "title": { "type": ["string", "null"] }, "subtitle": { "type": ["string", "null"] }, "slug": { "type": ["string", "null"] }, "url": { "type": ["string", "null"] }, "publication": { "type": ["string", "null"] }, "publicationName": { "type": ["string", "null"] }, "publishedAt": { "type": ["string", "null"], "format": "date-time" }, "audience": { "type": ["string", "null"] }, "isPaid": { "type": ["boolean", "null"] }, "postType": { "type": ["string", "null"] }, "coverImage": { "type": ["string", "null"] }, "description": { "type": ["string", "null"] }, "truncatedBodyText": { "type": ["string", "null"] }, "bodyHtml": { "type": ["string", "null"] }, "bodyText": { "type": ["string", "null"] }, "wordcount": { "type": ["integer", "null"] }, "reactionCount": { "type": ["integer", "null"] }, "commentCount": { "type": ["integer", "null"] }, "restacks": { "type": ["integer", "null"] }, "author": { "type": ["string", "null"] }, "authors": { "type": ["array", "null"] }, "postTags": { "type": ["array", "null"] }, "note": { "type": ["string", "null"] }, "scrapedAt": { "type": "string", "format": "date-time" }, "name": { "type": ["string", "null"] }, "subdomain": { "type": ["string", "null"] }, "customDomain": { "type": ["string", "null"] }, "heroText": { "type": ["string", "null"] }, "logoUrl": { "type": ["string", "null"] }, "copyright": { "type": ["string", "null"] }, "language": { "type": ["string", "null"] }, "foundingPlanName": { "type": ["string", "null"] }, "paidSubscribers": { "type": ["integer", "null"] }, "totalSubscribers": { "type": ["integer", "null"] }, "subscriberCountString": { "type": ["string", "null"] }, "categoryName": { "type": ["string", "null"] }, "createdAt": { "type": ["string", "null"], "format": "date-time" }, "handle": { "type": ["string", "null"] }, "bio": { "type": ["string", "null"] }, "photoUrl": { "type": ["string", "null"] }, "profileUrl": { "type": ["string", "null"] }, "subscriptions": { "type": ["array", "null"] }, "publications": { "type": ["array", "null"] }, "postId": { "type": ["integer", "null"] }, "postSlug": { "type": ["string", "null"] }, "postTitle": { "type": ["string", "null"] }, "parentId": { "type": ["integer", "null"] }, "depth": { "type": ["integer", "null"] }, "body": { "type": ["string", "null"] }, "authorName": { "type": ["string", "null"] }, "authorHandle": { "type": ["string", "null"] }, "authorId": { "type": ["integer", "null"] } } }, "views": { "posts": { "title": "Posts", "description": "Scraped articles. All record types share one dataset — filter exports by the `type` field (post / publication / author / comment); rows of other types show empty cells in this view.", "transformation": { "fields": [ "type", "title", "subtitle", "author", "publication", "publicationName", "publishedAt", "url", "audience", "isPaid", "wordcount", "reactionCount", "commentCount", "restacks", "postTags", "coverImage", "bodyText", "scrapedAt" ] }, "display": { "component": "table", "properties": { "type": { "label": "Type", "format": "text" }, "title": { "label": "Title", "format": "text" }, "subtitle": { "label": "Subtitle", "format": "text" }, "author": { "label": "Author", "format": "text" }, "publication": { "label": "Publication", "format": "text" }, "publicationName": { "label": "Publication name", "format": "text" }, "publishedAt": { "label": "Published", "format": "date" }, "url": { "label": "URL", "format": "link" }, "audience": { "label": "Audience", "format": "text" }, "isPaid": { "label": "Paid?", "format": "boolean" }, "wordcount": { "label": "Words", "format": "number" }, "reactionCount": { "label": "Likes", "format": "number" }, "commentCount": { "label": "Comments", "format": "number" }, "restacks": { "label": "Restacks", "format": "number" }, "postTags": { "label": "Tags", "format": "array" }, "coverImage": { "label": "Cover", "format": "image" }, "bodyText": { "label": "Body", "format": "text" }, "scrapedAt": { "label": "Scraped", "format": "date" } } } }, "publications": { "title": "Publications", "description": "Publication metadata records (type = publication).", "transformation": { "fields": [ "type", "name", "subdomain", "customDomain", "url", "description", "categoryName", "language", "totalSubscribers", "paidSubscribers", "subscriberCountString", "logoUrl", "createdAt", "scrapedAt" ] }, "display": { "component": "table", "properties": { "type": { "label": "Type", "format": "text" }, "name": { "label": "Name", "format": "text" }, "subdomain": { "label": "Subdomain", "format": "text" }, "customDomain": { "label": "Custom domain", "format": "text" }, "url": { "label": "URL", "format": "link" }, "description": { "label": "Description", "format": "text" }, "categoryName": { "label": "Category", "format": "text" }, "language": { "label": "Language", "format": "text" }, "totalSubscribers": { "label": "Subscribers", "format": "number" }, "paidSubscribers": { "label": "Paid subs", "format": "number" }, "subscriberCountString": { "label": "Subs (text)", "format": "text" }, "logoUrl": { "label": "Logo", "format": "image" }, "createdAt": { "label": "Created", "format": "date" }, "scrapedAt": { "label": "Scraped", "format": "date" } } } }, "authors": { "title": "Authors", "description": "Author profile records (type = author).", "transformation": { "fields": [ "type", "name", "handle", "profileUrl", "bio", "photoUrl", "publications", "subscriptions", "scrapedAt" ] }, "display": { "component": "table", "properties": { "type": { "label": "Type", "format": "text" }, "name": { "label": "Name", "format": "text" }, "handle": { "label": "Handle", "format": "text" }, "profileUrl": { "label": "Profile", "format": "link" }, "bio": { "label": "Bio", "format": "text" }, "photoUrl": { "label": "Photo", "format": "image" }, "publications": { "label": "Writes for", "format": "array" }, "subscriptions": { "label": "Subscribes to", "format": "array" }, "scrapedAt": { "label": "Scraped", "format": "date" } } } }, "comments": { "title": "Comments", "description": "Comment records (type = comment), flattened from nested threads with depth/parentId.", "transformation": { "fields": [ "type", "postTitle", "postSlug", "publication", "authorName", "authorHandle", "body", "depth", "parentId", "publishedAt", "reactionCount", "scrapedAt" ] }, "display": { "component": "table", "properties": { "type": { "label": "Type", "format": "text" }, "postTitle": { "label": "Post", "format": "text" }, "postSlug": { "label": "Post slug", "format": "text" }, "publication": { "label": "Publication", "format": "text" }, "authorName": { "label": "Author", "format": "text" }, "authorHandle": { "label": "Handle", "format": "text" }, "body": { "label": "Comment", "format": "text" }, "depth": { "label": "Depth", "format": "number" }, "parentId": { "label": "Parent", "format": "number" }, "publishedAt": { "label": "Posted", "format": "date" }, "reactionCount": { "label": "Likes", "format": "number" }, "scrapedAt": { "label": "Scraped", "format": "date" } } } } } } }}FROM apify/actor-node:22
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm ci --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
COPY . ./
CMD npm start --silent{ "title": "Substack Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Substack URLs to scrape: publication homepages (subdomain or custom domain), individual post URLs, author profiles (https://substack.com/@handle), and open.substack.com share links. Leave empty only if you enable Discovery mode below.", "editor": "requestListSources", "prefill": [{ "url": "https://noahpinion.substack.com" }] }, "mode": { "title": "Scrape mode", "type": "string", "description": "What to scrape for each publication URL. 'posts' = publication metadata + all posts in the archive. 'publication' = publication metadata only. Post and author URLs are always scraped as posts/authors regardless of this setting ('author' is accepted as an alias of 'publication').", "editor": "select", "enum": ["posts", "publication", "author"], "enumTitles": [ "Posts — full archive of each publication", "Publication metadata only", "Publication metadata only (legacy alias)" ], "default": "posts" }, "maxPostsPerPublication": { "title": "Max posts per publication", "type": "integer", "description": "Limit the number of posts scraped per publication (lower = cheaper runs). Set 0 to scrape the ENTIRE archive — large publications can have thousands of posts. Leaving the field empty uses the default of 50.", "editor": "number", "minimum": 0, "default": 50 }, "includeContent": { "title": "Include full post content", "type": "boolean", "description": "Fetch each post's full HTML + plain text body. Adds one request per post but gives you the actual article content.", "editor": "checkbox", "default": true }, "includeComments": { "title": "Include comments", "type": "boolean", "description": "Fetch the full comment thread for each post (adds one request per post). Note: every comment becomes a separate result in the dataset, and popular posts can have hundreds of comments.", "editor": "checkbox", "default": false }, "onlyFreePosts": { "title": "Only free posts", "type": "boolean", "description": "Skip paid/subscriber-only posts in publication archives (their bodies can't be fully scraped anyway). Post URLs given directly in Start URLs are always scraped.", "editor": "checkbox", "default": false }, "searchQuery": { "title": "Search query (optional)", "type": "string", "description": "Filter each publication archive by a search term. Only posts matching the term are scraped.", "editor": "textfield" }, "discoveryMode": { "title": "Discovery mode", "type": "string", "description": "Auto-discover publications without providing URLs — seeds from 5 curated top publications and expands through each publication's /recommendations page until the limit below is hit. 'Search' additionally filters every discovered publication's archive by the Discovery query. Off by default: a discovery run scrapes MANY publications and produces a correspondingly large (and more expensive) dataset.", "editor": "select", "enum": ["none", "leaderboard", "search"], "enumTitles": [ "Off — use Start URLs", "Top publications (recommendation graph)", "Search (discover + filter archives by keyword)" ], "default": "none", "sectionCaption": "Discovery", "sectionDescription": "Scrape many publications automatically instead of (or in addition to) Start URLs." }, "discoveryQuery": { "title": "Discovery query", "type": "string", "description": "Used with 'Search' Discovery mode: keyword used to filter the archive of each discovered publication. Ignored otherwise.", "editor": "textfield", "example": "AI" }, "maxPublicationsToDiscover": { "title": "Max publications to discover", "type": "integer", "description": "Cap on how many publications Discovery mode will feed into the scraper (the 5 seed publications count toward it; Start URLs do not). Lower = cheaper runs.", "editor": "number", "minimum": 1, "maximum": 500, "default": 25 }, "maxConcurrency": { "title": "Max concurrency", "type": "integer", "description": "Parallel requests. Higher = faster runs; proxy rotation is handled automatically, but very aggressive settings can hit Substack rate limits.", "editor": "number", "minimum": 1, "maximum": 25, "default": 5, "sectionCaption": "Performance" } }, "required": []}{ "actorOutputSchemaVersion": 1, "title": "Output of Substack Scraper", "description": "All scraped records are stored in the run's default dataset. Each record carries a `type` field (post / publication / author / comment) — filter on it to split record types. Per-type table views (Posts, Publications, Authors, Comments) are available in the Output tab.", "properties": { "results": { "type": "string", "title": "All records", "description": "Every scraped record as JSON: posts (full article text, engagement metrics), publications (metadata, subscriber counts), author profiles, and comment threads. Distinguish records by the `type` field.", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "posts": { "type": "string", "title": "Posts", "description": "Records of type `post`: title, author, publish date, full HTML and plain-text body, word count, reactions, comments, restacks, tags, paywall status.", "template": "{{links.apiDefaultDatasetUrl}}/items?view=posts" }, "publications": { "type": "string", "title": "Publications", "description": "Records of type `publication`: name, domain, description, language, subscriber counts (when public), creation date.", "template": "{{links.apiDefaultDatasetUrl}}/items?view=publications" }, "authors": { "type": "string", "title": "Authors", "description": "Records of type `author`: name, handle, bio, photo, publications they write for and subscribe to.", "template": "{{links.apiDefaultDatasetUrl}}/items?view=authors" }, "comments": { "type": "string", "title": "Comments", "description": "Records of type `comment`: flattened comment threads with body, author, depth, parent id, publish date, and reactions.", "template": "{{links.apiDefaultDatasetUrl}}/items?view=comments" } }}1# Substack Scraper2
3**Scrape any Substack newsletter, post, author, or comment — fast, cheap, and at scale.**4
5This Apify actor extracts structured data from Substack publications via their public JSON API. No browser, no JavaScript rendering, no login required. Built for newsletter research, content monitoring, author discovery, competitive intelligence, and LLM training datasets.6
7## What you can scrape8
9- **Substack posts** — title, subtitle, full HTML and plain-text body, word count, publish date, tags, cover image, paywall status, reactions, comment count, restacks10- **Substack publications** — name, subdomain, custom domain, description, logo, category, language, subscriber count (when public), founding plan11- **Substack authors** — profile, handle, bio, photo, the publications they write for, the publications they subscribe to12- **Substack comments** — full nested comment threads, author handles, publish dates, reactions, reply depth13
14Works with any Substack URL: `https://*.substack.com`, custom domains (`https://stratechery.com`), individual post URLs, `https://substack.com/@handle` author profiles, and `https://open.substack.com/pub/...` share links.15
16## Why use this Substack scraper17
18- **Pay only for data, not for browser time** — no Playwright, no rendering overhead, no per-minute compute billing. You pay per result, and failed requests are never charged.19- **Full archives, not just the front page** — paginates through the entire publication archive until the very first post.20- **Clean, typed output** — one dataset with a `type` field (`post` / `publication` / `author` / `comment`) and per-type table views, so you can export straight to BI tools, CSV, JSON, Excel, or Google Sheets.21- **No duplicates, no surprises** — every post is delivered exactly once, limits are enforced even across platform restarts, and proxy rotation is handled for you.22
23## Common use cases24
25- **Newsletter research** — download the full archive of a competitor's Substack for content analysis, topic clustering, or SEO research26- **Content monitoring** — schedule a daily run with `maxPostsPerPublication: 5` to capture new posts from a tracked list of newsletters and pipe to Slack or email27- **Author discovery and lead generation** — crawl author profiles to map who writes for which publications, then export handles for outreach28- **LLM training data** — bulk-extract long-form Substack content (with word counts and metadata) for fine-tuning datasets29- **Competitive intelligence** — track subscriber counts, post frequency, paywall strategy, and engagement metrics (reactions, comments, restacks) across a competitor set30- **Academic and journalism research** — gather statements, essays, and commentary from Substack writers with citable timestamps31- **Archiving and backup** — export your own Substack publication before a migration32
33## Input34
35| Field | Type | Default | Description |36|---|---|---|---|37| `startUrls` | array of URLs | — | Substack publication, post, or author URLs. Leave empty only when using Discovery mode |38| `mode` | `posts` / `publication` | `posts` | What to pull for each publication URL |39| `maxPostsPerPublication` | integer | 50 | Cap per publication. `0` = entire archive. Lower = cheaper |40| `includeContent` | boolean | true | Fetch each post's full HTML body |41| `includeComments` | boolean | false | Fetch comments for each post (each comment is a separate result) |42| `onlyFreePosts` | boolean | false | Skip paid / subscriber-only posts in archives |43| `searchQuery` | string | — | Filter the publication archive by keyword |44| `discoveryMode` | `none` / `leaderboard` / `search` | `none` | Auto-discover many publications without providing URLs |45| `discoveryQuery` | string | — | Keyword for `search` discovery |46| `maxPublicationsToDiscover` | integer | 25 | Cap on discovered publications. Lower = cheaper |47| `maxConcurrency` | integer | 5 | Parallel requests |4849### Discovery mode — scrape many publications without a list50
51If you don't have a list of specific newsletters, turn on **Discovery mode** and the actor will find publications for you:52
53- **Top publications** (`leaderboard`) — seeds from 5 curated top Substacks and expands through each publication's recommendations until the limit is hit54- **Search** (`search`) — same expansion, plus your `discoveryQuery` keyword filters every discovered publication's archive55
56Each discovered publication is then scraped using the same `mode` / `maxPostsPerPublication` settings as `startUrls`, so you can go from zero URLs to a full corpus in one run. Discovery is **off by default** — a discovery run scrapes many publications and produces a correspondingly large dataset.57
58```json59{60 "discoveryMode": "search",61 "discoveryQuery": "AI",62 "maxPublicationsToDiscover": 50,63 "mode": "posts",64 "maxPostsPerPublication": 20,65 "includeContent": true66}67```68
69## Example input70
71```json72{73 "startUrls": [74 { "url": "https://www.thefitzwilliam.com" },75 { "url": "https://noahpinion.substack.com" },76 { "url": "https://substack.com/@mattyglesias" }77 ],78 "mode": "posts",79 "maxPostsPerPublication": 100,80 "includeContent": true,81 "includeComments": false82}83```84
85## Output86
87All records land in the run's dataset with a `type` discriminator (`post`, `publication`, `author`, `comment`). The Output tab offers per-type table views (**Posts**, **Publications**, **Authors**, **Comments**); for exports, filter on the `type` field to split record types into separate files.88
89### Post record90
91```json92{93 "type": "post",94 "id": 123456,95 "title": "Why newsletters won",96 "slug": "why-newsletters-won",97 "url": "https://example.substack.com/p/why-newsletters-won",98 "publication": "example",99 "publicationName": "The Example",100 "publishedAt": "2026-02-01T14:00:00Z",101 "audience": "everyone",102 "isPaid": false,103 "author": "Jane Author",104 "authors": [{ "id": 99, "name": "Jane Author", "handle": "janeauthor" }],105 "bodyHtml": "<p>...</p>",106 "bodyText": "...",107 "wordcount": 1842,108 "reactionCount": 213,109 "commentCount": 42,110 "restacks": 18,111 "postTags": ["media", "business"]112}113```114
115### Publication record116
117```json118{119 "type": "publication",120 "id": 42,121 "name": "The Example",122 "subdomain": "example",123 "customDomain": null,124 "url": "https://example.substack.com",125 "description": "A newsletter about newsletters.",126 "categoryName": "Business",127 "totalSubscribers": 48211,128 "paidSubscribers": 1203,129 "createdAt": "2022-06-14T09:12:00Z"130}131```132
133### Author record134
135```json136{137 "type": "author",138 "id": 99,139 "name": "Jane Author",140 "handle": "janeauthor",141 "profileUrl": "https://substack.com/@janeauthor",142 "bio": "Writing about media.",143 "photoUrl": "https://.../photo.jpg",144 "publications": [{ "publicationName": "The Example", "subdomain": "example", "role": "admin" }],145 "subscriptions": [{ "publicationName": "Noahpinion", "subdomain": "noahpinion" }]146}147```148
149### Comment record150
151```json152{153 "type": "comment",154 "id": 55512,155 "postId": 123456,156 "postSlug": "why-newsletters-won",157 "postTitle": "Why newsletters won",158 "publication": "example",159 "parentId": null,160 "depth": 0,161 "body": "Great piece.",162 "authorName": "A Reader",163 "authorHandle": "areader",164 "publishedAt": "2026-02-01T16:30:00Z",165 "reactionCount": 4166}167```168
169## How to scrape Substack (step-by-step)170
1711. **Click "Try for free"** at the top of this page — you'll be taken to the Apify console.1722. **Paste your target URLs** into the Start URLs field. Examples:173 - A publication: `https://stratechery.com` or `https://noahpinion.substack.com`174 - A single post: `https://example.substack.com/p/some-post`175 - An author profile: `https://substack.com/@handle`176 - A share link: `https://open.substack.com/pub/astralcodexten/p/some-post`1773. **Set `maxPostsPerPublication`** — start with `10` for a test, then bump it (or set `0` for the whole archive).1784. **Click "Start"**. When the run completes, open the **Output** tab to browse results or hit **Export** for CSV / JSON / Excel.179
180## FAQ181
182**How am I charged?** Per record in your results — each post, publication, author, and comment counts as one result. Failed or retried requests are never charged, and you'll never receive the same post twice. Control your bill with `maxPostsPerPublication`, `includeComments`, and `maxPublicationsToDiscover`; you can also set a maximum budget for any run in the Apify Console.183
184**Does it scrape paywalled posts?** Paid posts are listed with metadata and the free preview text; full paid bodies require a subscriber login, which this scraper does not use. Enable `onlyFreePosts` to skip them entirely.185
186**How many comments will a post produce?** Whatever the thread holds — popular posts can carry hundreds of comments, each delivered (and charged) as its own result. Leave `includeComments` off unless you need them.187
188**Will it get blocked?** No setup needed on your side — proxy rotation, retries, and rate-limit handling are built in.189
190**Can I schedule it?** Yes — use Apify Schedules for daily/weekly monitoring runs, and connect the dataset to Google Sheets, webhooks, or the API for delivery.191
192## Use with AI agents (MCP)193
194This Actor works as a tool for AI agents via the [Apify MCP server](https://mcp.apify.com). Add it to Claude, Cursor, or any MCP-compatible assistant with this server URL:195
196```text197https://mcp.apify.com?tools=doggo/substack-scraper-posts-comments-authors198```199
200Then just ask something like *"Scrape the latest 20 posts from noahpinion"* — the agent fills in the input, runs the Actor, and reads the results from the dataset. Standard pay-per-result pricing applies to the calling account.1// Quick offline check of URL classification — run with: node scripts/check-classify.mjs2import assert from 'node:assert';3import { classifyUrl, stripHtml, isSubstackInfraSubdomain } from '../src/utils.js';4
5const cases = [6 ['https://noahpinion.substack.com', { kind: 'publication', publication: 'noahpinion', origin: 'https://noahpinion.substack.com' }],7 ['https://www.astralcodexten.com', { kind: 'publication', publication: 'astralcodexten.com', origin: 'https://www.astralcodexten.com' }],8 ['https://astralcodexten.com', { kind: 'publication', publication: 'astralcodexten.com' }],9 ['https://example.substack.com/p/some-post', { kind: 'post', slug: 'some-post', publication: 'example' }],10 ['https://www.astralcodexten.com/p/my-ai-opinions?utm_source=x', { kind: 'post', slug: 'my-ai-opinions', publication: 'astralcodexten.com' }],11 ['https://substack.com/@mattyglesias', { kind: 'author', handle: 'mattyglesias' }],12 ['https://www.substack.com/@someone/notes', { kind: 'author', handle: 'someone' }],13 ['https://substack.com/profile/123-some-name', { kind: 'author-profile', path: '/profile/123-some-name' }],14 ['https://open.substack.com/pub/astralcodexten/p/my-ai-opinions?r=abc&utm_medium=ios', { kind: 'post', publication: 'astralcodexten', slug: 'my-ai-opinions', origin: 'https://astralcodexten.substack.com' }],15 ['https://open.substack.com/pub/noahpinion', { kind: 'publication', publication: 'noahpinion', origin: 'https://noahpinion.substack.com' }],16 ['https://substack.com/home/post/p-190662510', { kind: 'redirect', url: 'https://substack.com/home/post/p-190662510' }],17 ['https://substack.com/browse/technology', null],18 ['https://on.substack.com/p/announcement', null],19 ['https://support.substack.com', null],20 ['not a url', null],21];22
23for (const [url, expected] of cases) {24 const got = classifyUrl(url);25 if (expected === null) {26 assert.strictEqual(got, null, `${url} should be null, got ${JSON.stringify(got)}`);27 } else {28 assert.ok(got, `${url} should classify, got null`);29 for (const [k, v] of Object.entries(expected)) {30 assert.strictEqual(got[k], v, `${url}: expected ${k}=${v}, got ${got[k]} (${JSON.stringify(got)})`);31 }32 }33}34
35assert.strictEqual(stripHtml('<p>Hello &lt; world — fine</p><script>bad()</script>'), 'Hello < world — fine');36assert.strictEqual(stripHtml('<style>p{}</style><p>A<b>B</b></p>'), 'AB');37assert.strictEqual(isSubstackInfraSubdomain('on.substack.com'), true);38assert.strictEqual(isSubstackInfraSubdomain('noahpinion.substack.com'), false);39
40console.log(`All ${cases.length} classifyUrl cases + stripHtml checks passed.`);1import { Actor } from 'apify';2import { CheerioCrawler, log } from 'crawlee';3import { router, salvageFailedRequest } from './routes.js';4import { classifyUrl, buildInitialRequests, buildDiscoverySeeds } from './utils.js';5import { setConfig, initState, markSeenPub, isSeenPub, incrementDiscovered, getTotalItems } from './state.js';6
7await Actor.init();8
9const input = await Actor.getInput() ?? {};10const {11 startUrls = [],12 mode = 'posts',13 maxPostsPerPublication = 50,14 includeContent = true,15 includeComments = false,16 onlyFreePosts = false,17 searchQuery = '',18 discoveryMode = 'none',19 discoveryQuery = '',20 maxPublicationsToDiscover = 25,21 maxConcurrency = 5,22} = input;23
24const hasStart = Array.isArray(startUrls) && startUrls.length > 0;25const hasDiscovery = discoveryMode && discoveryMode !== 'none';26
27if (!hasStart && !hasDiscovery) {28 throw new Error('Provide either startUrls or a discoveryMode.');29}30
31const resolvedSearchQuery = (discoveryMode === 'search' && discoveryQuery) ? discoveryQuery : searchQuery;32// 0 means "no limit"; an empty field falls back to the default of 50.33const postLimit = maxPostsPerPublication > 0 ? maxPostsPerPublication : null;34// Set by the platform when the buyer caps paid results — items pushed beyond35// it are never paid for, so the crawl stops there.36const maxPaidItems = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS) || null;37
38// Must come after Actor.init(); restores limits/dedupe state after migrations.39await initState();40
41// Pay-per-event runs charge one event per record (event name == record type);42// per-result runs are charged by the platform per default-dataset item.43const pricingInfo = Actor.getChargingManager().getPricingInfo();44if (pricingInfo.isPayPerEvent) {45 const missing = ['post', 'publication', 'author', 'comment']46 .filter((e) => !(e in pricingInfo.perEventPrices));47 if (missing.length) {48 log.warning(`Pay-per-event pricing has no price configured for event(s): ${missing.join(', ')} — records of these types will NOT be charged.`);49 }50}51
52setConfig({53 maxPostsPerPublication: postLimit,54 includeContent,55 includeComments,56 onlyFreePosts,57 searchQuery: resolvedSearchQuery,58 mode: ['posts', 'publication', 'author'].includes(mode) ? mode : 'posts',59 maxPublicationsToDiscover,60 maxPaidItems,61 isPayPerEvent: pricingInfo.isPayPerEvent,62});63
64// Automatic datacenter proxies: Substack's JSON API is not challenge-gated,65// it only rate-limits per IP, and the session pool rotates IPs on 403/429.66// Local runs connect directly.67const proxyConfiguration = Actor.isAtHome()68 ? await Actor.createProxyConfiguration()69 : undefined;70
71const initialRequests = [];72for (const entry of startUrls) {73 const rawUrl = typeof entry === 'string' ? entry : entry?.url;74 if (!rawUrl) continue;75 const classified = classifyUrl(rawUrl);76 if (!classified) {77 log.warning(`Skipping unrecognised URL: ${rawUrl}`);78 continue;79 }80 // Start-URL publications are marked seen (so discovery won't re-scrape81 // them) but do NOT consume the discovery budget.82 if (classified.kind === 'publication') markSeenPub(classified.publication);83 initialRequests.push(...buildInitialRequests(classified, mode, { searchQuery: resolvedSearchQuery }));84}85
86if (hasDiscovery) {87 const seeds = buildDiscoverySeeds({ mode, searchQuery: resolvedSearchQuery });88 const freshSeeds = seeds.filter((seed) => !isSeenPub(seed.userData.publication));89 for (const seed of freshSeeds) {90 markSeenPub(seed.userData.publication);91 incrementDiscovered();92 }93 if (freshSeeds.length) {94 initialRequests.push(...freshSeeds);95 log.info(`Discovery enabled (${discoveryMode}): seeded ${freshSeeds.length} publications, expanding via recommendation graph up to ${maxPublicationsToDiscover} total`);96 if (discoveryMode === 'search' && resolvedSearchQuery) {97 log.info(`Search query "${resolvedSearchQuery}" will be applied to each discovered publication's archive.`);98 }99 }100}101
102// Generous safety ceiling so a logic bug can never run the queue unbounded.103// Unlimited-posts runs (postLimit null) are intentionally uncapped.104let maxRequestsPerCrawl;105if (postLimit != null) {106 const totalPubs = startUrls.length + (hasDiscovery ? maxPublicationsToDiscover : 0) + 1;107 const perPub = 2 // homepage + recommendations108 + Math.ceil(postLimit / 50) + 1 // archive pages + trailing empty page109 + postLimit * ((includeContent ? 1 : 0) + (includeComments ? 1 : 0));110 maxRequestsPerCrawl = Math.ceil(totalPubs * (perPub + 2) * 1.25) + 25;111}112
113const crawler = new CheerioCrawler({114 proxyConfiguration,115 requestHandler: router,116 maxConcurrency,117 maxRequestRetries: 3,118 maxRequestsPerCrawl,119 requestHandlerTimeoutSecs: 60,120 additionalMimeTypes: ['application/json', 'text/json'],121 // Default got-scraping header generation produces consistent browser122 // fingerprints — do not override user-agent/accept headers manually.123 useSessionPool: true,124 persistCookiesPerSession: true,125 sessionPoolOptions: {126 maxPoolSize: 100,127 },128 failedRequestHandler: async ({ request }) => {129 const label = request.userData?.label ?? 'none';130 log.warning(`Request failed after retries: ${request.url} (label=${label})`);131 if (label === 'PUBLICATION_HTML') {132 log.warning(`Is ${request.url} really a Substack publication?`);133 }134 await salvageFailedRequest(request);135 },136});137
138await Actor.setStatusMessage('Crawling Substack...').catch(() => {});139await crawler.run(initialRequests);140
141const total = getTotalItems();142await Actor.exit(`Finished — saved ${total} item(s) to the dataset.`);1import { Actor } from 'apify';2import { Dataset, createCheerioRouter, log } from 'crawlee';3import {4 buildArchiveRequest,5 buildInitialRequests,6 classifyUrl,7 extractPreload,8 findPublicationObject,9 isLikelyNonPublicationHost,10 stripHtml,11 wordCount,12} from './utils.js';13import {14 state,15 getCount,16 incrementCount,17 setPublicationInfo,18 getPublicationInfo,19 isSeenPub,20 markSeenPub,21 getDiscoveredCount,22 incrementDiscovered,23 markPubSaved,24 markPostEnqueued,25 markPostSavedRecord,26 reserveItemSlots,27 isCapReached,28 isTypeExhausted,29 markTypeExhausted,30 addItems,31 getTotalItems,32} from './state.js';33
34export const router = createCheerioRouter();35
36// All record types go to the default dataset with a `type` discriminator;37// named datasets are account-global (they accumulate across runs and never38// expire), so they are never used. Under pay-per-event pricing each record39// type charges its own event (event name == record type) and the SDK trims40// pushes to the buyer's budget; under per-result pricing the platform charges41// per default-dataset item and ACTOR_MAX_PAID_DATASET_ITEMS caps it.42let lastStatusCount = 0;43async function saveRecords(records) {44 if (!records.length) return;45
46 if (state.config.isPayPerEvent) {47 // The SDK trims each push to the buyer's remaining budget and reports48 // chargedCount — count only what was actually pushed, and track budget49 // exhaustion per event type (e.g. comments can run out while posts50 // remain chargeable).51 const byType = new Map();52 for (const row of records) {53 if (isTypeExhausted(row.type)) continue;54 if (!byType.has(row.type)) byType.set(row.type, []);55 byType.get(row.type).push(row);56 }57 for (const [eventName, items] of byType) {58 const result = await Actor.pushData(items, eventName);59 // When the budget limit trims a push, only chargedCount items were60 // written; otherwise (incl. free/unpriced events) the whole batch was.61 const delivered = result?.eventChargeLimitReached62 ? (result?.chargedCount ?? 0)63 : items.length;64 if (delivered > 0) addItems(delivered);65 if (result?.eventChargeLimitReached) {66 markTypeExhausted(eventName);67 log.warning(`Budget for '${eventName}' results exhausted — no further ${eventName} records will be scraped.`);68 }69 }70 } else {71 const allowed = reserveItemSlots(records.length);72 if (allowed < records.length) {73 log.warning(`Maximum paid dataset items reached — dropped ${records.length - allowed} record(s).`);74 }75 const rows = records.slice(0, allowed);76 if (!rows.length) return;77 await Dataset.pushData(rows);78 }79
80 const total = getTotalItems();81 if (total - lastStatusCount >= 50) {82 lastStatusCount = total;83 await Actor.setStatusMessage(`Saved ${total} items so far...`).catch(() => {});84 }85}86
87// The crawl stops only when its PRIMARY deliverable can no longer be charged88// (posts in posts mode, publications otherwise) or the per-result item cap is89// hit — exhausting a secondary type (comments) just stops that record type.90function crawlBudgetExhausted() {91 const primaryType = state.config.mode === 'posts' ? 'post' : 'publication';92 return isCapReached() || isTypeExhausted(primaryType);93}94
95let stopRequested = false;96async function stopIfCapReached(crawler) {97 if (!crawlBudgetExhausted() || stopRequested) return;98 stopRequested = true;99 log.warning('Paid results budget reached — stopping the crawl.');100 if (typeof crawler.stop === 'function') crawler.stop();101}102
103function parseJsonBody(body) {104 if (!body) return null;105 if (typeof body === 'object' && !Buffer.isBuffer(body)) return body;106 try {107 return JSON.parse(body.toString());108 } catch {109 return null;110 }111}112
113function commentsRequest(origin, publication, post) {114 return {115 url: `${origin}/api/v1/post/${post.id}/comments?token=&all_comments=true&sort=best_first`,116 userData: {117 label: 'COMMENTS',118 origin,119 publication,120 postId: post.id,121 postSlug: post.slug,122 postTitle: post.title,123 },124 };125}126
127// Substack data carries stray whitespace in titles/names — trim for clean exports.128function clean(s) {129 return typeof s === 'string' ? s.trim() : s ?? null;130}131
132// "450,000" → 450000; passes numbers through; null for anything else.133function parseCount(value) {134 if (typeof value === 'number') return value;135 if (typeof value === 'string') {136 const n = Number(value.replace(/[,\s]/g, ''));137 return Number.isFinite(n) ? n : null;138 }139 return null;140}141
142function shapePost(raw, ctx = {}) {143 const canonical = raw.canonical_url || (raw.slug && ctx.origin ? `${ctx.origin}/p/${raw.slug}` : null);144 const bodyHtml = raw.body_html ?? null;145 const bodyText = bodyHtml ? stripHtml(bodyHtml) : (raw.truncated_body_text ?? raw.description ?? '');146 const pubKey = ctx.publication ?? raw.publication?.subdomain ?? null;147 const cachedPub = pubKey ? getPublicationInfo(pubKey) : {};148 const publicationName = clean(raw.publication?.name ?? raw.publicationName ?? cachedPub.name ?? null);149 return {150 type: 'post',151 id: raw.id,152 title: clean(raw.title),153 subtitle: clean(raw.subtitle),154 slug: raw.slug,155 url: canonical,156 publication: pubKey,157 publicationName,158 publishedAt: raw.post_date ?? raw.published_at ?? null,159 audience: raw.audience ?? null,160 postType: raw.type ?? null,161 coverImage: raw.cover_image ?? null,162 description: raw.description ?? null,163 truncatedBodyText: raw.truncated_body_text ?? null,164 bodyHtml,165 bodyText: bodyText || null,166 wordcount: wordCount(bodyText),167 reactionCount: raw.reaction_count ?? raw.reactions?.heart ?? null,168 commentCount: raw.comment_count ?? null,169 restacks: raw.restacks ?? null,170 isPaid: raw.audience ? raw.audience !== 'everyone' : null,171 author: clean(raw.publishedBylines?.[0]?.name172 ?? raw.publishedBylines?.[0]?.handle173 ?? raw.author?.name174 ?? null),175 authors: (raw.publishedBylines ?? []).map((b) => ({176 id: b.id,177 name: b.name,178 handle: b.handle,179 photoUrl: b.photo_url ?? null,180 bio: b.bio ?? null,181 })),182 postTags: (raw.postTags ?? []).map((t) => t.name ?? t),183 scrapedAt: new Date().toISOString(),184 };185}186
187function shapePublication(raw, ctx = {}) {188 const pub = raw.publication ?? raw;189 return {190 type: 'publication',191 id: pub.id,192 name: clean(pub.name),193 subdomain: pub.subdomain ?? ctx.publication ?? null,194 customDomain: pub.custom_domain ?? null,195 url: pub.custom_domain ? `https://${pub.custom_domain}` : (pub.subdomain ? `https://${pub.subdomain}.substack.com` : ctx.origin),196 heroText: pub.hero_text ?? null,197 description: pub.description ?? pub.about ?? null,198 logoUrl: pub.logo_url ?? null,199 copyright: pub.copyright ?? null,200 language: pub.language ?? null,201 authorId: pub.author_id ?? null,202 foundingPlanName: pub.founding_plan_name ?? null,203 paidSubscribers: pub.paid_subscribers_visible ? pub.paid_subscribers : null,204 totalSubscribers: pub.subscriber_count_number ?? pub.subscriber_count ?? null,205 subscriberCountString: pub.subscriber_count_string ?? null,206 categoryName: pub.base_category_name ?? null,207 createdAt: pub.created_at ?? null,208 scrapedAt: new Date().toISOString(),209 };210}211
212router.addHandler('PUBLICATION_HTML', async ({ request, $, body, crawler }) => {213 const ctx = request.userData;214 if (typeof $ !== 'function') {215 throw new Error(`Non-HTML response on publication page ${request.url} — retrying`);216 }217 const html = body?.toString() ?? '';218
219 // Apex domains (astralcodexten.com) redirect pages to the canonical host220 // (www.astralcodexten.com) but 404 API paths — follow-up API requests must221 // use the origin the page actually loaded from.222 let effectiveOrigin = ctx.origin;223 let effectiveHost = null;224 try {225 const loaded = new URL(request.loadedUrl ?? request.url);226 effectiveOrigin = loaded.origin;227 effectiveHost = loaded.hostname.toLowerCase();228 } catch { /* keep ctx.origin */ }229
230 // Every real Substack page embeds window._preloads; its absence means a231 // block page / interstitial (throw → retry on a new session) or, for232 // discovered links, a site that isn't Substack at all.233 const preload = extractPreload(html);234 const pubObj = findPublicationObject(preload);235
236 if (!preload) {237 if (ctx.fromDiscovery) {238 log.info(`Skipping ${request.url} — no Substack preload, likely not a Substack publication`);239 return;240 }241 throw new Error(`No Substack preload data on ${request.url} — blocked or not a Substack publication`);242 }243
244 markSeenPub(ctx.publication, pubObj?.subdomain, pubObj?.custom_domain, effectiveHost);245
246 let shaped;247 if (pubObj) {248 shaped = shapePublication({ publication: pubObj }, ctx);249 } else {250 if (ctx.fromDiscovery) {251 log.info(`Skipping ${request.url} — no publication object in preload`);252 return;253 }254 const get = (sel, attr = 'content') => {255 const v = $(sel).attr(attr);256 return v ? v.trim() : null;257 };258 const cleanPubName = (s) => {259 if (!s) return null;260 const parts = s261 .split(/\s*[|\-–—]\s*/)262 .map((x) => x.trim())263 .filter(Boolean)264 .filter((x) => x.toLowerCase() !== 'substack');265 return parts[0] ?? null;266 };267 const metaName = cleanPubName(get('meta[property="og:title"]'))268 || cleanPubName(get('meta[name="twitter:title"]'))269 || cleanPubName(($('title').text() ?? '').trim())270 || cleanPubName(get('meta[property="og:site_name"]'))271 || null;272 shaped = {273 type: 'publication',274 id: null,275 name: metaName,276 subdomain: ctx.publication ?? null,277 customDomain: ctx.origin?.replace(/^https?:\/\//, '') ?? null,278 url: ctx.origin,279 description: get('meta[property="og:description"]') || get('meta[name="description"]') || null,280 heroText: null,281 logoUrl: get('meta[property="og:image"]') || null,282 copyright: null,283 language: null,284 authorId: null,285 foundingPlanName: null,286 paidSubscribers: null,287 totalSubscribers: null,288 subscriberCountString: null,289 categoryName: null,290 createdAt: null,291 scrapedAt: new Date().toISOString(),292 };293 }294
295 if (ctx.publication && shaped.name) {296 setPublicationInfo(ctx.publication, {297 name: shaped.name,298 description: shaped.description,299 logoUrl: shaped.logoUrl,300 });301 }302
303 if (pubObj) {304 // Subscriber counts only appear on the /about page preload — enrich305 // there before saving. The request queue dedupes repeat visits.306 await crawler.addRequests([{307 url: `${effectiveOrigin}/about`,308 userData: {309 label: 'PUBLICATION_ABOUT',310 origin: effectiveOrigin,311 publication: ctx.publication,312 shaped,313 },314 }]);315 } else if (markPubSaved(shaped.subdomain ?? ctx.publication)) {316 await saveRecords([shaped]);317 log.info(`Saved publication: ${shaped.name || ctx.publication} (from meta tags)`);318 }319
320 if (crawlBudgetExhausted()) {321 await stopIfCapReached(crawler);322 return;323 }324
325 if (ctx.followWithArchive) {326 await crawler.addRequests([buildArchiveRequest({327 origin: effectiveOrigin,328 publication: ctx.publication,329 searchQuery: ctx.searchQuery,330 })]);331 }332
333 if (ctx.harvestRecommendations) {334 const limit = state.config.maxPublicationsToDiscover ?? 25;335 if (getDiscoveredCount() < limit) {336 await crawler.addRequests([{337 url: `${effectiveOrigin}/recommendations`,338 userData: {339 label: 'RECOMMENDATIONS_HTML',340 origin: effectiveOrigin,341 publication: ctx.publication,342 },343 }]);344 }345 }346});347
348// The /about page preload carries subscriber stats the homepage lacks349// (subscriberCountDetails text + pub.freeSubscriberCount like "450,000").350router.addHandler('PUBLICATION_ABOUT', async ({ request, body, crawler }) => {351 const { publication, shaped } = request.userData;352 const preload = extractPreload(body?.toString() ?? '');353 if (!preload) {354 throw new Error(`No Substack preload data on ${request.url} — retrying`);355 }356
357 const aboutPub = findPublicationObject(preload) ?? {};358 const enriched = {359 ...shaped,360 description: shaped.description ?? clean(aboutPub.hero_text) ?? null,361 heroText: shaped.heroText ?? clean(aboutPub.hero_text) ?? null,362 totalSubscribers: shaped.totalSubscribers363 ?? parseCount(aboutPub.freeSubscriberCount ?? preload.pub?.freeSubscriberCount),364 subscriberCountString: shaped.subscriberCountString365 ?? clean(preload.subscriberCountDetails)366 ?? null,367 categoryName: shaped.categoryName ?? clean(aboutPub.base_category_name) ?? null,368 };369
370 if (markPubSaved(enriched.subdomain ?? publication)) {371 await saveRecords([enriched]);372 log.info(`Saved publication: ${enriched.name || publication}${enriched.totalSubscribers ? ` (${enriched.totalSubscribers} subscribers)` : ''}`);373 }374 await stopIfCapReached(crawler);375});376
377router.addHandler('ARCHIVE', async ({ request, json, body, crawler }) => {378 const config = state.config;379 const payload = json ?? parseJsonBody(body);380 if (!Array.isArray(payload)) {381 // Soft blocks come back 200 with an HTML body; retry on a fresh session382 // instead of silently dropping a whole page of posts.383 throw new Error(`Archive endpoint returned non-JSON for ${request.url} — retrying`);384 }385 if (payload.length === 0) return; // true end of archive386
387 if (crawlBudgetExhausted()) {388 await stopIfCapReached(crawler);389 return;390 }391
392 const { origin, publication, offset, searchQuery } = request.userData;393 const countKey = `posts:${publication}`;394 const limit = config.maxPostsPerPublication;395
396 const toEnqueue = [];397 const toStoreInline = [];398 let limitReached = false;399
400 for (const raw of payload) {401 if (config.onlyFreePosts && raw.audience && raw.audience !== 'everyone') continue;402 if (limit != null && getCount(countKey) >= limit) {403 limitReached = true;404 break;405 }406 // Idempotent across page retries/re-runs after a migration.407 if (!markPostEnqueued(publication, raw.id)) continue;408 incrementCount(countKey);409
410 if (config.includeContent) {411 toEnqueue.push({412 url: `${origin}/api/v1/posts/${raw.slug}`,413 userData: {414 label: 'POST_JSON',415 origin,416 publication,417 slug: raw.slug,418 archiveFallback: raw,419 },420 });421 } else if (markPostSavedRecord(publication, raw.id)) {422 toStoreInline.push(shapePost(raw, { origin, publication }));423 }424
425 if (config.includeComments && raw.comment_count !== 0 && !isTypeExhausted('comment')) {426 toEnqueue.push(commentsRequest(origin, publication, raw));427 }428 }429
430 if (toStoreInline.length) await saveRecords(toStoreInline);431 if (toEnqueue.length) await crawler.addRequests(toEnqueue);432
433 // The API can return short pages mid-archive (offset=0 returns ~23 items),434 // so only an empty page means the end. Advance by the actual page length.435 const hasMore = !limitReached436 && (limit == null || getCount(countKey) < limit)437 && !isCapReached();438 if (hasMore) {439 const nextOffset = offset + payload.length;440 await crawler.addRequests([buildArchiveRequest({ origin, publication, offset: nextOffset, searchQuery })]);441 }442});443
444router.addHandler('POST_JSON', async ({ request, json, body, crawler }) => {445 const payload = json ?? parseJsonBody(body);446 if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {447 // Throw so the request retries; if all retries fail, the448 // failedRequestHandler salvages archive metadata where available.449 throw new Error(`Post endpoint returned non-JSON for ${request.url} — retrying`);450 }451 const { origin, publication } = request.userData;452
453 // 404s come back as JSON like {"error":"Post not found","type":"single"} —454 // never push those as records. Salvage the archive listing if we have it.455 if (payload.error || payload.id == null) {456 await salvageFailedRequest(request);457 log.warning(`Not a post (${payload.error ?? 'no id'}): ${request.url} — skipped`);458 return;459 }460
461 if (markPostSavedRecord(publication, payload.id)) {462 await saveRecords([shapePost(payload, { origin, publication })]);463 }464
465 // Direct post start URLs skip the ARCHIVE handler, so enqueue comments466 // here too — the request queue deduplicates by URL.467 if (state.config.includeComments && payload.id && payload.comment_count !== 0 && !isTypeExhausted('comment')) {468 await crawler.addRequests([commentsRequest(origin, publication, payload)]);469 }470
471 await stopIfCapReached(crawler);472});473
474router.addHandler('COMMENTS', async ({ request, json, body, crawler }) => {475 if (isTypeExhausted('comment')) return; // budget for comments already spent476 const payload = json ?? parseJsonBody(body);477 if (!payload || typeof payload !== 'object') {478 throw new Error(`Comments endpoint returned non-JSON for ${request.url} — retrying`);479 }480 const comments = payload.comments ?? payload.children ?? [];481 if (!comments.length) return;482
483 const flat = [];484 const walk = (nodes, depth = 0, parentId = null) => {485 for (const node of nodes) {486 flat.push({487 type: 'comment',488 postId: request.userData.postId,489 postSlug: request.userData.postSlug,490 postTitle: request.userData.postTitle,491 publication: request.userData.publication,492 id: node.id,493 parentId,494 depth,495 body: node.body,496 authorName: node.name,497 authorHandle: node.handle,498 authorId: node.user_id,499 publishedAt: node.date,500 reactionCount: node.reactions?.heart ?? node.reaction_count ?? 0,501 scrapedAt: new Date().toISOString(),502 });503 if (node.children && node.children.length) walk(node.children, depth + 1, node.id);504 }505 };506 walk(comments);507 if (flat.length) await saveRecords(flat);508 await stopIfCapReached(crawler);509});510
511router.addHandler('AUTHOR_JSON', async ({ request, json, body, crawler }) => {512 const payload = json ?? parseJsonBody(body);513 if (!payload || typeof payload !== 'object') {514 throw new Error(`Profile endpoint returned non-JSON for ${request.url} — retrying`);515 }516 if (payload.error || payload.id == null) {517 log.warning(`Not an author profile (${payload.error ?? 'no id'}): ${request.url} — skipped`);518 return;519 }520 const handle = payload.handle ?? request.userData.handle ?? null;521 await saveRecords([{522 type: 'author',523 id: payload.id,524 name: payload.name,525 handle,526 bio: payload.bio ?? null,527 photoUrl: payload.photo_url ?? null,528 profileUrl: handle ? `https://substack.com/@${handle}` : null,529 subscriptions: (payload.subscriptions ?? []).map((s) => ({530 publicationId: s.publication?.id,531 publicationName: s.publication?.name,532 subdomain: s.publication?.subdomain,533 customDomain: s.publication?.custom_domain,534 })),535 publications: (payload.publicationUsers ?? []).map((p) => ({536 publicationId: p.publication?.id,537 publicationName: p.publication?.name,538 subdomain: p.publication?.subdomain,539 role: p.role,540 })),541 scrapedAt: new Date().toISOString(),542 }]);543 await stopIfCapReached(crawler);544});545
546// Numeric profile pages (substack.com/profile/123-name): pull the handle out547// of the page preload, then hit the public_profile API like @handle URLs.548router.addHandler('AUTHOR_HTML', async ({ request, $, body, crawler }) => {549 const html = body?.toString() ?? '';550 const preload = extractPreload(html);551 let handle = preload?.profile?.handle ?? preload?.user?.handle ?? null;552 if (!handle && typeof $ === 'function') {553 const ogUrl = $('meta[property="og:url"]').attr('content') ?? '';554 handle = ogUrl.match(/substack\.com\/@([\w.-]+)/)?.[1] ?? null;555 }556 if (!handle) {557 throw new Error(`Could not resolve author handle from ${request.url}`);558 }559 await crawler.addRequests([{560 url: `https://substack.com/api/v1/user/${handle}/public_profile`,561 userData: { label: 'AUTHOR_JSON', origin: 'https://substack.com', handle },562 }]);563});564
565// Reader links that 302 to a canonical URL: classify wherever we landed.566router.addHandler('RESOLVE', async ({ request, crawler }) => {567 const finalUrl = request.loadedUrl ?? request.url;568 const classified = classifyUrl(finalUrl);569 if (!classified || classified.kind === 'redirect') {570 log.warning(`Could not resolve ${request.url} (landed on ${finalUrl})`);571 return;572 }573 if (classified.kind === 'publication') markSeenPub(classified.publication);574 await crawler.addRequests(buildInitialRequests(classified, state.config.mode, {575 searchQuery: state.config.searchQuery,576 }));577});578
579router.addHandler('RECOMMENDATIONS_HTML', async ({ request, $, crawler }) => {580 const config = state.config;581 const limit = config.maxPublicationsToDiscover ?? 25;582 if (typeof $ !== 'function') {583 throw new Error(`Non-HTML response on ${request.url} — retrying`);584 }585 const sourceOrigin = request.userData.origin;586 const sourcePub = request.userData.publication;587 const found = new Map();588
589 $('a[href]').each((_i, el) => {590 if (getDiscoveredCount() + found.size >= limit) return false;591 const href = $(el).attr('href');592 if (!href) return;593 let abs;594 try { abs = new URL(href, request.url); } catch { return; }595 if (abs.origin === sourceOrigin) return;596 if (isLikelyNonPublicationHost(abs.hostname)) return;597 // classifyUrl handles *.substack.com, custom domains, and598 // open.substack.com/pub/... share links; infra subdomains return null.599 const classified = classifyUrl(abs.href);600 if (!classified || classified.kind !== 'publication') return;601 const key = classified.publication;602 if (!key || isSeenPub(key) || found.has(key)) return;603 found.set(key, classified);604 });605
606 if (!found.size) {607 log.info(`No new publications on ${sourcePub}/recommendations (discovered: ${getDiscoveredCount()}/${limit})`);608 return;609 }610
611 const toEnqueue = [];612 for (const [key, classified] of found) {613 // Budget is committed at enqueue time so concurrent harvests can't614 // overshoot; seenPubs makes this idempotent across retries.615 markSeenPub(key);616 incrementDiscovered();617 toEnqueue.push({618 url: classified.origin,619 userData: {620 label: 'PUBLICATION_HTML',621 origin: classified.origin,622 publication: classified.publication,623 followWithArchive: config.mode === 'posts',624 harvestRecommendations: getDiscoveredCount() < limit,625 fromDiscovery: true,626 searchQuery: config.searchQuery,627 },628 });629 }630 await crawler.addRequests(toEnqueue);631 log.info(`Discovery: harvested ${toEnqueue.length} publications from ${sourcePub} (total: ${getDiscoveredCount()}/${limit})`);632});633
634router.addDefaultHandler(async ({ request }) => {635 log.warning(`No handler for ${request.url} (label=${request.label ?? 'none'})`);636});637
638// Called from failedRequestHandler: when a POST_JSON request exhausts its639// retries, the archive listing entry still has title/date/description —640// save that instead of losing the post entirely.641export async function salvageFailedRequest(request) {642 const { label, archiveFallback, origin, publication, shaped } = request.userData ?? {};643 if (label === 'POST_JSON' && archiveFallback) {644 if (!markPostSavedRecord(publication, archiveFallback.id)) return;645 const salvaged = shapePost(archiveFallback, { origin, publication });646 salvaged.note = 'full content fetch failed; archive metadata only';647 await saveRecords([salvaged]);648 log.info(`Salvaged archive metadata for failed post ${request.url}`);649 return;650 }651 // /about enrichment failed — save the homepage-derived publication record.652 if (label === 'PUBLICATION_ABOUT' && shaped) {653 if (!markPubSaved(shaped.subdomain ?? publication)) return;654 await saveRecords([shaped]);655 log.info(`Saved publication ${shaped.name || publication} without /about enrichment`);656 }657}1import { Actor } from 'apify';2
3export const state = {4 config: {5 maxPostsPerPublication: 50,6 includeContent: true,7 includeComments: false,8 onlyFreePosts: false,9 searchQuery: '',10 mode: 'posts',11 maxPublicationsToDiscover: 25,12 maxPaidItems: null,13 isPayPerEvent: false,14 },15 data: null,16};17
18// Crawl progress lives in Actor.useState so it survives migrations/restarts.19// The request queue is persisted by the platform, so any counter that gates20// what gets enqueued or pushed must be persisted too — otherwise a migration21// resets the limits and the run over-delivers (and over-charges the buyer).22export async function initState() {23 state.data = await Actor.useState('CRAWL_STATE', {24 counts: {},25 publicationInfo: {},26 seenPubs: {},27 savedPubs: {},28 enqueuedPostIds: {},29 savedPostIds: {},30 discoveredCount: 0,31 totalItems: 0,32 capReached: false,33 exhaustedTypes: {},34 });35 return state.data;36}37
38export function setConfig(partial) {39 state.config = { ...state.config, ...partial };40}41
42export function getCount(key) {43 return state.data.counts[key] ?? 0;44}45
46export function incrementCount(key, by = 1) {47 state.data.counts[key] = (state.data.counts[key] ?? 0) + by;48 return state.data.counts[key];49}50
51export function setPublicationInfo(key, info) {52 if (!key) return;53 state.data.publicationInfo[key] = { ...(state.data.publicationInfo[key] ?? {}), ...info };54}55
56export function getPublicationInfo(key) {57 return state.data.publicationInfo[key] ?? {};58}59
60// seenPubs deduplicates publications across start URLs, discovery seeds, and61// recommendation harvesting (including subdomain/custom-domain aliases).62// discoveredCount tracks only discovery-found publications against the budget,63// so start URLs don't silently consume maxPublicationsToDiscover.64export function isSeenPub(key) {65 return Boolean(key && state.data.seenPubs[key.toLowerCase?.() ?? key]);66}67
68export function markSeenPub(...keys) {69 for (const key of keys) {70 if (key) state.data.seenPubs[key.toLowerCase?.() ?? key] = true;71 }72}73
74export function getDiscoveredCount() {75 return state.data.discoveredCount;76}77
78export function incrementDiscovered(by = 1) {79 state.data.discoveredCount += by;80 return state.data.discoveredCount;81}82
83export function markPubSaved(key) {84 if (!key) return true;85 const k = key.toLowerCase?.() ?? key;86 if (state.data.savedPubs[k]) return false;87 state.data.savedPubs[k] = true;88 return true;89}90
91// Enqueue-side guard: makes per-publication post counting idempotent when an92// archive page is retried or re-run after a migration.93export function markPostEnqueued(pub, id) {94 if (id == null) return true;95 const key = `${pub ?? ''}:${id}`;96 if (state.data.enqueuedPostIds[key]) return false;97 state.data.enqueuedPostIds[key] = true;98 return true;99}100
101// Push-side guard: a post id is billed at most once per publication.102export function markPostSavedRecord(pub, id) {103 if (id == null) return true;104 const key = `${pub ?? ''}:${id}`;105 if (state.data.savedPostIds[key]) return false;106 state.data.savedPostIds[key] = true;107 return true;108}109
110// Reserve dataset slots under the buyer's max-paid-items cap111// (ACTOR_MAX_PAID_DATASET_ITEMS). Items pushed beyond the cap are never paid112// for, so scraping past it only burns the owner's proxy/compute budget.113export function reserveItemSlots(n) {114 const cap = state.config.maxPaidItems;115 if (cap == null) {116 state.data.totalItems += n;117 return n;118 }119 const allowed = Math.max(0, Math.min(n, cap - state.data.totalItems));120 state.data.totalItems += allowed;121 if (allowed < n) state.data.capReached = true;122 return allowed;123}124
125export function isCapReached() {126 return state.data.capReached;127}128
129// Under pay-per-event, the SDK reports budget exhaustion PER EVENT TYPE.130// Track it per type so e.g. running out of comment budget stops comment131// scraping but lets posts (still chargeable) continue.132export function markTypeExhausted(type) {133 state.data.exhaustedTypes[type] = true;134}135
136export function isTypeExhausted(type) {137 return Boolean(state.data.exhaustedTypes[type]);138}139
140// Count of records actually pushed (and charged, under pay-per-event).141export function addItems(n) {142 state.data.totalItems += n;143 return state.data.totalItems;144}145
146export function getTotalItems() {147 return state.data.totalItems;148}1import * as cheerio from 'cheerio';2
3// Substack's archive API rejects limit > 50. Note the API may return SHORT4// pages that are not the end of the archive (offset=0 reliably returns ~235// items), so pagination must continue until an empty page — never treat a6// short page as exhaustion.7const ARCHIVE_PAGE_LIMIT = 50;8
9export function normaliseUrl(input) {10 try {11 return new URL(input);12 } catch {13 return null;14 }15}16
17// Canonical publication key for a host: bare subdomain for *.substack.com,18// custom domain with any leading "www." stripped (www.astralcodexten.com and19// astralcodexten.com are the same publication and must dedupe together).20export function publicationFromHost(host) {21 const h = host.toLowerCase();22 if (h.endsWith('.substack.com')) return h.replace(/\.substack\.com$/, '');23 return h.replace(/^www\./, '');24}25
26export function classifyUrl(rawUrl) {27 const url = normaliseUrl(rawUrl);28 if (!url) return null;29
30 const host = url.hostname.toLowerCase();31 const segments = url.pathname.split('/').filter(Boolean);32
33 // Reader share links: https://open.substack.com/pub/{publication}[/p/{slug}]34 if (host === 'open.substack.com') {35 if (segments[0] === 'pub' && segments[1]) {36 const publication = segments[1].toLowerCase();37 const origin = `https://${publication}.substack.com`;38 if (segments[2] === 'p' && segments[3]) {39 return { kind: 'post', origin, publication, slug: segments[3] };40 }41 return { kind: 'publication', origin, publication };42 }43 return null;44 }45
46 if (host === 'substack.com' || host === 'www.substack.com') {47 // Author profile: https://substack.com/@handle48 if (url.pathname.startsWith('/@')) {49 const handle = url.pathname.slice(2).split('/')[0];50 return { kind: 'author', handle, origin: 'https://substack.com' };51 }52 // Numeric profile: https://substack.com/profile/123-name53 if (url.pathname.startsWith('/profile/')) {54 return { kind: 'author-profile', origin: 'https://substack.com', path: url.pathname };55 }56 // Reader post links: https://substack.com/home/post/p-{id} — these 30257 // to the canonical post URL, so fetch and classify the landing URL.58 if (/^\/home\/post\/p-\d+/.test(url.pathname)) {59 return { kind: 'redirect', url: `https://substack.com${url.pathname}` };60 }61 return null;62 }63
64 // Infra subdomains (on.substack.com, support.substack.com, ...) are not publications.65 if (isSubstackInfraSubdomain(host)) return null;66
67 // Custom-domain or subdomain publications.68 const publication = publicationFromHost(host);69 const origin = `${url.protocol}//${host}`;70
71 if (url.pathname.startsWith('/p/')) {72 const slug = url.pathname.split('/')[2];73 return { kind: 'post', origin, publication, slug };74 }75
76 return { kind: 'publication', origin, publication };77}78
79export function buildInitialRequests(classified, mode, opts = {}) {80 const { searchQuery = '' } = opts;81
82 if (classified.kind === 'redirect') {83 return [{84 url: classified.url,85 userData: { label: 'RESOLVE' },86 }];87 }88
89 if (classified.kind === 'post') {90 return [{91 url: `${classified.origin}/api/v1/posts/${classified.slug}`,92 userData: {93 label: 'POST_JSON',94 origin: classified.origin,95 publication: classified.publication,96 slug: classified.slug,97 },98 }];99 }100
101 if (classified.kind === 'author' || classified.kind === 'author-profile') {102 const profileUrl = classified.kind === 'author'103 ? `${classified.origin}/api/v1/user/${classified.handle}/public_profile`104 : `${classified.origin}${classified.path}`;105 return [{106 url: profileUrl,107 userData: {108 label: classified.kind === 'author' ? 'AUTHOR_JSON' : 'AUTHOR_HTML',109 origin: classified.origin,110 handle: classified.handle,111 },112 }];113 }114
115 if (classified.kind === 'publication') {116 return [{117 url: classified.origin,118 userData: {119 label: 'PUBLICATION_HTML',120 origin: classified.origin,121 publication: classified.publication,122 followWithArchive: mode === 'posts',123 searchQuery,124 },125 }];126 }127
128 return [];129}130
131// Substack embeds page data as: window._preloads = JSON.parse("<escaped JSON>").132// Scan to the closing quote of the string literal, honouring backslash escapes.133export function extractPreload(html) {134 if (!html) return null;135
136 const re = /window\._preloads\s*=\s*JSON\.parse\(\s*(["'])/g;137 const match = re.exec(html);138 if (!match) return null;139
140 const quote = match[1];141 let i = re.lastIndex;142 while (i < html.length) {143 if (html[i] === '\\' && i + 1 < html.length) { i += 2; continue; }144 if (html[i] === quote) break;145 i++;146 }147 if (i >= html.length) return null;148
149 try {150 const jsonStr = JSON.parse(`${quote}${html.slice(re.lastIndex, i)}${quote}`);151 return JSON.parse(jsonStr);152 } catch {153 return null;154 }155}156
157export function findPublicationObject(preload) {158 if (!preload || typeof preload !== 'object') return null;159 const candidates = [160 preload.publication,161 preload.pub,162 preload.pageData?.publication,163 preload.props?.pageProps?.publication,164 preload.props?.pageProps?.pub,165 ];166 for (const c of candidates) {167 if (c && typeof c === 'object' && (c.name || c.subdomain || c.id)) return c;168 }169 return null;170}171
172const DISCOVERY_SEEDS = [173 'https://www.noahpinion.blog',174 'https://www.slowboring.com',175 'https://www.astralcodexten.com',176 'https://www.lennysnewsletter.com',177 'https://www.thefp.com',178];179
180const RESERVED_SUBSTACK_SUBDOMAINS = new Set([181 'www', 'on', 'pages', 'reader', 'support', 'open', 'explore',182 'discover', 'blog', 'about', 'help', 'shop', 'api', 'cdn',183 'assets', 'static', 'images', 'img', 'login', 'signup',184 'terms', 'privacy', 'press', 'careers', 'tos', 'email',185 'mail', 'media', 'status', 'jobs',186]);187
188const NON_PUBLICATION_HOSTS = new Set([189 'twitter.com', 'x.com', 'facebook.com', 'instagram.com',190 'linkedin.com', 'youtube.com', 'youtu.be', 'github.com',191 'google.com', 'apple.com', 'tiktok.com', 'mastodon.social',192 'bsky.app', 'threads.net', 'medium.com', 'paypal.com',193 'amazon.com', 'patreon.com', 'ko-fi.com', 'wikipedia.org',194 'reddit.com', 'discord.com', 'discord.gg', 'spotify.com',195 'soundcloud.com', 'vimeo.com', 't.me', 'telegram.me',196]);197
198export function isSubstackInfraSubdomain(host) {199 if (!host) return true;200 const h = host.toLowerCase();201 if (h === 'substack.com') return true;202 if (!h.endsWith('.substack.com')) return false;203 const sub = h.slice(0, -'.substack.com'.length);204 return RESERVED_SUBSTACK_SUBDOMAINS.has(sub);205}206
207export function isLikelyNonPublicationHost(host) {208 if (!host) return true;209 const h = host.toLowerCase();210 if (NON_PUBLICATION_HOSTS.has(h)) return true;211 const parts = h.split('.');212 if (parts.length >= 2) {213 const base = parts.slice(-2).join('.');214 if (NON_PUBLICATION_HOSTS.has(base)) return true;215 }216 return false;217}218
219export function buildDiscoverySeeds({ mode = 'posts', searchQuery = '' } = {}) {220 return DISCOVERY_SEEDS.map((rawUrl) => {221 const parsed = new URL(rawUrl);222 const origin = parsed.origin;223 const publication = publicationFromHost(parsed.hostname);224 return {225 url: origin,226 userData: {227 label: 'PUBLICATION_HTML',228 origin,229 publication,230 followWithArchive: mode === 'posts',231 harvestRecommendations: true,232 fromDiscovery: true,233 searchQuery,234 },235 };236 });237}238
239export function buildArchiveRequest({ origin, publication, offset = 0, searchQuery = '' }) {240 const params = new URLSearchParams({241 sort: 'new',242 offset: String(offset),243 limit: String(ARCHIVE_PAGE_LIMIT),244 });245 if (searchQuery) params.set('search', searchQuery);246 return {247 url: `${origin}/api/v1/archive?${params.toString()}`,248 userData: {249 label: 'ARCHIVE',250 origin,251 publication,252 offset,253 searchQuery,254 },255 };256}257
258export { ARCHIVE_PAGE_LIMIT };259
260// cheerio handles script/style removal and full entity decoding (named and261// numeric), which the previous regex chain got wrong (&lt; double-decoded).262export function stripHtml(html) {263 if (!html) return '';264 try {265 const $ = cheerio.load(html);266 $('script, style').remove();267 return $.root().text().replace(/\s+/g, ' ').trim();268 } catch {269 return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();270 }271}272
273export function wordCount(text) {274 if (!text) return 0;275 return text.split(/\s+/).filter(Boolean).length;276}