Pricing

from $0.09 / 1,000 results

Hacker News Scraper

Scrape Hacker News stories, comments, Ask HN, Show HN, and polls. Search by keyword, filter by points or comments, sort by relevance or date. Get title, url, author, points, comment count, and timestamps as clean JSON. No login, no API key.

Pricing

from $0.09 / 1,000 results

Rating

0.0

(0)

Developer

Mangudäi

Actor stats

Bookmarked

Total users

Monthly active users

2 days ago

Last modified

.actor/actor.json

{
    "actorSpecification": 1,
    "name": "hacker-news-scraper",
    "title": "Hacker News Scraper",
    "description": "Scrape Hacker News stories, comments, Ask HN, Show HN, and polls. No login, no API key.",
    "version": "0.0",
    "dockerfile": "../Dockerfile",
    "input": {
        "title": "Hacker News Scraper",
        "type": "object",
        "schemaVersion": 1,
        "properties": {
            "query": {
                "title": "Search query",
                "type": "string",
                "description": "Keyword to search across Hacker News. Leave empty to pull a whole content type such as the front page.",
                "editor": "textfield"
            },
            "contentType": {
                "title": "Content type",
                "type": "string",
                "description": "Which items to return.",
                "editor": "select",
                "enum": ["front_page", "story", "comment", "ask_hn", "show_hn", "poll", "all"],
                "enumTitles": ["Front page", "Stories", "Comments", "Ask HN", "Show HN", "Polls", "All"],
                "default": "front_page"
            },
            "sortBy": {
                "title": "Sort by",
                "type": "string",
                "description": "Relevance (best match) or date (newest first). Date sort can page past the 1,000-result cap.",
                "editor": "select",
                "enum": ["relevance", "date"],
                "enumTitles": ["Relevance", "Date (newest first)"],
                "default": "relevance"
            },
            "minPoints": {
                "title": "Minimum points",
                "type": "integer",
                "description": "Only return items with at least this many points.",
                "minimum": 0
            },
            "minComments": {
                "title": "Minimum comments",
                "type": "integer",
                "description": "Only return items with at least this many comments.",
                "minimum": 0
            },
            "authors": {
                "title": "Authors",
                "type": "array",
                "description": "Restrict to these Hacker News usernames.",
                "editor": "stringList"
            },
            "maxItems": {
                "title": "Max items",
                "type": "integer",
                "description": "Maximum number of items to collect.",
                "default": 50,
                "minimum": 1,
                "maximum": 100000
            },
            "proxyConfiguration": {
                "title": "Proxy configuration",
                "type": "object",
                "description": "Optional. Route requests through Apify Proxy.",
                "editor": "proxy",
                "prefill": { "useApifyProxy": true },
                "default": { "useApifyProxy": true }
            }
        }
    },
    "storages": {
        "dataset": {
            "actorSpecification": 1,
            "views": {
                "overview": {
                    "title": "Items",
                    "transformation": {
                        "fields": ["type", "title", "author", "points", "numComments", "createdAt", "url", "hnUrl"]
                    },
                    "display": {
                        "component": "table",
                        "properties": {
                            "type": { "label": "Type", "format": "text" },
                            "title": { "label": "Title", "format": "text" },
                            "author": { "label": "Author", "format": "text" },
                            "points": { "label": "Points", "format": "number" },
                            "numComments": { "label": "Comments", "format": "number" },
                            "createdAt": { "label": "Created", "format": "text" },
                            "url": { "label": "URL", "format": "link" },
                            "hnUrl": { "label": "HN link", "format": "link" }
                        }
                    }
                }
            }
        }
    },
    "output": {
        "actorOutputSchemaVersion": 1,
        "title": "Hacker News output",
        "properties": {
            "datasetItems": {
                "title": "Scraped items",
                "description": "The Hacker News items collected in this run.",
                "type": "string",
                "template": "{{links.apiDefaultDatasetUrl}}/items"
            }
        }
    }
}

src/main.js

1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3
4await Actor.init();
5
6const input = (await Actor.getInput()) ?? {};
7const {
8    query = '',
9    contentType = 'front_page',
10    sortBy = 'relevance',
11    minPoints,
12    minComments,
13    authors = [],
14    maxItems = 50,
15    proxyConfiguration: proxyInput,
16} = input;
17
18const TAG_MAP = {
19    front_page: 'front_page',
20    story: 'story',
21    comment: 'comment',
22    poll: 'poll',
23    show_hn: 'show_hn',
24    ask_hn: 'ask_hn',
25    all: null,
26};
27
28const endpoint = sortBy === 'date'
29    ? 'https://hn.algolia.com/api/v1/search_by_date'
30    : 'https://hn.algolia.com/api/v1/search';
31
32const tagParts = [];
33const baseTag = TAG_MAP[contentType];
34if (baseTag) tagParts.push(baseTag);
35const authorList = authors.map((a) => String(a).trim().replace(/^@/, '')).filter(Boolean);
36if (authorList.length === 1) tagParts.push(`author_${authorList[0]}`);
37else if (authorList.length > 1) tagParts.push(`(${authorList.map((a) => `author_${a}`).join(',')})`);
38const tags = tagParts.join(',');
39
40function buildNumericFilters(maxCreated) {
41    const f = [];
42    if (Number.isFinite(minPoints)) f.push(`points>=${minPoints}`);
43    if (Number.isFinite(minComments)) f.push(`num_comments>=${minComments}`);
44    if (Number.isFinite(maxCreated)) f.push(`created_at_i<${maxCreated}`);
45    return f.join(',');
46}
47
48const proxyConfiguration = await Actor.createProxyConfiguration(proxyInput);
49
50async function fetchPage(params) {
51    const url = `${endpoint}?${params.toString()}`;
52    const proxyUrl = proxyConfiguration ? await proxyConfiguration.newUrl() : undefined;
53    const res = await gotScraping({
54        url,
55        proxyUrl,
56        responseType: 'json',
57        timeout: { request: 30000 },
58        retry: { limit: 5, statusCodes: [429, 500, 502, 503, 504] },
59        headers: { accept: 'application/json' },
60    });
61    return res.body;
62}
63
64function mapHit(h) {
65    const tagSet = h._tags || [];
66    let type = 'story';
67    if (tagSet.includes('comment')) type = 'comment';
68    else if (tagSet.includes('poll')) type = 'poll';
69    else if (tagSet.includes('show_hn')) type = 'show_hn';
70    else if (tagSet.includes('ask_hn')) type = 'ask_hn';
71    return {
72        id: h.objectID,
73        type,
74        title: h.title ?? h.story_title ?? null,
75        url: h.url ?? h.story_url ?? null,
76        author: h.author,
77        points: h.points ?? null,
78        numComments: h.num_comments ?? null,
79        createdAt: h.created_at,
80        createdAtI: h.created_at_i,
81        storyText: h.story_text || null,
82        commentText: h.comment_text || null,
83        storyId: h.story_id ?? null,
84        parentId: h.parent_id ?? null,
85        tags: tagSet,
86        hnUrl: `https://news.ycombinator.com/item?id=${h.objectID}`,
87    };
88}
89
90const PER_PAGE = 100;
91let pushed = 0;
92let maxCreated; // for date-window chaining beyond Algolia's 1000-result cap
93
94while (pushed < maxItems) {
95    const params = new URLSearchParams({ hitsPerPage: String(PER_PAGE) });
96    if (query) params.set('query', query);
97    if (tags) params.set('tags', tags);
98    const nf = buildNumericFilters(maxCreated);
99    if (nf) params.set('numericFilters', nf);
100
101    let page = 0;
102    let nbPages = 1;
103    let windowMinCreated;
104    let windowGotAny = false;
105
106    while (pushed < maxItems && page < nbPages) {
107        params.set('page', String(page));
108        let body;
109        try {
110            body = await fetchPage(params);
111        } catch (err) {
112            log.warning(`Fetch failed on page ${page}: ${err.message}`);
113            break;
114        }
115        const hits = body?.hits ?? [];
116        nbPages = body?.nbPages ?? 0;
117        if (!hits.length) break;
118
119        const batch = [];
120        for (const h of hits) {
121            batch.push(mapHit(h));
122            if (typeof h.created_at_i === 'number') {
123                windowMinCreated = windowMinCreated === undefined ? h.created_at_i : Math.min(windowMinCreated, h.created_at_i);
124            }
125            if (pushed + batch.length >= maxItems) break;
126        }
127        await Actor.pushData(batch);
128        pushed += batch.length;
129        windowGotAny = windowGotAny || batch.length > 0;
130        log.info(`Collected ${pushed}/${maxItems} (page ${page + 1}/${nbPages})`);
131        page += 1;
132    }
133
134    // Only date-sorted queries can page past Algolia's 1000-result cap via time windows.
135    if (sortBy !== 'date' || !windowGotAny || windowMinCreated === undefined || pushed >= maxItems) break;
136    if (maxCreated !== undefined && windowMinCreated >= maxCreated) break;
137    maxCreated = windowMinCreated;
138}
139
140log.info(`Done. Pushed ${pushed} items.`);
141await Actor.exit();

test/main.test.js

1import { describe, expect, it } from 'vitest';
2
3describe('Example test', () => {
4    it('works', () => {
5        expect(1 + 1).toBe(2);
6    });
7});

.github/workflows/test.yaml

name: CI

on:
    pull_request:
    push:
        branches: [main, master]

jobs:
    test:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v7
            - uses: actions/setup-node@v6
              with:
                  node-version: 24
                  cache: npm
            - run: npm ci
            - run: npm test

.dockerignore

# configurations
.idea
.vscode
.zed

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

.editorconfig

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
quote_type = single

.gitignore

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
.vscode
.zed
node_modules
storage

.npmrc

min-release-age=1

.prettierrc

{
    "printWidth": 120,
    "tabWidth": 4,
    "singleQuote": true
}

AGENTS.md

1# Apify Actors Development Guide
2
3Important: Before you begin, fill in the `generatedBy` property in the meta section of `.actor/actor.json`. Replace it with the tool and model you're currently using, such as "Claude Code with Claude Sonnet 4.5". This helps Apify monitor and improve AGENTS.md for specific AI tools and models.
4
5## What are Apify Actors?
6
7- Actors are serverless programs that run in the cloud. They're inspired by the UNIX philosophy - programs that do one thing well and can be easily combined to build complex systems.
8- Actors are programs packaged as Docker images that run in isolated containers
9
10## Core Concepts
11
12- Accept well-defined JSON input
13- Perform isolated tasks (web scraping, automation, data processing)
14- Produce structured JSON output to datasets and/or store data in key-value stores
15- Can run from seconds to hours or even indefinitely
16- Persist state and can be restarted
17
18## Do
19
20- accept well-defined JSON input and produce structured JSON output
21- use Apify SDK (`apify`) for code running ON Apify platform
22- validate input early with proper error handling and fail gracefully
23- use CheerioCrawler for static HTML content (10x faster than browsers)
24- use PlaywrightCrawler only for JavaScript-heavy sites and dynamic content
25- use router pattern (createCheerioRouter/createPlaywrightRouter) for complex crawls
26- implement retry strategies with exponential backoff for failed requests
27- use proper concurrency settings (HTTP: 10-50, Browser: 1-5)
28- set sensible defaults in `.actor/input_schema.json` for all optional fields
29- set up output schema in `.actor/output_schema.json`
30- clean and validate data before pushing to dataset
31- use semantic CSS selectors and fallback strategies for missing elements
32- respect robots.txt, ToS, and implement rate limiting with delays
33- check which tools (cheerio/playwright/crawlee) are installed before applying guidance
34- use `apify/log` package for logging (censors sensitive data)
35- implement readiness probe handler for standby Actors
36- handle the `aborting` event to gracefully shut down when Actor is stopped
37
38## Don't
39
40- do not rely on `Dataset.getInfo()` for final counts on Cloud platform
41- do not use browser crawlers when HTTP/Cheerio works (massive performance gains with HTTP)
42- do not hard code values that should be in input schema or environment variables
43- do not skip input validation or error handling
44- do not overload servers - use appropriate concurrency and delays
45- do not scrape prohibited content or ignore Terms of Service
46- do not store personal/sensitive data unless explicitly permitted
47- do not use deprecated options like `requestHandlerTimeoutMillis` on CheerioCrawler (v3.x)
48- do not use `additionalHttpHeaders` - use `preNavigationHooks` instead
49- do not assume that local storage is persistent or automatically synced to Apify Console - when running locally with `apify run`, the `storage/` directory is local-only and is NOT pushed to the Cloud
50- do not disable standby mode (`usesStandbyMode: false`) without explicit permission
51
52## Logging
53
54- **ALWAYS use the `apify/log` package for logging** - This package contains critical security logic including censoring sensitive data (Apify tokens, API keys, credentials) to prevent accidental exposure in logs
55
56### Available Log Levels in `apify/log`
57
58The Apify log package provides the following methods for logging:
59
60- `log.debug()` - Debug level logs (detailed diagnostic information)
61- `log.info()` - Info level logs (general informational messages)
62- `log.warning()` - Warning level logs (warning messages for potentially problematic situations)
63- `log.warningOnce()` - Warning level logs (same warning message logged only once)
64- `log.error()` - Error level logs (error messages for failures)
65- `log.exception()` - Exception level logs (for exceptions with stack traces)
66- `log.perf()` - Performance level logs (performance metrics and timing information)
67- `log.deprecated()` - Deprecation level logs (warnings about deprecated code)
68- `log.softFail()` - Soft failure logs (non-critical failures that don't stop execution, e.g., input validation errors, skipped items)
69- `log.internal()` - Internal level logs (internal/system messages)
70
71**Best practices:**
72
73- Use `log.debug()` for detailed operation-level diagnostics (inside functions)
74- Use `log.info()` for general informational messages (API requests, successful operations)
75- Use `log.warning()` for potentially problematic situations (validation failures, unexpected states)
76- Use `log.error()` for actual errors and failures
77- Use `log.exception()` for caught exceptions with stack traces
78
79## Graceful Abort Handling
80
81Handle the `aborting` event to terminate the Actor quickly when stopped by user or platform, minimizing costs especially for PPU/PPE+U billing.
82
83```javascript
84import { setTimeout } from 'node:timers/promises';
85
86Actor.on('aborting', async () => {
87    // Persist any state, do any cleanup you need, and terminate the Actor using `await Actor.exit()` explicitly as soon as possible
88    // This will help ensure that the Actor is doing best effort to honor any potential limits on costs of a single run set by the user
89    // Wait 1 second to allow Crawlee/SDK useState and other state persistence operations to complete
90    // This is a temporary workaround until SDK implements proper state persistence in the aborting event
91    await setTimeout(1000);
92    await Actor.exit();
93});
94```
95
96## Standby Mode
97
98- **NEVER disable standby mode (`usesStandbyMode: false`) in `.actor/actor.json` without explicit permission** - Actor Standby mode solves this problem by letting you have the Actor ready in the background, waiting for the incoming HTTP requests. In a sense, the Actor behaves like a real-time web server or standard API server instead of running the logic once to process everything in batch. Always keep `usesStandbyMode: true` unless there is a specific documented reason to disable it
99- **ALWAYS implement readiness probe handler for standby Actors** - Handle the `x-apify-container-server-readiness-probe` header at GET / endpoint to ensure proper Actor lifecycle management
100
101You can recognize a standby Actor by checking the `usesStandbyMode` property in `.actor/actor.json`. Only implement the readiness probe if this property is set to `true`.
102
103### Readiness Probe Implementation Example
104
105```javascript
106// Apify standby readiness probe at root path
107app.get('/', (req, res) => {
108    res.writeHead(200, { 'Content-Type': 'text/plain' });
109    if (req.headers['x-apify-container-server-readiness-probe']) {
110        res.end('Readiness probe OK\n');
111    } else {
112        res.end('Actor is ready\n');
113    }
114});
115```
116
117Key points:
118
119- Detect the `x-apify-container-server-readiness-probe` header in incoming requests
120- Respond with HTTP 200 status code for both readiness probe and normal requests
121- This enables proper Actor lifecycle management in standby mode
122
123## Commands
124
125```bash
126# Bootstrap & local development
127apify create [name]                    # Create new Actor project from a template
128apify init                             # Initialize Actor in current directory
129apify run                              # Run Actor locally with simulated platform env
130apify run --purge                      # Run after clearing previous local storage
131apify validate-schema                  # Validate .actor/input_schema.json
132
133# Authentication & account
134apify login                            # Authenticate account (token stored in ~/.apify)
135apify logout                           # Remove stored credentials
136apify info                             # Print currently authenticated account info
137
138# Deployment & remote execution
139apify push                             # Deploy Actor to platform per .actor/actor.json
140apify pull <actor>                     # Download Actor code from the platform
141apify call <actor>                     # Execute Actor remotely on the platform
142apify actors build <actor>             # Create a new build of an Actor
143apify runs ls                          # List recent runs
144
145# Discovery (search Apify Store for community Actors)
146apify actors search "<query>" --user-agent <your-agent-name>
147apify actors info <actor>              # Details about a specific Actor
148
149# Secrets (referenced from actor.json via "@mySecret")
150apify secrets add <name> <value>       # Store a secret locally; uploaded on push
151apify secrets ls                       # List stored secret keys
152
153# Direct API access
154apify api <endpoint>                   # Authenticated HTTP request to Apify API
155
156# Help
157apify help                             # List all commands
158apify <command> --help                 # Detailed help for a specific command
159```
160
161Note: If no dedicated Actor exists for your target, search Apify Store for community options with `apify actors search "<query>" --user-agent <your-agent-name>` before building from scratch.
162
163Tip: Inside a running Actor, prefer the SDK (`Actor.getInput()`, `Actor.pushData()`, `Actor.setValue()`) over the equivalent `apify actor` runtime subcommands.
164
165## Apify Platform Environment
166
167When the Actor runs on the Apify platform, the API token is automatically available via the `APIFY_TOKEN` environment variable (note: the variable is `APIFY_TOKEN`, not `APIFY_API_TOKEN`). The Apify SDK reads it automatically, so you do not need to pass it explicitly. Locally, run `apify login` once and the SDK will use your stored credentials.
168
169## Safety and Permissions
170
171Allowed without prompt:
172
173- read files with `Actor.getValue()`
174- push data with `Actor.pushData()`
175- set values with `Actor.setValue()`
176- enqueue requests to RequestQueue
177- run locally with `apify run`
178
179Ask first:
180
181- npm/pip package installations
182- apify push (deployment to cloud)
183- proxy configuration changes (requires paid plan)
184- Dockerfile changes affecting builds
185- deleting datasets or key-value stores
186
187## Project Structure
188
189.actor/
190├── actor.json # Actor config: name, version, env vars, runtime settings
191├── input_schema.json # Input validation & Console form definition
192└── output_schema.json # Specifies where an Actor stores its output
193src/
194└── main.js # Actor entry point and orchestrator
195storage/ # Local-only storage for development (NOT synced to Cloud)
196├── datasets/ # Output items (JSON objects)
197├── key_value_stores/ # Files, config, INPUT
198└── request_queues/ # Pending crawl requests
199Dockerfile # Container image definition
200AGENTS.md # AI agent instructions (this file)
201
202## Local vs Cloud Storage
203
204When running locally with `apify run`, the Apify SDK emulates Cloud storage APIs using the local `storage/` directory. This local storage behaves differently from Cloud storage:
205
206- **Local storage is NOT persistent** - The `storage/` directory is meant for local development and testing only. Data stored there (datasets, key-value stores, request queues) exists only on your local disk.
207- **Local storage is NOT automatically pushed to Apify Console** - Running `apify run` does not upload any storage data to the Apify platform. The data stays local.
208- **Each local run may overwrite previous data** - The local `storage/` directory is reused between runs, but this is local-only behavior, not Cloud persistence.
209- **Cloud storage only works when running on Apify platform** - After deploying with `apify push` and running the Actor in the Cloud, storage calls (`Actor.pushData()`, `Actor.setValue()`, etc.) interact with real Apify Cloud storage, which is then visible in the Apify Console.
210- **To verify Actor output, deploy and run in Cloud** - Do not rely on local `storage/` contents as proof that data will appear in the Apify Console. Always test by deploying (`apify push`) and running the Actor on the platform.
211
212## Actor Input Schema
213
214The input schema defines the input parameters for an Actor. It's a JSON object comprising various field types supported by the Apify platform.
215
216### Structure
217
218```json
219{
220    "title": "<INPUT-SCHEMA-TITLE>",
221    "type": "object",
222    "schemaVersion": 1,
223    "properties": {/* define input fields here */},
224    "required": []
225}
226```
227
228### Example
229
230```json
231{
232    "title": "E-commerce Product Scraper Input",
233    "type": "object",
234    "schemaVersion": 1,
235    "properties": {
236        "startUrls": {
237            "title": "Start URLs",
238            "type": "array",
239            "description": "URLs to start scraping from (category pages or product pages)",
240            "editor": "requestListSources",
241            "default": [{ "url": "https://example.com/category" }],
242            "prefill": [{ "url": "https://example.com/category" }]
243        },
244        "followVariants": {
245            "title": "Follow Product Variants",
246            "type": "boolean",
247            "description": "Whether to scrape product variants (different colors, sizes)",
248            "default": true
249        },
250        "maxRequestsPerCrawl": {
251            "title": "Max Requests per Crawl",
252            "type": "integer",
253            "description": "Maximum number of pages to scrape (0 = unlimited)",
254            "default": 1000,
255            "minimum": 0
256        },
257        "proxyConfiguration": {
258            "title": "Proxy Configuration",
259            "type": "object",
260            "description": "Proxy settings for anti-bot protection",
261            "editor": "proxy",
262            "default": { "useApifyProxy": false }
263        },
264        "locale": {
265            "title": "Locale",
266            "type": "string",
267            "description": "Language/country code for localized content",
268            "default": "cs",
269            "enum": ["cs", "en", "de", "sk"],
270            "enumTitles": ["Czech", "English", "German", "Slovak"]
271        }
272    },
273    "required": ["startUrls"]
274}
275```
276
277## Actor Output Schema
278
279The Actor output schema builds upon the schemas for the dataset and key-value store. It specifies where an Actor stores its output and defines templates for accessing that output. Apify Console uses these output definitions to display run results.
280
281### Structure
282
283```json
284{
285    "actorOutputSchemaVersion": 1,
286    "title": "<OUTPUT-SCHEMA-TITLE>",
287    "properties": {/* define your outputs here */}
288}
289```
290
291### Example
292
293```json
294{
295    "actorOutputSchemaVersion": 1,
296    "title": "Output schema of the files scraper",
297    "properties": {
298        "files": {
299            "type": "string",
300            "title": "Files",
301            "template": "{{links.apiDefaultKeyValueStoreUrl}}/keys"
302        },
303        "dataset": {
304            "type": "string",
305            "title": "Dataset",
306            "template": "{{links.apiDefaultDatasetUrl}}/items"
307        }
308    }
309}
310```
311
312### Output Schema Template Variables
313
314- `links` (object) - Contains quick links to most commonly used URLs
315- `links.publicRunUrl` (string) - Public run url in format `https://console.apify.com/view/runs/:runId`
316- `links.consoleRunUrl` (string) - Console run url in format `https://console.apify.com/actors/runs/:runId`
317- `links.apiRunUrl` (string) - API run url in format `https://api.apify.com/v2/actor-runs/:runId`
318- `links.apiDefaultDatasetUrl` (string) - API url of default dataset in format `https://api.apify.com/v2/datasets/:defaultDatasetId`
319- `links.apiDefaultKeyValueStoreUrl` (string) - API url of default key-value store in format `https://api.apify.com/v2/key-value-stores/:defaultKeyValueStoreId`
320- `links.containerRunUrl` (string) - URL of a webserver running inside the run in format `https://<containerId>.runs.apify.net/`
321- `run` (object) - Contains information about the run same as it is returned from the `GET Run` API endpoint
322- `run.defaultDatasetId` (string) - ID of the default dataset
323- `run.defaultKeyValueStoreId` (string) - ID of the default key-value store
324
325## Dataset Schema Specification
326
327The dataset schema defines how your Actor's output data is structured, transformed, and displayed in the Output tab in the Apify Console.
328
329### Example
330
331Consider an example Actor that calls `Actor.pushData()` to store data into dataset:
332
333```javascript
334import { Actor } from 'apify';
335// Initialize the JavaScript SDK
336await Actor.init();
337
338/**
339 * Actor code
340 */
341await Actor.pushData({
342    numericField: 10,
343    pictureUrl: 'https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_92x30dp.png',
344    linkUrl: 'https://google.com',
345    textField: 'Google',
346    booleanField: true,
347    dateField: new Date(),
348    arrayField: ['#hello', '#world'],
349    objectField: {},
350});
351
352// Exit successfully
353await Actor.exit();
354```
355
356To set up the Actor's output tab UI, reference a dataset schema file in `.actor/actor.json`:
357
358```json
359{
360    "actorSpecification": 1,
361    "name": "book-library-scraper",
362    "title": "Book Library Scraper",
363    "version": "1.0.0",
364    "storages": {
365        "dataset": "./dataset_schema.json"
366    }
367}
368```
369
370Then create the dataset schema in `.actor/dataset_schema.json`:
371
372```json
373{
374    "actorSpecification": 1,
375    "fields": {},
376    "views": {
377        "overview": {
378            "title": "Overview",
379            "transformation": {
380                "fields": [
381                    "pictureUrl",
382                    "linkUrl",
383                    "textField",
384                    "booleanField",
385                    "arrayField",
386                    "objectField",
387                    "dateField",
388                    "numericField"
389                ]
390            },
391            "display": {
392                "component": "table",
393                "properties": {
394                    "pictureUrl": {
395                        "label": "Image",
396                        "format": "image"
397                    },
398                    "linkUrl": {
399                        "label": "Link",
400                        "format": "link"
401                    },
402                    "textField": {
403                        "label": "Text",
404                        "format": "text"
405                    },
406                    "booleanField": {
407                        "label": "Boolean",
408                        "format": "boolean"
409                    },
410                    "arrayField": {
411                        "label": "Array",
412                        "format": "array"
413                    },
414                    "objectField": {
415                        "label": "Object",
416                        "format": "object"
417                    },
418                    "dateField": {
419                        "label": "Date",
420                        "format": "date"
421                    },
422                    "numericField": {
423                        "label": "Number",
424                        "format": "number"
425                    }
426                }
427            }
428        }
429    }
430}
431```
432
433### Structure
434
435```json
436{
437    "actorSpecification": 1,
438    "fields": {},
439    "views": {
440        "<VIEW_NAME>": {
441            "title": "string (required)",
442            "description": "string (optional)",
443            "transformation": {
444                "fields": ["string (required)"],
445                "unwind": ["string (optional)"],
446                "flatten": ["string (optional)"],
447                "omit": ["string (optional)"],
448                "limit": "integer (optional)",
449                "desc": "boolean (optional)"
450            },
451            "display": {
452                "component": "table (required)",
453                "properties": {
454                    "<FIELD_NAME>": {
455                        "label": "string (optional)",
456                        "format": "text|number|date|link|boolean|image|array|object (optional)"
457                    }
458                }
459            }
460        }
461    }
462}
463```
464
465**Dataset Schema Properties:**
466
467- `actorSpecification` (integer, required) - Specifies the version of dataset schema structure document (currently only version 1)
468- `fields` (JSONSchema object, required) - Schema of one dataset object (use JsonSchema Draft 2020-12 or compatible)
469- `views` (DatasetView object, required) - Object with API and UI views description
470
471**DatasetView Properties:**
472
473- `title` (string, required) - Visible in UI Output tab and API
474- `description` (string, optional) - Only available in API response
475- `transformation` (ViewTransformation object, required) - Data transformation applied when loading from Dataset API
476- `display` (ViewDisplay object, required) - Output tab UI visualization definition
477
478**ViewTransformation Properties:**
479
480- `fields` (string[], required) - Fields to present in output (order matches column order)
481- `unwind` (string[], optional) - Deconstructs nested children into parent object
482- `flatten` (string[], optional) - Transforms nested object into flat structure
483- `omit` (string[], optional) - Removes specified fields from output
484- `limit` (integer, optional) - Maximum number of results (default: all)
485- `desc` (boolean, optional) - Sort order (true = newest first)
486
487**ViewDisplay Properties:**
488
489- `component` (string, required) - Only `table` is available
490- `properties` (Object, optional) - Keys matching `transformation.fields` with ViewDisplayProperty values
491
492**ViewDisplayProperty Properties:**
493
494- `label` (string, optional) - Table column header
495- `format` (string, optional) - One of: `text`, `number`, `date`, `link`, `boolean`, `image`, `array`, `object`
496
497## Key-Value Store Schema Specification
498
499The key-value store schema organizes keys into logical groups called collections for easier data management.
500
501### Example
502
503Consider an example Actor that calls `Actor.setValue()` to save records into the key-value store:
504
505```javascript
506import { Actor } from 'apify';
507// Initialize the JavaScript SDK
508await Actor.init();
509
510/**
511 * Actor code
512 */
513await Actor.setValue('document-1', 'my text data', { contentType: 'text/plain' });
514
515await Actor.setValue(`image-${imageID}`, imageBuffer, { contentType: 'image/jpeg' });
516
517// Exit successfully
518await Actor.exit();
519```
520
521To configure the key-value store schema, reference a schema file in `.actor/actor.json`:
522
523```json
524{
525    "actorSpecification": 1,
526    "name": "data-collector",
527    "title": "Data Collector",
528    "version": "1.0.0",
529    "storages": {
530        "keyValueStore": "./key_value_store_schema.json"
531    }
532}
533```
534
535Then create the key-value store schema in `.actor/key_value_store_schema.json`:
536
537```json
538{
539    "actorKeyValueStoreSchemaVersion": 1,
540    "title": "Key-Value Store Schema",
541    "collections": {
542        "documents": {
543            "title": "Documents",
544            "description": "Text documents stored by the Actor",
545            "keyPrefix": "document-"
546        },
547        "images": {
548            "title": "Images",
549            "description": "Images stored by the Actor",
550            "keyPrefix": "image-",
551            "contentTypes": ["image/jpeg"]
552        }
553    }
554}
555```
556
557### Structure
558
559```json
560{
561    "actorKeyValueStoreSchemaVersion": 1,
562    "title": "string (required)",
563    "description": "string (optional)",
564    "collections": {
565        "<COLLECTION_NAME>": {
566            "title": "string (required)",
567            "description": "string (optional)",
568            "key": "string (conditional - use key OR keyPrefix)",
569            "keyPrefix": "string (conditional - use key OR keyPrefix)",
570            "contentTypes": ["string (optional)"],
571            "jsonSchema": "object (optional)"
572        }
573    }
574}
575```
576
577**Key-Value Store Schema Properties:**
578
579- `actorKeyValueStoreSchemaVersion` (integer, required) - Version of key-value store schema structure document (currently only version 1)
580- `title` (string, required) - Title of the schema
581- `description` (string, optional) - Description of the schema
582- `collections` (Object, required) - Object where each key is a collection ID and value is a Collection object
583
584**Collection Properties:**
585
586- `title` (string, required) - Collection title shown in UI tabs
587- `description` (string, optional) - Description appearing in UI tooltips
588- `key` (string, conditional) - Single specific key for this collection
589- `keyPrefix` (string, conditional) - Prefix for keys included in this collection
590- `contentTypes` (string[], optional) - Allowed content types for validation
591- `jsonSchema` (object, optional) - JSON Schema Draft 07 format for `application/json` content type validation
592
593Either `key` or `keyPrefix` must be specified for each collection, but not both.
594
595## Actor README
596
597**Always generate a README.md file as part of Actor development.** The README is the Actor's public landing page on Apify Store - it serves as SEO, first impression, documentation, and support page combined.
598
599### Required: Generate README automatically
600
601When building an Actor, always create a `README.md` in the project root. Do not wait for the user to ask for it. The README is a critical part of a complete Actor.
602
603### README structure
604
605Write in Markdown. Use H2 (`##`) for main sections (these become the table of contents) and H3 (`###`) for subsections. Do not use H1 - the Actor name is automatically the H1. Aim for at least 300 words.
606
607Include these sections in order:
608
6091. **What does [Actor name] do?** - 2-3 sentences explaining what it does, what data it extracts, and how to try it. Link to the target website. Mention Apify platform advantages (API access, scheduling, integrations, proxy rotation, monitoring).
6102. **Why use [Actor name]?** - Business use cases and benefits.
6113. **How to use [Actor name]** - Numbered step-by-step tutorial. Keep it simple and reassuring.
6124. **Input** - Describe input fields. Reference the Input tab. Optionally include a screenshot or JSON example of the input schema.
6135. **Output** - Show a simplified JSON output example. Mention "You can download the dataset in various formats such as JSON, HTML, CSV, or Excel."
6146. **Data table** - If the Actor extracts data, include a table of the main data fields it outputs.
6157. **Pricing / Cost estimation** - Set expectations on cost. Mention free tier limits if applicable. Frame as "How much does it cost to scrape [target site]?"
6168. **Tips or Advanced options** - How to optimize runs, limit compute units, improve speed or accuracy.
6179. **FAQ, disclaimers, and support** - Legality disclaimer for scrapers, known limitations, link to Issues tab for feedback, mention custom solution availability.
618
619### README best practices
620
621- Write SEO-friendly headings with relevant keywords (e.g., "How to scrape [site] data" not just "Tutorial")
622- Bold the most important words in the intro
623- The first 25% of the README matters most - front-load the value proposition
624- Match the tone to the target audience: simple language for no-code users, technical details for developers
625- Include a JSON output example showing 1-2 representative items
626- Reference these top Actors for README best practices: https://apify.com/apify/instagram-scraper and https://apify.com/compass/crawler-google-places
627- Embed YouTube video URLs on their own line (Apify Console auto-renders them)
628- Use HTML for image sizing if needed; CSS is not supported
629
630## MCP Tools
631
632### Apify MCP
633
634If the Apify MCP server is configured, use these tools for documentation:
635
636- `search-apify-docs` - Search documentation
637- `fetch-apify-docs` - Get full doc pages
638
639Otherwise, reference: `@https://mcp.apify.com/`
640
641### Playwright MCP (debugging)
642
643The Playwright MCP server is a useful tool for debugging Actors that interact with the web - it lets the agent drive a real browser to inspect pages, capture selectors, and reproduce issues.
644
645Install with the Claude Code CLI:
646
647```bash
648claude mcp add playwright npx @playwright/mcp@latest
649```
650
651Or add it manually to your MCP config:
652
653```json
654{
655    "mcpServers": {
656        "playwright": {
657            "command": "npx",
658            "args": ["@playwright/mcp@latest"]
659        }
660    }
661}
662```
663
664## Resources
665
666- [docs.apify.com/llms.txt](https://docs.apify.com/llms.txt) - Quick reference
667- [docs.apify.com/llms-full.txt](https://docs.apify.com/llms-full.txt) - Complete docs
668- [crawlee.dev](https://crawlee.dev) - Crawlee documentation
669- [whitepaper.actor](https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md) - Complete Actor specification

CLAUDE.md

1@AGENTS.md

Dockerfile

FROM apify/actor-node:20

COPY package*.json ./

RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version

COPY . ./

CMD ["npm", "start", "--silent"]

eslint.config.mjs

1import prettier from 'eslint-config-prettier';
2
3import apify from '@apify/eslint-config/js.js';
4
5// eslint-disable-next-line import-x/no-default-export
6export default [{ ignores: ['**/dist'] }, ...apify, prettier];

package.json

{
    "name": "hacker-news-scraper",
    "version": "0.1.0",
    "type": "module",
    "description": "Scrape Hacker News stories, comments, Ask HN, Show HN, and polls via the public Algolia HN API.",
    "engines": {
        "node": ">=18"
    },
    "dependencies": {
        "apify": "^3.2.6",
        "got-scraping": "^4.0.4"
    },
    "scripts": {
        "start": "node src/main.js"
    }
}

Hacker News Scraper — Stories, Comments, Ask/Show HN

hipersoft/hacker-news-scraper

Scrape Hacker News stories, comments, Ask HN, Show HN, jobs and polls by keyword, points, comments and date range. Sort by relevance or newest. Title, URL, author, points, comment count, text and HN link. Fast clean HTTP via the HN Algolia API, no key.

hiper soft

Hacker News Search & Monitor

agentictools/hacker-news-search

Search and export Hacker News stories, comments, and Ask or Show HN by keyword, author, points, or date. No API key.

Ken Agland

Hacker News Search Scraper Stories, Comments, Show HN, Ask HN

seemuapps/hn-search-scraper

Search Hacker News stories, comments, Show HN, Ask HN, polls, and jobs by keyword, author, date range, points, and comment count. Full text and engagement metrics. No login.

Andrew

Hacker News Scraper — Stories & Comments Search

omao/hackernews

Search Hacker News stories, comments, Show HN and Ask HN by keyword into clean JSON: title, URL, author, points, comment count and dates. Powered by the official HN Algolia API. No anti-bot.

Marouane Oulabass

Hacker News Search Scraper (Free)

fit_melon/hacker-news-search-scraper

Search Hacker News stories, comments, Ask HN & Show HN by keyword. Filter by points, sort by date or relevance, full HN archive. Clean JSON output via the official Algolia HN API. 100% free — no API key needed.

D N

Hacker News Search — Stories, Comments & Developer Sentiment

ryanclinton/hackernews-search

Search and extract stories, comments, polls, Show HN, and Ask HN posts from Hacker News. This actor uses the Algolia HN Search API to find content by keyword, filter by author, date range, minimum points, and comment count -- then returns clean, structured JSON ready for analysis, monitoring, or ...

Ryan Clinton

Hacker News Search & Monitor

scrapeworks/hacker-news-search

Search and monitor Hacker News stories, comments, Ask HN, Show HN, jobs and users. No auth, clean structured JSON. Filter by points, date, author, and tags.

Nicolas van Arkens

Hacker News API Scraper - Stories, Comments & Polls

logiover/hacker-news-search-scraper

Hacker News scraper & HN API alternative. Scrape stories, comments, Ask HN & Show HN without login; export to CSV/JSON. No key, no proxy.

Logiover

🟧 Hacker News Scraper — Stories, Comments & Search by Keyword

iskoren/hacker-news-scraper

Search and scrape Hacker News stories, comments, and polls by keyword — points, authors, comment counts, dates, and links. Powered by the official HN API.