No credit card required

Productboard Roadmap Scraper

zuzana_stetinova/productboard-roadmap-scraper

No credit card required

This scraper extracts data from Productboard roadmap to a key-value store (OUTPUT) as a map (id -- feature). Scraped feature data includes feature name, description, timeline, teams and connected features with their title, description and timeline.

.dockerignore

1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10node_modules
11
12# git folder
13.git

.editorconfig

1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf

.eslintrc

1{
2    "root": true,
3    "env": {
4        "browser": true,
5        "es2020": true,
6        "node": true
7    },
8    "extends": [
9        "@apify/eslint-config-ts"
10    ],
11    "parserOptions": {
12        "project": "./tsconfig.json",
13        "ecmaVersion": 2020
14    },
15    "ignorePatterns": [
16        "node_modules",
17        "dist",
18        "**/*.d.ts"
19    ]
20}

.gitignore

1# This file tells Git which files shouldn't be added to source control
2
3.DS_Store
4.idea
5dist
6node_modules
7apify_storage
8storage
9
10# Added by Apify CLI
11.venv

package.json

1{
2	"name": "productboard-roadmap-scraper",
3	"version": "0.0.1",
4	"type": "module",
5	"description": "Apify actor to scrape data from Productboard roadmap.",
6	"engines": {
7		"node": ">=18.0.0"
8	},
9	"dependencies": {
10		"@apify/log": "^2.5.0",
11		"apify": "^3.1.10",
12		"axios": "^1.6.7",
13		"crawlee": "^3.5.4",
14		"playwright": "1.42.1"
15	},
16	"devDependencies": {
17		"@apify/eslint-config-ts": "^0.3.0",
18		"@apify/tsconfig": "^0.1.0",
19		"@typescript-eslint/eslint-plugin": "^6.7.2",
20		"@typescript-eslint/parser": "^6.7.2",
21		"eslint": "^8.50.0",
22		"tsx": "^4.6.2",
23		"typescript": "^5.3.3"
24	},
25	"scripts": {
26		"start": "npm run start:dev",
27		"start:prod": "node dist/main.js",
28		"start:dev": "tsx src/main.ts",
29		"build": "tsc",
30		"lint": "eslint ./src --ext .ts",
31		"lint:fix": "eslint ./src --ext .ts --fix"
32	},
33	"license": "ISC"
34}

tsconfig.json

1{
2    "extends": "@apify/tsconfig",
3    "compilerOptions": {
4        "module": "NodeNext",
5        "moduleResolution": "NodeNext",
6        "target": "ES2022",
7        "outDir": "dist",
8        "noUnusedLocals": false,
9        "skipLibCheck": true,
10        "lib": ["DOM"]
11    },
12    "include": [
13        "./src/**/*"
14    ]
15}

.actor/Dockerfile

1FROM apify/actor-node-playwright-chrome:18 AS builder
2
3COPY --chown=myuser package*.json ./
4
5RUN npm install --include=dev --audit=false
6
7COPY --chown=myuser . ./
8
9RUN npm run build
10
11FROM apify/actor-node-playwright-chrome:18
12
13COPY --chown=myuser package*.json ./
14
15RUN npm --quiet set progress=false \
16    && npm install --omit=dev --omit=optional \
17    && echo "Installed NPM packages:" \
18    && (npm list --omit=dev --all || true) \
19    && echo "Node.js version:" \
20    && node --version \
21    && echo "NPM version:" \
22    && npm --version \
23    && rm -r ~/.npm
24
25COPY --from=builder --chown=myuser /home/myuser/dist ./dist
26
27COPY --chown=myuser . ./
28
29
30CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

.actor/actor.json

1{
2	"actorSpecification": 1,
3	"name": "productboard-roadmap-scraper",
4	"title": "Productboard roadmap scraper",
5	"description": "Actor is scraping basic data from Productboard roadmap to a key-value store.",
6	"version": "0.0",
7	"input": "./input_schema.json",
8	"dockerfile": "./Dockerfile"
9}

.actor/input_schema.json

1{
2    "title": "Productboard roadmap scraper",
3    "type": "object",
4    "schemaVersion": 1,
5    "properties": {
6        "productboardRoadmapUrl": {
7            "title": "Productboard roadmap URL",
8            "type": "string",
9            "description": "URL of the Productboard roadmap.",
10            "editor": "textfield"
11        },
12        "userEmail": {
13            "title": "Productboard user email",
14            "type": "string",
15            "description": "Email for Productboard account used for access.",
16            "editor": "textfield"
17        },
18        "userPassword": {
19            "title": "Productboard user password",
20            "type": "string",
21            "description": "Password for Productboard account used for access.",
22            "editor": "textfield",
23            "isSecret": true
24        }
25    },
26    "required": ["productboardRoadmapUrl", "userEmail", "userPassword"]
27}

src/constants.ts

1export const loginSelectors = {
2    emailInput: 'input#email',
3    passwordInput: 'input#password',
4    submitButton: 'button[type="submit"]',
5} as const;
6
7export const STORE_ID = 'result';
8export const STORE_RESULT_KEY = 'OUTPUT';
9
10const getRoadmapId = (url: string) => url.split('/roadmap/')[1].split('-')[0];
11export const isRoadmapInitialRequest = ({ requestUrl, productboardRoadmapUrl }:{requestUrl: string, productboardRoadmapUrl: string}) => (
12    requestUrl.includes(`/${getRoadmapId(productboardRoadmapUrl)}/initial`)
13);

src/handlers.ts

1import { Page, Response } from 'playwright';
2import axios from 'axios';
3import log from '@apify/log';
4import { isArrayOfObjects, isObject } from './utils.js';
5import { FeatureData } from './types.js';
6import { loginSelectors } from './constants.js';
7
8export const handleLogin = async (page: Page, { userEmail, userPassword }: {
9    userEmail: string;
10    userPassword: string
11}) => {
12    await page.click(loginSelectors.emailInput);
13    await page.keyboard.type(userEmail);
14    await page.click(loginSelectors.passwordInput);
15    await page.keyboard.type(userPassword);
16    await page.click(loginSelectors.submitButton);
17};
18
19const isFeature = (test: Record<string, unknown>) => test.featureType === 'feature';
20const isSubfeature = (test: Record<string, unknown>) => test.featureType === 'subfeature';
21
22type RequiredResponse = {
23    releases: Array<Record<string, unknown>>
24    features: Array<Record<string, unknown>>
25    listColumnItems: Array<Record<string, unknown>>
26    releaseAssignments: Array<Record<string, unknown>>
27    columnValues: Array<Record<string, unknown>>
28}
29
30export const parseRequiredResponse = async ({ response } : { response: Response }): Promise<RequiredResponse | null> => {
31    try {
32        const jsonResponse = await response.json();
33        if (!isObject(jsonResponse)) {
34            throw new Error('Invalid response format');
35        }
36        const { releases, features, listColumnItems, releaseAssignments, columnValues } = jsonResponse;
37        if (!isArrayOfObjects(releases)) throw new Error('Invalid data format of releases');
38        if (!isArrayOfObjects(features)) throw new Error('Invalid data format of features');
39        if (!isArrayOfObjects(listColumnItems)) throw new Error('Invalid data format of listColumnItems');
40        if (!isArrayOfObjects(releaseAssignments)) throw new Error('Invalid data format of releaseAssignments');
41        if (!isArrayOfObjects(columnValues)) throw new Error('Invalid data format of columnValues');
42
43        return { releases, features, listColumnItems, releaseAssignments, columnValues };
44    } catch (err) {
45        log.error('Failed to parse response as JSON.', { err });
46        return null;
47    }
48};
49
50const getReleasesMap = (releases: Array<Record<string, unknown>>): Record<string, string> => (
51    Object.fromEntries(releases.map((release) => ([release.id, release.name])))
52);
53
54const getTeamsMap = (listColumnItems: Array<Record<string, unknown>>): Record<string, string> => (
55    Object.fromEntries(listColumnItems.map((columnItem) => ([columnItem.id, columnItem.label])))
56);
57
58type FeatureItemsMap = {[p: string]: FeatureData};
59type SubfeatureItemsMap = {[p: string]: { id: string; parentId: string; title: string; timeline: string | null }};
60const getFeatureItemMaps = (features: Array<Record<string, unknown>>): {
61    featureItemsMap: FeatureItemsMap;
62    subfeatureItemsMap: SubfeatureItemsMap
63} => {
64    const allItems = Object.values(features);
65    const featureItemsMap = Object.fromEntries(allItems
66        .filter(isFeature)
67        .map((item) => (
68            [String(item.id),
69                { title: String(item.name), description: null, timeline: [] as string[], team: null, features: null }])),
70    );
71
72    const subfeatureItemsMap = Object.fromEntries(allItems
73        .filter(isSubfeature)
74        .map((item) => (
75            [String(item.id),
76                { id: String(item.id), title: String(item.name), parentId: String(item.parentId), timeline: null }])));
77    return {
78        featureItemsMap,
79        subfeatureItemsMap,
80    };
81};
82
83const addTeamsToFeatureMaps = (
84    { columnValues, teamsMap, featureItemsMap }:{
85        columnValues: Array<Record<string, unknown>>,
86        teamsMap: Record<string, string>,
87        featureItemsMap: FeatureItemsMap
88    },
89) => {
90    columnValues.forEach(({ value, featureId }) => {
91        if ((typeof featureId !== 'number' && typeof featureId !== 'string') || (typeof value !== 'number' && typeof value !== 'string')) {
92            log.warning('Invalid feature id or value in columnValues // skipped', { featureId, value });
93            return;
94        }
95        if (featureId in featureItemsMap) {
96            featureItemsMap[featureId].team = teamsMap[value];
97        }
98    });
99};
100
101const addReleasesToFeatureMaps = (
102    { releaseAssignments, releasesMap, featureItemsMap, subfeatureItemsMap }:{
103        releaseAssignments: Array<Record<string, unknown>>,
104        releasesMap: Record<string, string>,
105        featureItemsMap: FeatureItemsMap,
106        subfeatureItemsMap: SubfeatureItemsMap
107    },
108) => {
109    releaseAssignments.forEach(({ releaseId, featureId }) => {
110        if ((typeof featureId !== 'number' && typeof featureId !== 'string') || (typeof releaseId !== 'number' && typeof releaseId !== 'string')) {
111            log.warning('Invalid release or feature id in releaseAssignments // skipped', { releaseId, featureId });
112            return;
113        }
114        if (featureId in featureItemsMap) {
115            featureItemsMap[featureId].timeline = [...featureItemsMap[featureId].timeline, releasesMap[releaseId]];
116        }
117        if (featureId in subfeatureItemsMap) {
118            subfeatureItemsMap[featureId].timeline = releasesMap[releaseId];
119        }
120    });
121};
122
123const addSubfeaturesToFeatureMaps = (
124    { featureItemsMap, subfeatureItemsMap }:{
125        featureItemsMap: FeatureItemsMap,
126        subfeatureItemsMap: SubfeatureItemsMap
127    },
128) => {
129    Object.values(subfeatureItemsMap)
130        .forEach((item) => {
131            const { id, title, timeline, parentId } = item;
132
133            featureItemsMap[parentId].features = { ...featureItemsMap[parentId].features, [id]: { title, description: null, timeline } };
134        });
135};
136
137export const handleInitialRequest = async (response: RequiredResponse): Promise<Record<string, FeatureData> | null> => {
138    const { releases, features, listColumnItems, releaseAssignments, columnValues } = response;
139
140    const releasesMap = getReleasesMap(releases);
141    const teamsMap = getTeamsMap(listColumnItems);
142    const { featureItemsMap, subfeatureItemsMap } = getFeatureItemMaps(features);
143
144    addTeamsToFeatureMaps({ columnValues, teamsMap, featureItemsMap });
145    addReleasesToFeatureMaps({ releaseAssignments, releasesMap, featureItemsMap, subfeatureItemsMap });
146
147    addSubfeaturesToFeatureMaps({ featureItemsMap, subfeatureItemsMap });
148
149    return featureItemsMap;
150};
151
152const getFeatureDetail = async ({ featureId, cookieHeader }:{featureId: string, cookieHeader: string}) => (await axios.get(`https://apify.productboard.com/api/features/${featureId}`, { headers: { Cookie: cookieHeader } })).data.feature;
153
154export const getHandleDetailRequest = ({ cookieHeader, featureItemsMap }: { cookieHeader: string, featureItemsMap: Record<string, FeatureData>}) => (
155    async ({ featureId }: { featureId: string }): Promise<FeatureData> => {
156        const featureDetail = await getFeatureDetail({ featureId, cookieHeader });
157
158        const subfeatures = featureItemsMap[featureId].features;
159        if (subfeatures === null) {
160            return {
161                ...featureItemsMap[featureId],
162                description: featureDetail.description,
163            };
164        }
165        const subfeaturesWithDescription = Object.fromEntries(
166            await Promise.all(
167                Object.entries(subfeatures).map(async ([subfeatureId, subfeature]) => {
168                    const subfeatureDetail = await getFeatureDetail({ featureId: subfeatureId, cookieHeader });
169                    return [subfeatureId, { ...subfeature, description: subfeatureDetail.description }];
170                })));
171
172        return {
173            ...featureItemsMap[featureId],
174            description: featureDetail.description,
175            features: subfeaturesWithDescription,
176        };
177    });

src/main.ts

1import { Actor } from 'apify';
2import { chromium } from 'playwright';
3import { getHandleDetailRequest, parseRequiredResponse, handleInitialRequest, handleLogin } from './handlers.js';
4import { isRoadmapInitialRequest, STORE_ID } from './constants.js';
5import {FeatureData} from "./types.js";
6
7await Actor.init();
8
9// actor input
10interface Input {
11    productboardRoadmapUrl: string;
12    userEmail: string;
13    userPassword: string
14}
15const { productboardRoadmapUrl, userEmail, userPassword } = await Actor.getInput<Input>() ?? {};
16if (!productboardRoadmapUrl || !userEmail || !userPassword) {
17    throw new Error('At least one of the required actor inputs is missing.');
18}
19
20// actor store
21const keyValueStore = await Actor.openKeyValueStore(STORE_ID);
22
23// launch and login
24const browser = await chromium.launch({ headless: true });
25
26const page = await browser.newPage();
27await page.goto(productboardRoadmapUrl);
28await page.waitForLoadState('load');
29
30await handleLogin(page, { userEmail, userPassword });
31
32// catch initial request and get and store data
33await page.waitForResponse(async (response) => {
34    const isInitialRequest = isRoadmapInitialRequest({ requestUrl: response.request().url(), productboardRoadmapUrl });
35    if (!isInitialRequest) return false;
36
37    const requiredResponse = await parseRequiredResponse({ response });
38    if (!requiredResponse) return false;
39
40    const featureItemsMap = await handleInitialRequest(requiredResponse);
41    if (!featureItemsMap) return false;
42
43    const cookieHeader = (await response.request().allHeaders()).cookie;
44    const handleDetailRequest = getHandleDetailRequest({ cookieHeader, featureItemsMap });
45
46    await Promise.all(Object.keys(featureItemsMap).map(async (featureId) => {
47        const valueToStore = await handleDetailRequest({ featureId });
48        await keyValueStore.setValue(featureId, valueToStore);
49    }));
50    return true;
51});
52
53const output: Record<string, FeatureData | null> = {};
54await keyValueStore.forEachKey(async (key) => {
55    output[key] = await keyValueStore.getValue(key);
56});
57
58await Actor.setValue('OUTPUT', output);
59
60await browser.close();
61await Actor.exit();

src/types.ts

1export type SubfeatureData = {
2    title: string,
3    description: string | null,
4    timeline: string | null
5}
6
7export type FeatureData = {
8    title: string,
9    description: string | null,
10    timeline: string[],
11    team: string | null,
12    features: Record<string, SubfeatureData> | null
13}

src/utils.ts

1export const isObject = (test: unknown): test is Record<string, unknown> => test !== null && typeof test === 'object';
2
3export const isArrayOfObjects = (test: unknown): test is Array<Record<string, unknown>> => Array.isArray(test) && test.every(isObject);

Developer

Zuzana Štětinová

Actor Metrics

1 monthly user
1 star
Created in Mar 2024
Modified a month ago

Categories

Developer tools

Automation

Open source

Website Content Crawler

apify/website-content-crawler

Crawl websites and extract text content to feed AI models, LLM applications, vector databases, or RAG pipelines. The Actor supports rich formatting using Markdown, cleans the HTML, downloads files, and integrates well with 🦜🔗 LangChain, LlamaIndex, and the wider LLM ecosystem.

Apify

31.7k

854

Cheerio Scraper

apify/cheerio-scraper

Crawls websites using raw HTTP requests, parses the HTML with the Cheerio library, and extracts data from the pages using a Node.js code. Supports both recursive crawling and lists of URLs. This actor is a high-performance alternative to apify/web-scraper for websites that do not require JavaScript.

Apify

Web Scraper

apify/web-scraper

Crawls arbitrary websites using the Chrome browser and extracts data from pages using JavaScript code. The Actor supports both recursive crawling and lists of URLs and automatically manages concurrency for maximum performance. This is Apify's basic tool for web crawling and scraping.

Apify

73.7k

334

Puppeteer Scraper

apify/puppeteer-scraper

Crawls websites with the headless Chrome and Puppeteer library using a provided server-side Node.js code. This crawler is an alternative to apify/web-scraper that gives you finer control over the process. Supports both recursive crawling and list of URLs. Supports login to website.

Apify

Youtube Video Downloader

epctex/youtube-video-downloader

Effortlessly download YouTube videos of your preferred quality with our user-friendly Video Downloader. Try it now!

epctex

600

Tiktok Shop Scraper

excavator/tiktok-shop-scraper

This is the Actor for crawling data from the TikTok shop product URLs. For example: https://shop.tiktok.com/view/product/XXXXXXXXXX These URLs are only available for TikTok Shop US. You can test it here: https://apify.com/excavator/tiktok-shop-product

Excavator

Download HTML from URLs

mtrunkat/url-list-download-html

This actor takes a list of URLs and downloads HTML of each page.

Marek Trunkát

8.3k

🔥 LinkedIn Jobs Scraper

bebity/linkedin-jobs-scraper

ℹ️ Designed for both personal and professional use, simply enter your desired job title and location to receive a tailored list of job opportunities. Try it today!

Bebity

122

Rightmove Scraper

dhrumil/rightmove-scraper

Scrape rightmove.co.uk to crawl millions of sale/rent real estate properties from United Kingdom. Our real estate scraper also lets you monitor specific listing for new updates/listing. You can provide multiple search result listings to scrape/monitor.

Dhrumil Bhankhar

170

Instagram Followers Count Scraper

apify/instagram-followers-count-scraper

Scrape the number of followers & follows from any Instagram profile. Schedule the scraper to run regularly to monitor how the numbers change. You can also export scraped data, run the scraper via API, monitor runs or integrate with other tools.

Apify

3.4k