Proweblook Twitter X Scraper
Pricing
from $0.0035 / actor start
Pricing
from $0.0035 / actor start
Rating
0.0
(0)
Developer
chandan kumar dubey
Actor stats
0
Bookmarked
2
Total users
1
Monthly active users
9 days ago
Last modified
Categories
Share
node_modules/storage/.envapify_storage/1import { Actor } from 'apify';2
3// ProWebLook X (Twitter) Scraper — Apify Actor4// Wraps the ProWebLook Scraper API to extract X/Twitter data.5// 3 modes: search tweets, post/tweet lookup, comment/reply extraction.6
7const API_BASE = 'https://proweblook.com/api/v1';8
9await Actor.init();10
11try {12 // ── Read input ──────────────────────────────────────────────13 const input = await Actor.getInput() ?? {};14 const {15 mode = 'search',16 query = '',17 sort = 'new',18 time = 'week',19 maxItems = 25,20 postUrl = '',21 maxComments = 20,22 } = input;23
24 // ── Resolve API key from environment variable ───────────────25 const apiKey = process.env.PROWEBLOOK_API_KEY;26 if (!apiKey) {27 throw new Error(28 'Missing PROWEBLOOK_API_KEY environment variable. '29 + 'Set it in Apify Console → Actor → Settings → Environment Variables.'30 );31 }32
33 // ── Route to the right mode ─────────────────────────────────34 if (mode === 'search') {35 await handleSearch({ apiKey, query, sort, time, maxItems });36 } else if (mode === 'post') {37 await handlePostLookup({ apiKey, postUrl, maxComments });38 } else if (mode === 'comment') {39 await handleCommentExtraction({ apiKey, postUrl, maxComments });40 } else {41 throw new Error(`Unknown mode "${mode}". Use "search", "post", or "comment".`);42 }43
44} catch (error) {45 console.error('Actor failed:', error.message);46 await Actor.setStatusMessage(`Failed: ${error.message}`, { isStatusMessageTerminal: true });47 await Actor.fail(error.message);48}49
50await Actor.exit();51
52
53// ═══════════════════════════════════════════════════════════════54// Search Mode — find tweets by keyword55// ═══════════════════════════════════════════════════════════════56async function handleSearch({ apiKey, query, sort, time, maxItems }) {57 if (!query || !query.trim()) {58 throw new Error('Search query is required when mode is "search".');59 }60
61 await Actor.setStatusMessage(`Searching X for "${query}"...`);62 console.log(`[search] query="${query}" sort=${sort} time=${time} maxItems=${maxItems}`);63
64 const url = new URL(`${API_BASE}/scraperxsearch`);65 url.searchParams.set('api_key', apiKey);66 url.searchParams.set('query', query);67 url.searchParams.set('sort', sort);68 url.searchParams.set('time', time);69 url.searchParams.set('maxItems', String(Math.min(Math.max(Number(maxItems) || 25, 1), 50)));70
71 const response = await fetch(url.toString());72 const data = await response.json();73
74 if (!response.ok || data.status === false) {75 const errMsg = data.error || data.message || `API returned HTTP ${response.status}`;76 throw new Error(errMsg);77 }78
79 const items = data.items || [];80 if (items.length === 0) {81 console.log('[search] No results found.');82 await Actor.setStatusMessage('Done — no results found.', { isStatusMessageTerminal: true });83 return;84 }85
86 const cleanItems = items.map((item) => mapTweetItem(item, { mode: 'search', query }));87 await Actor.pushData(cleanItems);88
89 const msg = `Done! Found ${cleanItems.length} tweet(s) for "${query}".`;90 console.log(`[search] ${msg}`);91 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });92}93
94
95// ═══════════════════════════════════════════════════════════════96// Post Lookup Mode — fetch a single tweet by URL97// ═══════════════════════════════════════════════════════════════98async function handlePostLookup({ apiKey, postUrl, maxComments }) {99 if (!postUrl || !postUrl.trim()) {100 throw new Error('Post URL is required when mode is "post".');101 }102
103 await Actor.setStatusMessage(`Fetching tweet: ${postUrl}`);104 console.log(`[post] url="${postUrl}"`);105
106 const url = new URL(`${API_BASE}/scraperxpost`);107 url.searchParams.set('api_key', apiKey);108 url.searchParams.set('url', postUrl.trim());109 url.searchParams.set('maxComments', String(maxComments || 20));110
111 const response = await fetch(url.toString());112 const data = await response.json();113
114 if (!response.ok || data.status === false) {115 const errMsg = data.error || data.message || `API returned HTTP ${response.status}`;116 throw new Error(errMsg);117 }118
119 const items = data.items || [];120 if (items.length === 0) {121 console.log('[post] No data returned for this URL.');122 await Actor.setStatusMessage('Done — no data found for this tweet.', { isStatusMessageTerminal: true });123 return;124 }125
126 const cleanItems = items.map((item) => mapTweetItem(item, { mode: 'post', postUrl }));127 await Actor.pushData(cleanItems);128
129 const msg = `Done! Extracted ${cleanItems.length} item(s) from tweet.`;130 console.log(`[post] ${msg}`);131 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });132}133
134
135// ═══════════════════════════════════════════════════════════════136// Comment Mode — fetch replies/comments on a tweet137// ═══════════════════════════════════════════════════════════════138async function handleCommentExtraction({ apiKey, postUrl, maxComments }) {139 if (!postUrl || !postUrl.trim()) {140 throw new Error('Post URL is required when mode is "comment".');141 }142
143 await Actor.setStatusMessage(`Fetching comments for: ${postUrl}`);144 console.log(`[comment] url="${postUrl}" maxComments=${maxComments}`);145
146 const url = new URL(`${API_BASE}/scraperxcomment`);147 url.searchParams.set('api_key', apiKey);148 url.searchParams.set('url', postUrl.trim());149 url.searchParams.set('maxComments', String(maxComments || 20));150
151 const response = await fetch(url.toString());152 const data = await response.json();153
154 if (!response.ok || data.status === false) {155 const errMsg = data.error || data.message || `API returned HTTP ${response.status}`;156 throw new Error(errMsg);157 }158
159 const items = data.items || [];160 if (items.length === 0) {161 console.log('[comment] No comments found.');162 await Actor.setStatusMessage('Done — no comments found for this tweet.', { isStatusMessageTerminal: true });163 return;164 }165
166 const cleanItems = items.map((item) => mapTweetItem(item, { mode: 'comment', postUrl }));167 await Actor.pushData(cleanItems);168
169 const msg = `Done! Extracted ${cleanItems.length} comment(s) from tweet.`;170 console.log(`[comment] ${msg}`);171 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });172}173
174
175// ═══════════════════════════════════════════════════════════════176// Shared: map raw API item → clean dataset object177// ═══════════════════════════════════════════════════════════════178function mapTweetItem(item, meta = {}) {179 return {180 // ── Core fields ─────────────────────────────────────────181 tweetId: item.tweetId || item.id || item.postId || null,182 dataType: item.dataType || meta.mode || 'tweet',183 text: item.text || item.fullText || item.body || item.title || '',184 author: item.author || item.username || item.user || item.authorName || '',185 authorId: item.authorId || item.userId || null,186 url: item.url || item.tweetUrl || item.postUrl || item.permalink || '',187 isVerified: item.isVerified || item.verified || false,188
189 // ── Engagement metrics ──────────────────────────────────190 likes: item.likes ?? item.favoriteCount ?? item.score ?? 0,191 retweets: item.retweets ?? item.retweetCount ?? 0,192 replies: item.replies ?? item.replyCount ?? item.numComments ?? item.commentCount ?? 0,193 views: item.views ?? item.viewCount ?? 0,194 bookmarks: item.bookmarks ?? item.bookmarkCount ?? 0,195 quotes: item.quotes ?? item.quoteCount ?? 0,196
197 // ── Media ───────────────────────────────────────────────198 hasMedia: item.hasMedia || (item.media && item.media.length > 0) || false,199 mediaUrls: item.mediaUrls || item.media || [],200 hashtags: item.hashtags || [],201 mentions: item.mentions || item.userMentions || [],202
203 // ── Timestamps ──────────────────────────────────────────204 createdAt: item.createdAt || item.created_at || item.publishedAt || item.date || null,205
206 // ── Metadata ────────────────────────────────────────────207 scrapedMode: meta.mode || 'search',208 scrapedQuery: meta.query || meta.postUrl || '',209 scrapedAt: new Date().toISOString(),210 };211}{ "name": "proweblook-twitter-x-scraper", "version": "1.0.0", "description": "Apify Actor to scrape X (Twitter) — search tweets, extract post details, and fetch comments/replies via the ProWebLook API.", "type": "module", "main": "main.js", "scripts": { "start": "node main.js", "test": "echo \"No tests yet\" && exit 0" }, "dependencies": { "apify": "^3.5.0" }, "author": "ProWebLook <support@proweblook.com>", "license": "ISC", "keywords": [ "twitter", "x", "scraper", "twitter-scraper", "tweet-scraper", "social-media", "data-extraction", "proweblook" ], "homepage": "https://proweblook.com", "repository": { "type": "git", "url": "https://github.com/dubeychandan996/proweblook-twitter-x-scraper.git" }}FROM apify/actor-node:22
COPY package*.json ./RUN npm install --omit=dev --omit=optional \ && npm cache clean --force
COPY . ./
CMD ["npm", "start"]{ "title": "ProWebLook X (Twitter) Scraper", "description": "Search tweets, extract post details, and fetch comments/replies from X (Twitter).", "type": "object", "schemaVersion": 1, "required": ["mode"], "properties": { "mode": { "title": "Mode", "description": "Choose what to scrape: search tweets, look up a specific post, or extract comments.", "type": "string", "enum": ["search", "post", "comment"], "enumTitles": ["Search Tweets", "Post / Tweet Lookup", "Get Comments / Replies"], "default": "search", "editor": "select" }, "query": { "title": "Search Query", "description": "Keywords to search for on X. Required when mode is 'search'.", "type": "string", "prefill": "AI tools", "editor": "textfield", "sectionCaption": "Search Settings", "sectionDescription": "Configure tweet search parameters (used when mode is 'search')." }, "sort": { "title": "Sort Order", "description": "How to sort search results.", "type": "string", "enum": ["new", "hot", "top", "relevance"], "enumTitles": ["New (Latest)", "Hot", "Top", "Relevance"], "default": "new", "editor": "select" }, "time": { "title": "Time Range", "description": "Filter results by time period.", "type": "string", "enum": ["day", "week", "month", "year", "all"], "enumTitles": ["Past Day", "Past Week", "Past Month", "Past Year", "All Time"], "default": "week", "editor": "select" }, "maxItems": { "title": "Max Results", "description": "Maximum number of items to return (1–50).", "type": "integer", "minimum": 1, "maximum": 50, "default": 25, "editor": "number" }, "postUrl": { "title": "Tweet / Post URL", "description": "Full URL of a tweet on X (e.g. https://x.com/username/status/1234567890). Required when mode is 'post' or 'comment'.", "type": "string", "editor": "textfield", "sectionCaption": "Post / Comment Lookup", "sectionDescription": "Provide a tweet URL for post lookup or comment extraction." }, "maxComments": { "title": "Max Comments", "description": "Maximum number of comments/replies to fetch (used in 'comment' and 'post' modes).", "type": "integer", "minimum": 1, "maximum": 50, "default": 20, "editor": "number" } }}{ "actorSpecification": 1, "name": "proweblook-twitter-x-scraper", "title": "ProWebLook X (Twitter) Scraper", "version": "1.0", "buildTag": "latest", "defaultMemoryMbytes": 256, "minMemoryMbytes": 128, "maxMemoryMbytes": 512, "dockerfile": "./Dockerfile", "readme": "../README.md", "input": "./INPUT_SCHEMA.json"}