skynet-scrapper
Try for free
No credit card required
View all Actors
.actor/Dockerfile
1# Specify the base Docker image. You can read more about
2# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
3# You can also use any other image from Docker Hub.
4FROM apify/actor-node:20
5
6# Copy just package.json and package-lock.json
7# to speed up the build using Docker layer cache.
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --omit=dev --omit=optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --omit=dev --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version \
21 && rm -r ~/.npm
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after NPM install, quick build will be really fast
25# for most source file changes.
26COPY . ./
27
28
29# Run the image.
30CMD npm run start.actor/actor.json
1{
2 "actorSpecification": 1,
3 "name": "skinet-scraper",
4 "title": "skinet scraper",
5 "version": "1.0.0",
6 "input": "./input_schema.json",
7 "dockerfile": "./Dockerfile",
8 "storages": {
9 "dataset": "./dataset_schema.json"
10 }
11}.actor/dataset_schema.json
1{
2 "actorSpecification": 1,
3 "fields": {},
4 "views": {
5 "overview": {
6 "title": "Overview",
7 "transformation": {},
8 "display": {
9 "component": "table"
10 }
11 }
12 }
13}.actor/input_schema.json
1{
2 "title": "Scrape data from a web page",
3 "type": "object",
4 "schemaVersion": 1,
5 "properties": {
6 "queries": {
7 "title": "Search Query",
8 "type": "string",
9 "description": "Search query to use on Google",
10 "editor": "textfield",
11 "prefill": "[tow truck near me california]"
12 },
13 "maxRequestsPerCrawl": {
14 "title": "Max Requests per Crawl",
15 "type": "integer",
16 "description": "Maximum number of requests per crawl",
17 "editor": "number",
18 "prefill": 200
19 }
20 },
21 "required": ["queries"]
22}.actor/output_schema.json
1{
2 "actorSpecification": 1,
3 "name": "skynet-scraper",
4 "title": "skynet Scraper",
5 "description": "",
6 "version": "1.0.0",
7 "properties": {
8 "url": {
9 "type": "string",
10 "title": "URL",
11 "description": "URL to scrape",
12 "required": true
13 },
14 "title": {
15 "type": "string",
16 "title": "title",
17 "description": "title to scrape",
18 "required": true
19 },
20 "phoneNumber": {
21 "type": "string",
22 "title": "phoneNumber",
23 "description": "phoneNumber to scrape",
24 "required": true
25 }
26 },
27 "fields": {},
28 "views": {
29 "overview": {
30 "title": "Overview",
31 "transformation": {},
32 "display": {}
33 }
34 }
35
36}src/main.js
1import { Actor, Dataset } from "apify";
2import { CheerioCrawler } from "crawlee";
3import fs from 'fs';
4
5try {
6 await Actor.init();
7
8 const input = await Actor.getInput();
9 let { queries, maxRequestsPerCrawl } = input;
10 if (typeof queries === 'string') {
11 queries = queries.replace(/^\[|\]$/g, '');
12 queries = `[${queries}]`;
13
14}
15 const searchQueries = Array.isArray(queries) ? queries : [queries];
16 const searchQuery = searchQueries.join(" ");
17 const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`;
18 console.log("URL ", searchUrl);
19
20 const phoneNumberRegex = /(\d{3}[-.\s]??\d{3}[-.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-.\s]??\d{4}|\d{3}[-.\s]??\d{4})/;
21
22 const resultsData = {
23 searchQuery: {
24 term: queries,
25 url: searchUrl,
26 device: "MOBILE",
27 page: 1,
28 type: "SEARCH",
29 domain: "google.com",
30 countryCode: "US",
31 languageCode: "en",
32 locationUule: null,
33 resultsPerPage: 10
34 },
35 resultsTotal: "N/A",
36 relatedQueries: [],
37 paidResults: [],
38 paidProducts: [],
39 organicResults: [],
40 peopleAlsoAsk: []
41 };
42
43 const crawler = new CheerioCrawler({
44 maxRequestsPerCrawl,
45 handlePageFunction: async ({ request, response, $, log}) => {
46
47
48 const searchResults = $("div.g, div.uEierd"); // "div.uEierd" is used for some ad blocks
49 console.log("search results:", searchResults.length);
50
51 if (searchResults.length === 0) {
52 console.log("No search results were found.");
53 }
54
55 searchResults.each((index, element) => {
56 const $result = $(element);
57 let title = $result.find("h3").text().trim() || $result.find(".xA33Gc").text().trim(); // Adjusted selector
58 const url = $result.find("a").attr("href") || '';
59 //const description = $result.find('span.VwiC3b').text().trim() || '';
60 const textContent = $result.text();
61 const description = extractDescription(textContent);
62 let phoneNumber = null;
63
64 // search phonenumber - "Call us"
65 const callElements = $result.find('*:contains("Call us")');
66 callElements.each((i, callElement) => {
67 const callText = $(callElement).text();
68 const phoneMatch = callText.match(phoneNumberRegex);
69 if (phoneMatch) {
70 phoneNumber = phoneMatch[0];
71 }
72 });
73
74 // Identify if the result is sponsored
75 const isSponsored = $result.hasClass('uEierd') || $result.find('span').text().toLowerCase().includes('ad');
76
77 if ((title || url || description) && phoneNumber) {
78 if(!title) {
79 title = `${phoneNumber}`;
80 } else {
81 title = `${phoneNumber}`;
82 }
83
84 if (isSponsored) {
85 resultsData.paidResults.push({
86 title,
87 url,
88 phoneNumber,
89 displayedUrl: url,
90 description,
91 emphasizedKeywords: [],
92 siteLinks: [],
93 type: "paid",
94 adPosition: index + 1
95 });
96 } else {
97 resultsData.organicResults.push({
98 title,
99 url,
100 phoneNumber,
101 displayedUrl: url,
102 description,
103 emphasizedKeywords: [],
104 siteLinks: [],
105 productInfo: {},
106 type: "organic",
107 position: index + 1
108 });
109 }
110 } else {
111 console.log(`Resultado ${index + 1} está vacío o incompleto.`);
112 }
113 });
114
115 // People Also Ask
116 const peopleAlsoAskElements = $('div.related-question-pair');
117 if (peopleAlsoAskElements.length === 0) {
118 console.log("No se encontraron 'People Also Ask'.");
119 }
120
121 peopleAlsoAskElements.each((index, element) => {
122 const $question = $(element).find('.yuRUbf').text().trim();
123 const $answer = $(element).find('.VwiC3b').text().trim();
124 const $url = $(element).find('a').attr('href') || '';
125
126 if ($question || $answer || $url) {
127 resultsData.peopleAlsoAsk.push({
128 question: $question,
129 answer: $answer,
130 url: $url,
131 title: $question,
132 date: ''
133 });
134 } else {
135 console.log(`'People Also Ask' ${index + 1} está vacío o incompleto.`);
136 }
137 });
138
139 // Related Queries
140 const relatedQueriesElements = $('a[data-hveid="CAEQAw"]');
141 if (relatedQueriesElements.length === 0) {
142 console.log("No se encontraron 'Related Queries'.");
143 }
144
145 relatedQueriesElements.each((index, element) => {
146 const $relatedQuery = $(element).text().trim();
147 const $relatedUrl = $(element).attr('href') || '';
148
149 if ($relatedQuery || $relatedUrl) {
150 resultsData.relatedQueries.push({
151 title: $relatedQuery,
152 url: `https://www.google.com${$relatedUrl}`
153 });
154 } else {
155 console.log(`'Related Query' ${index + 1} está vacío o incompleto.`);
156 }
157 });
158
159 // Handling pagination: Check for the next page link
160 const nextPageLink = $('a#pnnext').attr('href');
161 if (nextPageLink && request.userData.page < 25) { // Limit to 25 pages for this example
162 await crawler.addRequests([{
163 url: `https://www.google.com${nextPageLink}`,
164 userData: { page: request.userData.page + 1 }
165 }]);
166 }
167 },
168 });
169
170 const extractDescription = (textContent) => {
171 // Elimina URLs y todo el texto antes de la URL
172 const cleanedText = textContent.replace(/https?:\/\/[^\s]+|\/\/[^\s]+/g, '').trim();
173
174 // Elimina cualquier texto al final que no sea necesario (por ejemplo, el texto después de '›' o ' · ')
175 const description = cleanedText.replace(/(?: ›| · | - |:| · )[^›]*$/, '').trim();
176
177 return description;
178 };
179
180 await crawler.run([{
181 url: searchUrl,
182 userData: { page: 1 }
183 }]);
184
185 if (!resultsData.paidResults.length && !resultsData.organicResults.length) {
186 console.log("No se encontraron resultados de búsqueda.");
187 }
188
189 // save result dataset of Apify
190 await Actor.setValue('OUTPUT', resultsData);
191 await Dataset.pushData(resultsData);
192
193 // write file JSON
194 fs.writeFileSync('./output.json', JSON.stringify(resultsData, null, 2));
195 console.log('Datos guardados en output.json');
196
197 await Actor.exit();
198} catch (error) {
199 console.error("Se produjo un error durante la ejecución del actor:", error);
200 process.exit(1);
201}.dockerignore
1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10node_modules
11
12# git folder
13.git.editorconfig
1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf.eslintrc
1{
2 "extends": "@apify",
3 "root": true
4}.gitignore
1# This file tells Git which files shouldn't be added to source control
2
3.DS_Store
4.idea
5dist
6node_modules
7apify_storage
8storagepackage.json
1{
2 "name": "my-web-scrapper",
3 "version": "0.0.1",
4 "type": "module",
5 "description": "This is an example of an Apify actor.",
6 "engines": {
7 "node": ">=18.0.0"
8 },
9 "dependencies": {
10 "@crawlee/http": "^3.9.2",
11 "apify": "^3.1.10",
12 "apify-client": "^2.9.3",
13 "axios": "^1.5.0",
14 "cheerio": "^1.0.0-rc.12",
15 "crawlee": "^3.9.2",
16 "random-useragent": "^0.5.0"
17 },
18 "scripts": {
19 "start": "node ./src/main.js",
20 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
21 },
22 "author": "It's not you it's me",
23 "license": "ISC"
24}Developer
tech simphonyMaintained by Community
Actor metrics
- 2 monthly users
- 1 star
- 100.0% runs succeeded
- Created in May 2024
- Modified about 1 month ago
Categories
