SoftwareAdviceProductNameSearch

undrtkr984/softwareadviceproductnamesearch

No credit card required

Dockerfile

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Second, copy just package.json and package-lock.json since those are the only
# files that affect "npm install" in the next step, to speed up the build.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start

INPUT_SCHEMA.json

{
    "title": "Input Schema",
    "description": "To update crawler to another site, you need to change startUrls options!",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "A static list of URLs to scrape. <br><br>For details, see <a href='https://apify.com/apify/web-scraper#start-urls' target='_blank' rel='noopener'>Start URLs</a> in README.",
            "prefill": [{ "url": "https://crawlee.dev" }],
            "editor": "requestListSources"
        },
        "maxRequestRetries": {
            "title": "maxRequestRetries",
            "type": "integer",
            "description": "Max Request Retries",
            "prefill": 2,
            "default": 2,
            "editor": "number"
        }
    },
    "required": ["startUrls"]
}

main.js

// This is the main Node.js source code file of your actor.
// It is referenced from the "scripts" section of the package.json file.

const Apify = require('apify');

Apify.main(async () => {
    // Get input of the actor. Input fields can be modified in INPUT_SCHEMA.json file.
    // For more information, see https://docs.apify.com/actors/development/input-schema
    const input = await Apify.getInput();
    console.log('Input:');
    console.dir(input);

    // Here you can prepare your input for actor apify/web-scraper this input is based on a actor
    // task you used as the starting point.
    const metamorphInput = {
        "breakpointLocation": "NONE",
        "browserLog": false,
        "debugLog": false,
        "downloadCss": true,
        "downloadMedia": true,
        "ignoreCorsAndCsp": false,
        "ignoreSslErrors": false,
        "injectJQuery": true,
        "keepUrlFragments": false,
        "maxRequestRetries": input.maxRequestRetries,
        "pageFunction": // The function accepts a single argument: the "context" object.
        // For a complete list of its properties and functions,
        // see https://apify.com/apify/web-scraper#page-function 
        async function pageFunction(context) {
            // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
            // debugger; 
            // jQuery is handy for finding DOM elements and extracting data from them.
            // To use it, make sure to enable the "Inject jQuery" option.
            const $ = context.jQuery;
            const search = "#SearchProductCardComponent";
            let timeoutMillis; // undefined
            await context.waitFor(2000);
                // 2 sec timeout after the first.
            
            const products = document.querySelectorAll(search);
        
        
            var results = [];
        
            for (let i = 0; i < products.length-1; i++ )
            {
             
                var logoURL = "";
                var url = "";
                var name = "";
                var category = "";
                try{logoURL = products[i].getElementsByTagName('img')[0].src;}
                catch(error){}
                try{url = products[i].childNodes[1].childNodes[0].href;}
                catch(error){}
                try{name = products[i].childNodes[1].childNodes[0].text;}
                catch(error){}
                try{category = products[i].childNodes[1].childNodes[1].text;}
                catch(error){}
                results.push({"name":name,"url":url,"logoURL":logoURL,"category":category})
                
            }
            
        
            // Print some information to actor log
            
        
            // Manually add a new page to the queue for scraping
        
            // Return an object with the data extracted from the page.
            // It will be stored to the resulting dataset.
            return {
                results: results
            };
        },
        "postNavigationHooks": `// We need to return array of (possibly async) functions here.
            // The functions accept a single argument: the "crawlingContext" object.
            [
                async (crawlingContext) => {
                    // ...
                },
            ]`,
        "preNavigationHooks": `// We need to return array of (possibly async) functions here.
            // The functions accept two arguments: the "crawlingContext" object
            // and "gotoOptions".
            [
                async (crawlingContext, gotoOptions) => {
                    // ...
                },
            ]`,
        "proxyConfiguration": {
            "useApifyProxy": true,
            "apifyProxyCountry": "US"
        },
        "startUrls": input.startUrls,
        "runMode": "PRODUCTION",
        "useChrome": false,
        "waitUntil": [
            "networkidle2"
        ]
    };

    // Now let's metamorph into actor apify/web-scraper using the created input.
    await Apify.metamorph('apify/web-scraper', metamorphInput);
});

package.json

{
    "name": "my-actor",
    "version": "0.0.1",
    "dependencies": {
        "apify": "^2.2.2"
    },
    "scripts": {
        "start": "node main.js"
    },
    "author": "Me!"
}