crawlee_storage

crawlee_storage/key_value_stores

import { Actor } from 'apify';
import { BasicCrawler, log } from 'crawlee';
import { gotScraping } from 'got-scraping';
import FormData from 'form-data';
import { JSDOM } from 'jsdom';
import { createHash } from 'crypto';

const createValidKVNameFromUrl = (url: string) => createHash('sha256').update(url).digest('hex').slice(0, -2);

const headers: Record<string, string> = {
    'Alt-Used': 'www.google.com',
    'Origin': 'https://www.google.com',
    'Referer': 'https://www.google.com/',
    'Cache-Control': 'no-cache',
    'Pragma': "no-cache",
};

await Actor.init();

interface InputSchema {
    startUrls: string[];
    debug?: boolean;
}

const { startUrls = [], debug, } = await Actor.getInput<InputSchema>() ?? {};

if (debug) {
    log.setLevel(log.LEVELS.DEBUG);
}

const proxyConfiguration = await Actor.createProxyConfiguration({
    groups: ['RESIDENTIAL'],
    countryCode: 'US'
});

const newUrl = (imageUrl: string) => {
    const nUrl = new URL('https://www.google.com.br/searchbyimage');

    nUrl.searchParams.set('image_url', imageUrl);
    nUrl.searchParams.set('btnG', 'Search by image');
    nUrl.searchParams.set('encoded_image', '');
    nUrl.searchParams.set('image_content', '');
    nUrl.searchParams.set('filename', '');
    nUrl.searchParams.set('hl', 'en');

    return nUrl.toString();
}

const { defaultKeyValueStoreId } = Actor.getEnv();

const crawler = new BasicCrawler({
    maxConcurrency: 3,
    useSessionPool: true,
    async requestHandler({ session, request }) {
        const { userData } = request;

        const response = await gotScraping({
            url: newUrl(request.url),
            method: 'GET',
            proxyUrl: await proxyConfiguration!.newUrl(session!.id),
            headers,
            responseType: 'text',
        } as any);

        const { window } = new JSDOM(response.body, {
            url: 'https://www.google.com/search/',
            runScripts: 'dangerously',
            pretendToBeVisual: true,
        });

        const { document } = window;

        if (!document.querySelectorAll('#rso').length) {
            throw new Error(`No results found`);
        }

        const relatedSearch = document.querySelector<HTMLAnchorElement>('#topstuff a.fKDtNb[href^="/search"]')?.href;

        const matches = Array.from<HTMLDivElement>(document.querySelectorAll<HTMLDivElement>('#search .normal-header ~ .g')).map((div) => {
            const [info, snippet] = Array.from(div.querySelectorAll('[data-content-feature="1"] > div > span'));
            const date = info?.querySelector?.('span:nth-child(3)')?.textContent ?? null;
            
            return {
                title: div?.querySelector?.('h3')?.textContent,
                url: div?.querySelector?.('a')?.href,
                date: /\d, \d/.test(date) ? date : null,
                text: snippet?.textContent || null,
            };
        });

        const results = Array.from<HTMLDivElement>(document.querySelectorAll('#rso > div:first-child .g')).map((div) => {
            const [info, snippet] = Array.from(div.querySelectorAll('[data-content-feature="1"] > div > span'));

            return {
                title: div?.querySelector?.('h3')?.textContent,
                url: div?.querySelector?.('a')?.href,
                date: /\d, \d/.test(info?.textContent) ? info.querySelector('span')?.textContent : null,
                text: snippet?.textContent || null,
            };
        });   

        const images = Array.from<HTMLDivElement>(document.querySelectorAll('[data-lpage]')).map((div) => {
            return {
                image: '',
                url: div?.dataset?.['lpage'],
                imageData: div?.querySelector?.('img')?.src,
            };
        }); 

        for (const image of images) {
            if (!image.imageData || !image.url) {
                continue;
            }

            const hash = createValidKVNameFromUrl(image.url);
            const [, contentType, imageData] = image.imageData.split(/data:|;base64,/);

            await Actor.setValue(
                hash,
                Buffer.from(imageData, 'base64'),
                { contentType }
            );

            delete image.imageData;
            image.image = `https://api.apify.com/v2/key-value-stores/${defaultKeyValueStoreId}/records/${hash}`;
        } 

        await Actor.pushData({
            relatedSearch: relatedSearch || null,
            matches,
            results,
            images,
            userData,
        });

        // await Actor.setValue('OUTPUT', dom.serialize(), { contentType: 'text/plain' });
    },
});

await crawler.addRequests(startUrls);

log.info('Starting the crawl.');
await crawler.run();
log.info('Crawl finished.');

await Actor.exit();


src/main.ts

crawlee_storage/key_value_stores/default

{
    "startUrls": [
        "https://apify.com"
    ]
}


crawlee_storage/key_value_stores/default/INPUT.json

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
crawlee_storage


.gitignore

# using multistage build, as we need dev deps to build the TS source code
FROM apify/actor-node:18-beta AS builder

# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
RUN npm install --include=dev \
    && npm run build

# create final image
FROM apify/actor-node:18-beta
# copy only necessary files
COPY --from=builder /usr/src/app/package.json ./
COPY --from=builder /usr/src/app/README.md ./
COPY --from=builder /usr/src/app/dist ./dist
COPY --from=builder /usr/src/app/apify.json ./apify.json
COPY --from=builder /usr/src/app/INPUT_SCHEMA.json ./INPUT_SCHEMA.json

# install only prod deps
RUN npm --quiet set progress=false \
    && npm install --only=prod --no-optional \
    && echo "Installed NPM packages:" \
    && (npm list --only=prod --no-optional --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version

# run compiled code
CMD npm run start:prod


Dockerfile

Debug mode

Image URLs

@crawlee/cheerio typescript template

INPUT_SCHEMA.json

# Google Search By Image

Search Google results by providing an image. 

It can be used for:

* Reverse search
* Finding people, objects, product names
* Find social media 
* Related images and terms
* Identify objects on photos

## Custom data

You can add custom data to each request by using the `userData` property on the request input:

```json
{
    "startUrls": [{
        "url": "...",
        "userData": {
            "myCustom": "data"
        }
    }]
}
```

## Proxy

This actor requires proxies to run



README.md

crawlee-cheerio-typescript

Crawlee + Cheerio + Typescript

apify.json

package.json

-----------------------------324172699818326226173603017879
Content-Disposition: form-data; name="image_url"


-----------------------------324172699818326226173603017879
Content-Disposition: form-data; name="encoded_image"; filename=""
Content-Type: application/octet-stream


-----------------------------324172699818326226173603017879
Content-Disposition: form-data; name="image_content"

base64
-----------------------------324172699818326226173603017879
Content-Disposition: form-data; name="filename"

x.jpg
-----------------------------324172699818326226173603017879
Content-Disposition: form-data; name="hl"

en-BR
-----------------------------324172699818326226173603017879--


payload

tsconfig.json

Search Google results by providing image URLs. It can be used for: reverse search, finding people by photo, detecting objects on images, getting product names, finding social media related to the person in the photo, finding related images and terms, and identifying objects on photos

Steve Jobs

google-search-by-image

Google Search By Image

Search Google results by providing image URLs. It can be used for: reverse search, finding people by photo, detecting objects on images, getting product names, finding social media related to the p...

Where next?