root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules


.gitignore

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

Proxy configuration

Start URLs

Canva templates scraper

INPUT_SCHEMA.json

{
     "env": { "npm_config_loglevel": "silent" }
}


apify.json

/* eslint-disable no-nested-ternary */
const Apify = require('apify');
const { v4: uuidv4 } = require('uuid');

const { utils: { log, sleep } } = Apify;

const TEMPLATE_MAX = 5000;

async function extract(json) {
    const pageUrl = `https://www.canva.com${json.page.DI}`;
    if (crawledTemplateUrls.includes(pageUrl)) {
        return;
    }

    crawledTemplateUrls.push(pageUrl);
    log.info(pageUrl);

    let size = null;
    let colorList = null;
    let fontList = null;
    // eslint-disable-next-line no-underscore-dangle
    const designDetails = json.page.H_ ? json.page.H_.C : [];
    for (const item of designDetails) {
        if (item.A === 'Colors') {
            colorList = [];
            for (const color of item.U) {
                colorList.push(color.A);
            }
        } else if (item.A === 'Fonts') {
            fontList = [];
            for (const font of item.U) {
                fontList.push(font);
            }
        } else {
            size = item;
        }
    }

    const categoryId = json.page.H5[0].P ? json.page.H5[0].P : null;
    const templateId = json.page.H5[0].K;
    const typeId = json.page.H5[0].Q;

    const frontImageObj = json.preloadImages[0];
    const frontImageUrl = frontImageObj.X ? (frontImageObj.X[0] ? frontImageObj.X[0].B : frontImageObj.X.V[0].B) : frontImageObj.V[0].B;
    const frontThumbImageUrl = frontImageObj.X ? (frontImageObj.X[1] ? frontImageObj.X[1].B : frontImageObj.X.V[1].B) : frontImageObj.V[1].B;

    const backImageObj = json.preloadImages.length > 1 ? json.preloadImages[1] : null;
    let backImageUrl = null;
    let backThumbImageUrl = null;

    if (backImageObj) {
        backImageUrl = backImageObj.X ? (backImageObj.X[0] ? backImageObj.X[0].B : backImageObj.X.V[0].B) : backImageObj.V[0].B;
        backThumbImageUrl = backImageObj.X ? (backImageObj.X[1] ? backImageObj.X[1].B : backImageObj.X.V[1].B) : backImageObj.V[1].B;
    }

    await Apify.pushData({
        url: pageUrl,
        name: json.page.H1.A,
        colorList,
        fontList,
        size: size ? size.U : null,
        categoryId,
        templateId,
        typeId,
        frontImageUrl,
        frontThumbImageUrl,
        backImageUrl,
        backThumbImageUrl,
    });
}

async function handleList(page, requestQueue) {
    await page.setRequestInterception(true);
    page.on('request', async (req) => {
        await req.continue();
    });

    page.on('response', async (response) => {
        const responseUrl = response.url();
        if (!responseUrl.includes('/_ajax/marketplace2/page/token')) return;
        try {
            const data = await response.text();
            const jsonText = data.substring(data.indexOf('{'), data.lastIndexOf('}') + 1);
            const json = JSON.parse(jsonText);
            await extract(json);
        } catch (e) {
            log.exception(e, responseUrl);
            await requestQueue.addRequest({
                url: responseUrl,
                userData: { label: 'DETAIL' },
            });
        }
    });

    let pageUrl = await page.url();
    let retries = 3;
    const list = await page.$$('a[href*="/templates/"] > p');

    if (list.length === 0) {
        throw Error('No templates found! Retrying..');
    }

    for (let index = 0; index < list.length && index < TEMPLATE_MAX; index++) {
        await page.evaluate((el) => {
            el.parentElement.click();
        }, list[index]);

        await sleep(1000);
        while (pageUrl === (await page.url()) && retries-- > 0) {
            await sleep(1000);
        }

        pageUrl = await page.url();
        retries = 3;
    }
}

const getAttribute = async (element, attr, fallback = '') => {
    try {
        const prop = await element.getProperty(attr);
        return (await prop.jsonValue()).trim();
    } catch (e) {
        return fallback;
    }
};

let crawledTemplateUrls = [];
Apify.events.on('migrating', async () => {
    await Apify.setValue('crawledTemplateUrls', crawledTemplateUrls);
});

Apify.main(async () => {
    const { startUrls, proxyConfig } = await Apify.getInput();
    const requestList = await Apify.openRequestList('start-urls', startUrls);
    const requestQueue = await Apify.openRequestQueue();
    const proxyConfiguration = await Apify.createProxyConfiguration(proxyConfig);

    crawledTemplateUrls = (await Apify.getValue('crawledTemplateUrls')) || [];

    const crawler = new Apify.PuppeteerCrawler({
        requestList,
        requestQueue,
        handlePageTimeoutSecs: 3600,
        navigationTimeoutSecs: 120,
        proxyConfiguration,
        launchContext: {
            useChrome: false,
            stealth: false,
            launchOptions: {
                headless: false,
                useIncognitoPages: true,
            },
        },
        useSessionPool: true,
        handlePageFunction: async (context) => {
            const { page } = context;
            const { url, userData: { label } } = context.request;
            log.info('Page opened.', { label, url });

            if (label === 'DETAIL') {
                const data = await page.content();
                const jsonText = data.substring(data.indexOf('{'), data.lastIndexOf('}') + 1);
                let json = null;
                try {
                    json = JSON.parse(jsonText);
                } catch (e) {
                    await Apify.setValue(uuidv4(), jsonText, { contentType: 'application/json' });
                    throw Error('Something wrong! Retrying..');
                }

                await extract(json);
            } else if (label === 'LIST') {
                await handleList(page, requestQueue);
            } else {
                await sleep(2000);
                const totalEl = (await page.$x('//section//p[contains(., " templates")]'))[0];
                const totalText = totalEl ? await getAttribute(totalEl, 'textContent') : null;
                log.info(totalText);

                const total = parseInt(totalText.match(/[\d,]+/)[0].split(',').join(''), 10);

                const pageTotal = Math.ceil(total / 50.0);
                log.info(`Pages: ${pageTotal}`);

                for (let index = 1; index < pageTotal; index++) {
                    await requestQueue.addRequest({
                        url: `${url}?continuation=${index * 50}`,
                        userData: { label: 'LIST' },
                    });
                }

                await handleList(page, requestQueue);
            }
        },

        preNavigationHooks: [
            async (crawlingContext, gotoOptions) => {
                const { page } = crawlingContext;
                await page.viewport({
                    width: 1920,
                    height: 1080,
                });

                await Apify.utils.puppeteer.blockRequests(page, {
                    urlPatterns: ['.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.webm', '.webp', 'data:image/'],
                });

                gotoOptions.waitUntil = 'networkidle0';
            },
        ],
    });

    log.info('Starting the crawl.');
    await crawler.run();
    log.info('Crawl finished.');
});


Canva Templates Scraper

Where next?