# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
crawlee_storage
storage
.DS_Store
INPUT.json


.gitignore

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node:16

# Copy only built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm run start:prod --silent


Dockerfile

Text which you would Craiyon to generate images for, like `Cats`, `Wood buildings`

Enter a text of what images you would like Craiyon to generate, you can add multiple values.

Search Text

Craiyon images crawler input

INPUT_SCHEMA.json

{
	"name": "craiyon-crawler",
	"version": "1.0",
	"buildTag": "latest",
	"env": null
}


apify.json

craiyon-crawler

package.json

tsconfig.json

// For more information, see https://crawlee.dev/
import { HttpCrawler, KeyValueStore, log } from "crawlee";
import { Actor } from "apify";

import { router } from "./routes.js";

await Actor.init();

interface InputSchema {
    debug?: boolean;
    searchStrings: string[];
}

let debug, searchStrings;

const input = await KeyValueStore.getInput<InputSchema>();

debug = input?.debug;
searchStrings = input?.searchStrings;

// Get input from Apify console
if (!searchStrings) {
    const input = await Actor.getInput<InputSchema>();

    debug = input?.debug;
    searchStrings = input?.searchStrings;
}

if (!(Array.isArray(searchStrings) && searchStrings.length > 0)) {
    throw new Error('Wrong INPUT: searchStrings has to be an array with at least one text');
}

if (debug) {
    log.setLevel(log.LEVELS.DEBUG);
}

const proxyConfiguration = await Actor.createProxyConfiguration({
    useApifyProxy: true,
});

const crawler = new HttpCrawler({
    proxyConfiguration,
    navigationTimeoutSecs: 60 * 5, // Takes a while to generate images
    requestHandler: router,
    errorHandler: async ({ response }) => {
        log.error(`Response status is: ${response?.statusCode} msg: ${response?.statusMessage}`);
    },
});

await crawler.run(
    searchStrings.map(searchString => {
        log.info(`Generating images for ${searchString}...`);

        return {
            url: "https://backend.craiyon.com/generate",
            method: "POST",
            payload: JSON.stringify({ prompt: searchString.toLowerCase() }),
            headers: {
                "content-type": "application/json",
            },
            useExtendedUniqueKey: true,
            userData: {
                searchString
            }
        }
    })
);

await Actor.exit();


src/main.ts

import { KeyValueStore, Dataset, createHttpRouter } from "crawlee";
import { v4 as uuid } from "uuid";

export const router = createHttpRouter();

router.addDefaultHandler(async ({ request, log, json }) => {
    const { userData: { searchString } } = request;

    if (!json || !json?.images || !json?.images?.length) {
        throw new Error (`Couldn't generate images`);
    }

    const { images } = json;

    log.info(`Successfully generated ${images.length} images for ${searchString}`);

    const keyValueStore = await KeyValueStore.open();

    for (const image of images) {
        const key = uuid();
        const imageBuffer = Buffer.from(image, "base64");

        await keyValueStore.setValue(key, imageBuffer, {
            contentType: "image/png",
        });

        const imageUrl = `https://api.apify.com/v2/key-value-stores/${keyValueStore.id}/records/${key}`;

        await Dataset.pushData({ searchString, imageUrl });
    }
});


src/routes.ts

Get links to images generated by Craiyon (formerly DALL·E mini) - an AI model that can draw images from any text prompt. Input any combination of text and get links to AI-generated images created by the engine. Export URLs of these images into a database in formats such as HTML, JSON, CSV, Excel.

Craiyon Crawler

Craiyon Crawler · Apify

Input · Craiyon Crawler · Apify

API · Craiyon Crawler · Apify

Source code · Craiyon Crawler · Apify

Related Actors · Craiyon Crawler · Apify

Get links to images generated by Craiyon (formerly DALL·E mini) - an AI model that can draw images from any text prompt. Input any combination of text and get links to AI-generated images created b...

Where next?