CRAWLEE_DISABLE_BROWSER_SANDBOX

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules


.dockerignore

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
crawlee_storage
storage


.gitignore

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16 AS builder

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package.json yarn.lock ./

# Install all dependencies. Don't audit to speed up the installation.
RUN yarn --production=false

# Next, copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN yarn build

# Create final image
FROM apify/actor-node-puppeteer-chrome:16

# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json yarn.lock ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN yarn --prod \
    && echo "Installed NPM packages:" \
    && (yarn list --depth 1 || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "Yarn version:" \
    && yarn --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
# CMD ./start_xvfb_and_run_cmd.sh && yarn start:prod --silent
CMD yarn start:prod --silent

Dockerfile

redfin-scraper

apify.json

redfin

package.json

tsconfig.json

instagram-accounts-scraper

Instagram Accounts Scraper

.actor/actor.json

Search for accounts with names containing the provided term

Search term

Maximum number of accounts to scrape (100 by default)

Max accounts

Proxy Configuration

Starter Crawler

.actor/input.json

Overview

.actor/output.json

export const GOOGLE_SEARCH_URL = 'https://www.google.com/search';
export const GOOGLE_SEARCH_ACCOUNTS_FILTER = 'site:https://www.instagram.com/ -inurl:/p/ -inurl:/reel/ -inurl:/explore/';
export const GOOGLE_SEARCH_RESULTS_PER_PAGE = 10;

// Matches urls only with the following pattern: https://www.instagram.com/[username]/
export const ACCOUNT_URL_REGEX = /^https:\/\/www\.instagram\.com\/([^/]+)\/$/;

export const X_IG_APP_ID_HEADER = '936619743392459';
export const INSTAGRAM_ACCOUNT_DATA_URL = 'https://www.instagram.com/api/v1/users/web_profile_info';


src/constants.ts

import { Actor } from 'apify';
import { CheerioCrawler } from 'crawlee';

import { router } from './routes';
import { Input } from './types';
import { buildGoogleSearchURL } from './utils';

Actor.main(async () => {
  const input: Input | null = await Actor.getInput<Input>();

  if (!input) {
    throw new Error("No input provided");
  }

  const proxyConfiguration = await Actor.createProxyConfiguration(
    input?.proxyConfiguration
  );

  const crawler = new CheerioCrawler({
    requestHandler: router,
    proxyConfiguration,
  });

  const pageNumber = 1;

  await crawler.run([
    {
      url: buildGoogleSearchURL(input.keyword, pageNumber),
      userData: {
        keyword: input.keyword,
        pageNumber
      }
    }
  ]);
});


src/main.ts

import { Actor, log } from 'apify';
import { Dataset, RequestQueue, createCheerioRouter } from 'crawlee';

import { ACCOUNT_URL_REGEX, X_IG_APP_ID_HEADER } from './constants';
import { Input } from './types';
import { buildGoogleSearchURL, buildInstagramAccountUrl } from './utils';

enum Label {
  ACCOUNT_PAGE = 'ACCOUNT_PAGE'
}

export const router = createCheerioRouter();

router.addDefaultHandler(async ({ request, $ }) => {
  const { keyword, pageNumber } = request.userData;

  const urls = $('a[href^="https://www.instagram.com/"]').map((_, el) => $(el).attr('href')).get();
  const usernames = urls
    .map((url) => ACCOUNT_URL_REGEX.exec(url))
    .filter(match => match)
    .map(match => match![1]);

  const queue = await RequestQueue.open();
  await Promise.all(
    usernames.map(username => queue.addRequest({
      url: buildInstagramAccountUrl(username),
      headers: { 'x-ig-app-id': X_IG_APP_ID_HEADER },
      label: Label.ACCOUNT_PAGE
    }))
  );

  queue.addRequest({
    url: buildGoogleSearchURL(keyword, pageNumber + 1),
    userData: {
      keyword,
      pageNumber: pageNumber + 1
    }
  });
});

router.addHandler(Label.ACCOUNT_PAGE, async ({ json }) => {
  const input = await Actor.getInput<Input>();
  const state = await Actor.useState('totalAccounts', { totalAccounts: 0 });

  const maxAccounts = input?.maxAccounts || 100;

  if (json?.data?.user) {
    const username = json.data.user.username;
    const avatarUrl = json.data.user.profile_pic_url;
    const name = json.data.user.full_name;

    await Dataset.pushData({
      username,
      avatarUrl,
      name,
      link: `https://www.instagram.com/${username}`
    });

    state.totalAccounts = state.totalAccounts + 1;
    log.info(`Collected ${state.totalAccounts}/${maxAccounts} accounts`);
    if (state.totalAccounts > maxAccounts) {
      await Actor.exit('Collected maximum number of accounts');
    }
  }
});


src/routes.ts

import { ProxyConfigurationOptions } from 'apify';

export type Input = {
  maxAccounts?: number;
  keyword: string;
  proxyConfiguration: ProxyConfigurationOptions;
};


src/types.ts

import {
  GOOGLE_SEARCH_ACCOUNTS_FILTER,
  GOOGLE_SEARCH_RESULTS_PER_PAGE,
  GOOGLE_SEARCH_URL,
  INSTAGRAM_ACCOUNT_DATA_URL
} from "./constants";

export const buildGoogleSearchURL = (keyword: string, page: number): string => {
  const url = new URL(GOOGLE_SEARCH_URL);

  url.searchParams.set('hl', 'en');
  url.searchParams.set('q', `${GOOGLE_SEARCH_ACCOUNTS_FILTER} ${keyword}`);
  url.searchParams.set('start', String((page - 1) * GOOGLE_SEARCH_RESULTS_PER_PAGE));

  return url.toString();
};

export const buildInstagramAccountUrl = (username: string): string => {
  const url = new URL(INSTAGRAM_ACCOUNT_DATA_URL);
  url.searchParams.set('username', username);
  return url.toString();
};


src/utils.ts

The Redfin scraper is an Apify-powered actor designed to collect data on real estate sales and rental prices in your target market efficiently.

Redfin Scraper

Where next?