CRAWLEE_DISABLE_BROWSER_SANDBOX

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules


.dockerignore

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
crawlee_storage
storage


.gitignore

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16 AS builder

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package.json yarn.lock ./

# Install all dependencies. Don't audit to speed up the installation.
RUN yarn --production=false

# Next, copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN yarn build

# Create final image
FROM apify/actor-node-puppeteer-chrome:16

# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json yarn.lock ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN yarn --prod \
    && echo "Installed NPM packages:" \
    && (yarn list --depth 1 || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "Yarn version:" \
    && yarn --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && yarn start:prod --silent

Dockerfile

alibaba-scraper

apify.json

alibaba

package.json

tsconfig.json

Alibaba scraper

.actor/actor.json

Max Items

Proxy Configuration

Start URLs

Alibaba Scraper

.actor/input.json

Overview

.actor/output.json

import { Actor } from 'apify';
import { PuppeteerCrawler } from 'crawlee';

import { defaultRequestHandler, getStartUrlsArray } from './routes';

const disallowedDomains = [
  'cookielaw.org',
  'cdn.cookielaw.org',
  'googletagmanager.com',
  'google-analytics.com',
  'connect.facebook.netsa',
  'cdn.cookielaw.org',
  'analytics.tiktok.com',
  'bat.bing.com',
  'accounts.google.com',
  'facebook.com',
  'adservice.google.com',
  'gj.mmstat.com',
  'img.alicdn.com',
];

Actor.main(async () => {
  const input: any = await Actor.getInput();

  const proxyConfiguration = await Actor.createProxyConfiguration(
    input.proxyConfiguration
  );
  const startUrls: any = getStartUrlsArray(input.startUrls);

  const crawler = new PuppeteerCrawler({
    requestHandler: defaultRequestHandler,
    preNavigationHooks: [
      async ({ addInterceptRequestHandler }) => {
        await addInterceptRequestHandler((request) => {
          const requestUrl = request.url();
          if (disallowedDomains.some((domain) => requestUrl.includes(domain))) {
            return request.abort();
          }
          return request.continue();
        });
      },
    ],
    headless: process.env.ACTOR_RUN_ID !== undefined,
    proxyConfiguration,
  });

  await crawler.run(startUrls);
});


src/main.ts

import { Actor } from 'apify';
import { Dataset, PuppeteerRequestHandler } from 'crawlee';

enum Label {
  'detailPage',
  'searchPage',
}

export const detailPageHandler: PuppeteerRequestHandler = async (args) => {
  const { page, request } = args;
  let data = await page.evaluate(() => {
    const ldJson: any = document.querySelector(
      'script[type="application/ld+json"]'
    );
    return JSON.parse(ldJson.textContent);
  });

  if (!data) {
    throw new Error('Not data found!');
  }

  if (data.length > 1) {
    data = data[0];
  }
  const link = request.url;
  const name = data?.name;
  const sku = data?.sku;
  const description = data?.description;
  const image = data?.image;
  const brand = data?.brand?.name;
  let price = data?.offers?.price;
  let reviewRatingValue;
  const labels = await page.evaluate(() => {
    const leadTimeElem = document.querySelector('.lead-time');

    if (leadTimeElem) {
      let readyToShip, inStock, fastDispatch;

      const preIconElem: any = leadTimeElem.querySelector('.pre-icon');
      const iconInsElem: any = leadTimeElem.querySelector('.icon-ins');

      if (preIconElem) {
        readyToShip = preIconElem.textContent.trim();
      }

      if (iconInsElem) {
        const fastDispatchElem = iconInsElem.querySelector(
          '.detail-next-icon-success'
        );
        if (fastDispatchElem) {
          fastDispatch = fastDispatchElem.nextSibling.textContent.trim();
        }
        inStock = iconInsElem.childNodes[1].textContent.trim();
      }
      if (readyToShip && inStock && fastDispatch)
        return [readyToShip, inStock, fastDispatch];
    }
    return null;
  });

  if (data.review) {
    reviewRatingValue = data?.review[0]?.reviewRating?.ratingValue;
  }

  const additionalPhotosArray = await page.evaluate(() => {
    const slickTrack: any = document.querySelector('.detail-next-slick-track');
    const imgElements = slickTrack.querySelectorAll('img');
    const links: any = [];

    imgElements.forEach(function (imgElement) {
      const link = imgElement.getAttribute('src');
      links.push(link);
    });
    return links;
  });

  let unitPricing: any, princeRange: any;

  if (price) {
    const prices: any = [];
    const quantities: any = [];

    const result = await page.evaluate(
      (prices, quantities) => {
        let priceItems = document.querySelectorAll('.price-item');
        let isLadderPrice = false;

        if (!priceItems.length) {
          priceItems = document.querySelectorAll(
            '[data-role="ladder-price-item"]'
          );
          isLadderPrice = true;
        }
        priceItems.forEach((item) => {
          let quantityText, priceText;

          if (isLadderPrice) {
            const quantityRange: any = item.querySelector('.ma-quantity-range');
            quantityText = quantityRange?.textContent;
            const priceVal: any = item.querySelector('.priceVal');
            priceText = priceVal.getAttribute('title').substring(1);
          } else {
            const quantityElement: any = item.querySelector('.quality');
            quantityText = quantityElement?.textContent;
            const priceElement: any = item.querySelector('.price span');
            priceText = priceElement?.textContent.substring(1);
          }
          quantities.push(quantityText);
          prices.push(priceText);
        });

        return { quantities, prices };
      },
      prices,
      quantities
    );

    unitPricing = result.quantities.map((quantity, i) => {
      const range = quantity?.match(/\d+/g).map(Number);
      const price = result.prices[i];
      return {
        minUnits: range?.[0] || null,
        maxUnits: range?.[1] || null,
        priceString: price || null,
        price: Number(price),
      };
    });
  }

  if (!price) {
    [unitPricing, price, princeRange] = await page.evaluate(() => {
      const pricePromotionElement: any = document.querySelector('.promotion'); // wholesale cases
      const priceElement: any = document.querySelector(
        '.promotion-price strong.normal'
      );

      if (pricePromotionElement) {
        const priceList: any = document.querySelector('.price-list'); // promotion sales
        const priceItems = priceList.querySelectorAll('.price-item');
        const unitPricing: any = [];

        priceItems.forEach((item) => {
          const qualityText = item.querySelector('.quality').textContent;
          const price = item.querySelector('.price span').textContent;
          const range = qualityText.match(/\d+/g).map(Number);
          unitPricing.push({
            minUnits: range[0],
            maxUnits: range[1],
            priceString: price,
            price: Number(price.replace('$', '')),
          });
        });

        return [unitPricing];
      } else if (priceElement) {
        // piece sale cases
        const priceText = priceElement.textContent;
        return [null, priceText];
      } else {
        const priceRange: any = document
          .querySelector('.price-range .price')
          ?.textContent?.split(' - ');
        const minOrderString: any = document
          .querySelector('.price-range .moq')
          ?.textContent?.replace('piece/pieces', '')
          .trim();

        return [
          null,
          null,
          {
            rangeMinString: priceRange[0],
            rangeMin: priceRange[0]
              ? Number(priceRange[0]?.replace('$', ''))
              : undefined,
            rangeMaxString: priceRange[1],
            rangeMax: priceRange[1]
              ? Number(priceRange[1]?.replace('$', ''))
              : undefined,
            minOrderString,
            minOrder: Number(minOrderString),
          },
        ];
      }

      return ['???', null, null];
    });
  }
  const dataObject = {
    name,
    sku,
    labels: labels || null,
    priceString: price,
    price: Number(price),
    princeRange,
    unitPricing,
    brand,
    link,
    description: description || null,
    image,
    reviewRatingValue: reviewRatingValue || null,
    additionalPhotosArray,
  };
  const filteredDataObject = Object.entries(dataObject).reduce(
    (obj, [key, value]) => {
      if (value !== null) {
        obj[key] = value;
      }
      return obj;
    },
    {}
  );
  await Dataset.pushData(filteredDataObject);
};

export const defaultRequestHandler: PuppeteerRequestHandler = async (args) => {
  const { enqueueLinks, request, log, page } = args;
  const input: any = await Actor.getInput();
  const maxItems = input?.maxItems;
  const label: Label = request.userData.label;
  const itemsAdded: number = request.userData.itemsAdded || 0;

  log.info(`crawling ${request.url}`);

  switch (label) {
    case Label.detailPage: {
      return detailPageHandler(args);
    }
    case Label.searchPage: {
      const links = await page.evaluate(() => {
        const elements = document.querySelectorAll(
          'div.list-no-v2-outter.J-offer-wrapper.traffic-product-card > div > div > a[href]'
        );
        return Array.from(elements).map((element: any) => element.href);
      });
      await enqueueLinks({
        urls: maxItems ? links.slice(0, itemsAdded) : links,
        userData: {
          label: Label.detailPage,
        },
      });
      return;
    }

    default: {
      if (request.url.includes('/product-detail/')) {
        return detailPageHandler(args);
      }
      const data = await page.evaluate(() => {
        const buttonArray = document.querySelectorAll(
          'div.seb-pagination__pages > a[href]'
        );
        const lastButton: any = buttonArray[buttonArray.length - 1];
        if (lastButton) {
          return lastButton.getAttribute('href').match(/page=(\d+)/)[1] || null;
        } else {
          return null;
        }
      });

      const hasMaxItems = Boolean(maxItems);
      const hasData = Boolean(data);

      let numPages = hasData ? Math.min(data, Math.ceil(maxItems / 48)) : 1;
      if (!numPages) numPages = data;
      const searchPagesArray = Array.from(
        { length: numPages },
        (_, i) => request.url + `&page=${i + 1}`
      );

      let items = hasMaxItems ? maxItems : numPages * 48;
      await Promise.all(
        searchPagesArray.map((element) => {
          const userData = {
            label: Label.searchPage,
            itemsAdded: items,
          };
          items -= 48;
          return enqueueLinks({
            urls: [element],
            userData: userData,
          });
        })
      );
    }
  }
};

export const getStartUrlsArray = (startUrls) => {
  if (startUrls) {
    return startUrls.map(({ url }) => {
      return url;
    });
  }
};


src/routes.ts

The Apify Alibaba Scraper is an efficient web crawling tool designed to scrape Alibaba, extracting product information, prices, and reviews. This crawler streamlines data collection by quickly crawling and scraping content, providing valuable insights for research and analysis.

Alibaba Scraper · Apify

Input · Alibaba Scraper · Apify

API · Alibaba Scraper · Apify

Source code · Alibaba Scraper · Apify

Related Actors · Alibaba Scraper · Apify

The Apify Alibaba Scraper is an efficient web crawling tool designed to scrape Alibaba, extracting product information, prices, and reviews. This crawler streamlines data collection by quickly craw...

Where next?