Alibaba Scraper

lexis-solutions/alibaba-scraper

No credit card required

The Apify Alibaba Scraper is an efficient web crawling tool designed to scrape Alibaba, extracting product information, prices, and reviews. This crawler streamlines data collection by quickly crawling and scraping content, providing valuable insights for research and analysis.

.dockerignore

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

.gitignore

# This file tells Git which files shouldn't be added to source control

.idea
dist
node_modules
apify_storage
crawlee_storage
storage

Dockerfile

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16 AS builder

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package.json yarn.lock ./

# Install all dependencies. Don't audit to speed up the installation.
RUN yarn --production=false

# Next, copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN yarn build

# Create final image
FROM apify/actor-node-puppeteer-chrome:16

# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist

# Copy just package.json and yarn.lock
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json yarn.lock ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN yarn --prod \
    && echo "Installed NPM packages:" \
    && (yarn list --depth 1 || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "Yarn version:" \
    && yarn --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && yarn start:prod --silent

apify.json

{
  "name": "alibaba-scraper",
  "version": "0.0",
  "buildTag": "latest",
  "env": null,
  "$schema": {
    "startUrl": {}
  }
}

package.json

{
  "name": "alibaba",
  "version": "0.0.1",
  "dependencies": {
    "@crawlee/puppeteer": "3.3.0",
    "apify": "3.1.2",
    "crawlee": "^3.2.2",
    "prettier": "^2.8.8",
    "puppeteer": "^19.7.2"
  },
  "devDependencies": {
    "@apify/tsconfig": "0.1.0",
    "@types/node": "18.15.0",
    "ts-node": "10.8.0",
    "typescript": "4.7.4"
  },
  "scripts": {
    "dev": "apify run",
    "start": "ts-node-esm -T src/main.ts",
    "start:prod": "node dist/main.js",
    "build": "tsc",
    "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
  },
  "license": "ISC"
}

tsconfig.json

{
  "extends": "@apify/tsconfig",
  "compilerOptions": {
    "module": "CommonJS",
    "target": "ES2022",
    "moduleResolution": "node",
    "outDir": "dist",
    "lib": ["DOM"],
    "noImplicitAny": false,
    "noUnusedLocals": false,
    "noUnusedParameters": false,
    "esModuleInterop": true
  },
  "include": ["./src/**/*"]
}

.actor/actor.json

{
  "actorSpecification": 1,
  "name": "alibaba-scraper",
  "title": "Alibaba scraper",
  "version": "0.0",
  "input": "./input.json",
  "storages": {
    "dataset": "./output.json"
  }
}

.actor/input.json

{
  "title": "Alibaba Scraper",
  "description": "This is alibaba scraper input schema",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "startUrls": {
      "title": "Start URLs",
      "type": "array",
      "description": "URLs to scrape",
      "editor": "requestListSources",
      "prefill": [
        {
          "url": "https://www.alibaba.com/trade/search?fsb=y&IndexArea=product_en&categoryId=201153401&keywords=Groom+Wear&knowledgeGraphId=100010232-10000166340&viewtype=L&&pricef=80&pricet"
        }
      ]
    },
    "maxItems": {
      "title": "Max Items",
      "type": "integer",
      "description": "The number of items to be scraped.",
      "prefill": 1
    },
    "proxyConfiguration": {
      "title": "Proxy Configuration",
      "type": "object",
      "description": "Your proxy configuration from Apify",
      "editor": "proxy"
    }
  },
  "required": ["startUrls"]
}

.actor/output.json

{
  "actorSpecification": 1,
  "fields": {},
  "views": {
    "overview": {
      "title": "Overview",
      "transformation": {
        "fields": [
          "name",
          "price",
          "brand",
          "link"
        ]
      },
      "display": {
        "component": "table",
        "properties": {
          "type": "object",
          "properties": {
            "name": {
              "type": "string",
              "description": "The name of the product",
              "label" : "Name"
            },
            "price": {
              "type": "string",
              "label" : "Price"
            },
            "brand": {
              "type": "string",
              "description": "The brand of the product",
              "label" : "Brand"
            },
            "link" : {
              "type" : "string",
              "description" : "Link to product",
              "label" : "Link"
            }
          }
        }
      }
    }
  }
}

src/main.ts

import { Actor } from 'apify';
import { PuppeteerCrawler } from 'crawlee';

import { defaultRequestHandler, getStartUrlsArray } from './routes';

const disallowedDomains = [
  'cookielaw.org',
  'cdn.cookielaw.org',
  'googletagmanager.com',
  'google-analytics.com',
  'connect.facebook.netsa',
  'cdn.cookielaw.org',
  'analytics.tiktok.com',
  'bat.bing.com',
  'accounts.google.com',
  'facebook.com',
  'adservice.google.com',
  'gj.mmstat.com',
  'img.alicdn.com',
];

Actor.main(async () => {
  const input: any = await Actor.getInput();

  const proxyConfiguration = await Actor.createProxyConfiguration(
    input.proxyConfiguration
  );
  const startUrls: any = getStartUrlsArray(input.startUrls);

  const crawler = new PuppeteerCrawler({
    requestHandler: defaultRequestHandler,
    preNavigationHooks: [
      async ({ addInterceptRequestHandler }) => {
        await addInterceptRequestHandler((request) => {
          const requestUrl = request.url();
          if (disallowedDomains.some((domain) => requestUrl.includes(domain))) {
            return request.abort();
          }
          return request.continue();
        });
      },
    ],
    headless: process.env.ACTOR_RUN_ID !== undefined,
    proxyConfiguration,
  });

  await crawler.run(startUrls);
});

src/routes.ts

import { Actor } from 'apify';
import { Dataset, PuppeteerRequestHandler } from 'crawlee';

enum Label {
  'detailPage',
  'searchPage',
}

export const detailPageHandler: PuppeteerRequestHandler = async (args) => {
  const { page, request } = args;
  let data = await page.evaluate(() => {
    const ldJson: any = document.querySelector(
      'script[type="application/ld+json"]'
    );
    return JSON.parse(ldJson.textContent);
  });

  if (!data) {
    throw new Error('Not data found!');
  }

  if (data.length > 1) {
    data = data[0];
  }
  const link = request.url;
  const name = data?.name;
  const sku = data?.sku;
  const description = data?.description;
  const image = data?.image;
  const brand = data?.brand?.name;
  let price = data?.offers?.price;
  let reviewRatingValue;
  const labels = await page.evaluate(() => {
    const leadTimeElem = document.querySelector('.lead-time');

    if (leadTimeElem) {
      let readyToShip, inStock, fastDispatch;

      const preIconElem: any = leadTimeElem.querySelector('.pre-icon');
      const iconInsElem: any = leadTimeElem.querySelector('.icon-ins');

      if (preIconElem) {
        readyToShip = preIconElem.textContent.trim();
      }

      if (iconInsElem) {
        const fastDispatchElem = iconInsElem.querySelector(
          '.detail-next-icon-success'
        );
        if (fastDispatchElem) {
          fastDispatch = fastDispatchElem.nextSibling.textContent.trim();
        }
        inStock = iconInsElem.childNodes[1].textContent.trim();
      }
      if (readyToShip && inStock && fastDispatch)
        return [readyToShip, inStock, fastDispatch];
    }
    return null;
  });

  if (data.review) {
    reviewRatingValue = data?.review[0]?.reviewRating?.ratingValue;
  }

  const additionalPhotosArray = await page.evaluate(() => {
    const slickTrack: any = document.querySelector('.detail-next-slick-track');
    const imgElements = slickTrack.querySelectorAll('img');
    const links: any = [];

    imgElements.forEach(function (imgElement) {
      const link = imgElement.getAttribute('src');
      links.push(link);
    });
    return links;
  });

  let unitPricing: any, princeRange: any;

  if (price) {
    const prices: any = [];
    const quantities: any = [];

    const result = await page.evaluate(
      (prices, quantities) => {
        let priceItems = document.querySelectorAll('.price-item');
        let isLadderPrice = false;

        if (!priceItems.length) {
          priceItems = document.querySelectorAll(
            '[data-role="ladder-price-item"]'
          );
          isLadderPrice = true;
        }
        priceItems.forEach((item) => {
          let quantityText, priceText;

          if (isLadderPrice) {
            const quantityRange: any = item.querySelector('.ma-quantity-range');
            quantityText = quantityRange?.textContent;
            const priceVal: any = item.querySelector('.priceVal');
            priceText = priceVal.getAttribute('title').substring(1);
          } else {
            const quantityElement: any = item.querySelector('.quality');
            quantityText = quantityElement?.textContent;
            const priceElement: any = item.querySelector('.price span');
            priceText = priceElement?.textContent.substring(1);
          }
          quantities.push(quantityText);
          prices.push(priceText);
        });

        return { quantities, prices };
      },
      prices,
      quantities
    );

    unitPricing = result.quantities.map((quantity, i) => {
      const range = quantity?.match(/\d+/g).map(Number);
      const price = result.prices[i];
      return {
        minUnits: range?.[0] || null,
        maxUnits: range?.[1] || null,
        priceString: price || null,
        price: Number(price),
      };
    });
  }

  if (!price) {
    [unitPricing, price, princeRange] = await page.evaluate(() => {
      const pricePromotionElement: any = document.querySelector('.promotion'); // wholesale cases
      const priceElement: any = document.querySelector(
        '.promotion-price strong.normal'
      );

      if (pricePromotionElement) {
        const priceList: any = document.querySelector('.price-list'); // promotion sales
        const priceItems = priceList.querySelectorAll('.price-item');
        const unitPricing: any = [];

        priceItems.forEach((item) => {
          const qualityText = item.querySelector('.quality').textContent;
          const price = item.querySelector('.price span').textContent;
          const range = qualityText.match(/\d+/g).map(Number);
          unitPricing.push({
            minUnits: range[0],
            maxUnits: range[1],
            priceString: price,
            price: Number(price.replace('$', '')),
          });
        });

        return [unitPricing];
      } else if (priceElement) {
        // piece sale cases
        const priceText = priceElement.textContent;
        return [null, priceText];
      } else {
        const priceRange: any = document
          .querySelector('.price-range .price')
          ?.textContent?.split(' - ');
        const minOrderString: any = document
          .querySelector('.price-range .moq')
          ?.textContent?.replace('piece/pieces', '')
          .trim();

        return [
          null,
          null,
          {
            rangeMinString: priceRange[0],
            rangeMin: priceRange[0]
              ? Number(priceRange[0]?.replace('$', ''))
              : undefined,
            rangeMaxString: priceRange[1],
            rangeMax: priceRange[1]
              ? Number(priceRange[1]?.replace('$', ''))
              : undefined,
            minOrderString,
            minOrder: Number(minOrderString),
          },
        ];
      }

      return ['???', null, null];
    });
  }
  const dataObject = {
    name,
    sku,
    labels: labels || null,
    priceString: price,
    price: Number(price),
    princeRange,
    unitPricing,
    brand,
    link,
    description: description || null,
    image,
    reviewRatingValue: reviewRatingValue || null,
    additionalPhotosArray,
  };
  const filteredDataObject = Object.entries(dataObject).reduce(
    (obj, [key, value]) => {
      if (value !== null) {
        obj[key] = value;
      }
      return obj;
    },
    {}
  );
  await Dataset.pushData(filteredDataObject);
};

export const defaultRequestHandler: PuppeteerRequestHandler = async (args) => {
  const { enqueueLinks, request, log, page } = args;
  const input: any = await Actor.getInput();
  const maxItems = input?.maxItems;
  const label: Label = request.userData.label;
  const itemsAdded: number = request.userData.itemsAdded || 0;

  log.info(`crawling ${request.url}`);

  switch (label) {
    case Label.detailPage: {
      return detailPageHandler(args);
    }
    case Label.searchPage: {
      const links = await page.evaluate(() => {
        const elements = document.querySelectorAll(
          'div.list-no-v2-outter.J-offer-wrapper.traffic-product-card > div > div > a[href]'
        );
        return Array.from(elements).map((element: any) => element.href);
      });
      await enqueueLinks({
        urls: maxItems ? links.slice(0, itemsAdded) : links,
        userData: {
          label: Label.detailPage,
        },
      });
      return;
    }

    default: {
      if (request.url.includes('/product-detail/')) {
        return detailPageHandler(args);
      }
      const data = await page.evaluate(() => {
        const buttonArray = document.querySelectorAll(
          'div.seb-pagination__pages > a[href]'
        );
        const lastButton: any = buttonArray[buttonArray.length - 1];
        if (lastButton) {
          return lastButton.getAttribute('href').match(/page=(\d+)/)[1] || null;
        } else {
          return null;
        }
      });

      const hasMaxItems = Boolean(maxItems);
      const hasData = Boolean(data);

      let numPages = hasData ? Math.min(data, Math.ceil(maxItems / 48)) : 1;
      if (!numPages) numPages = data;
      const searchPagesArray = Array.from(
        { length: numPages },
        (_, i) => request.url + `&page=${i + 1}`
      );

      let items = hasMaxItems ? maxItems : numPages * 48;
      await Promise.all(
        searchPagesArray.map((element) => {
          const userData = {
            label: Label.searchPage,
            itemsAdded: items,
          };
          items -= 48;
          return enqueueLinks({
            urls: [element],
            userData: userData,
          });
        })
      );
    }
  }
};

export const getStartUrlsArray = (startUrls) => {
  if (startUrls) {
    return startUrls.map(({ url }) => {
      return url;
    });
  }
};