Baidu Scraper

jan.turon/baidu-scraper

No credit card required

This actor allows you automatically crawl the result page for your keyword for the Baidu Search engine

.DS_Store

Download

.eslintrc

{
  "extends": "@apify"
}

.gitignore

apify_storage
node_modules
.idea
dist
memory_storage
crawlee_storage

Dockerfile

FROM apify/actor-node-playwright-chrome:16

RUN npm -v
COPY . .

RUN npm i
RUN npx playwright install
RUN npm run build

INPUT_SCHEMA.json

{
  "title": "Baidu Scraper",
  "description": "Scraper for Chinese Search Engine Baidu",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "searchPhrases": {
      "title": "Search Phrases",
      "type": "array",
      "description": "Phrases to search on Baidu",
      "prefill": ["Apify"],
      "editor": "stringList"
    },
    "pages": {
      "title": "Pages",
      "type": "integer",
      "description": "Number of pages to search",
      "editor": "number"
    }
  },
  "required": ["searchPhrases"]
}

apify.json.deprecated

{
	"name": "baidu-scraper",
	"version": "0.0",
	"buildTag": "latest",
	"env": null
}

baidu-scraper.zip

Download

package.json

{
  "name": "baidu-scraper",
  "version": "0.0.1",
  "description": "Apify scraper for Baidu search engine",
  "main": "dist/main.js",
  "scripts": {
    "build": "tsc -p tsconfig.json && tsc-alias",
    "start": "node --experimental-specifier-resolution=node dist/main.js",
    "buildAndRun": "rm -rf dist crawlee_storage && npm run build && npm run start",
    "publish": "npm run build && apify push"
  },
  "keywords": [
    "apify"
  ],
  "author": "Jan Turoň",
  "license": "ISC",
  "dependencies": {
    "@apify/tsconfig": "^0.1.0",
    "@crawlee/playwright": "^3.0.3",
    "apify": "^3.0.2",
    "axios": "^0.27.2",
    "cheerio": "^1.0.0-rc.12",
    "crawlee": "^3.0.3",
    "playwright": "^1.25.0",
    "tsc-alias": "^1.7.0",
    "typescript": "^4.7.4"
  },
  "type": "module",
  "devDependencies": {
    "@types/node": "^17.0.41",
    "apify-cli": "^0.7.4",
    "husky": "^8.0.1",
    "prettier": "^2.7.1"
  },
  "optionalDependencies": {
    "fsevents": "^2.3.2"
  },
  "engines": {
    "node": ">=16"
  }
}

tsconfig.json

{
  "extends": "@apify/tsconfig",
  "compilerOptions": {
    "module": "ES2022",
    "sourceMap": false,
    "declaration": false,
    "declarationMap": false,
    "target": "ES2022",
    "outDir": "dist",
    "lib": ["DOM"],
    "baseUrl": "src",
    "paths": { "@/*": ["./*"] }
  },
  "include": ["src"]
}

.actor/actor.json

{
	"actorSpecification": 1,
	"name": "baidu-scraper",
	"version": "0.0",
	"buildTag": "latest"
}

.husky/pre-commit

#!/usr/bin/env sh
. "$(dirname -- "$0")/_/husky.sh"

src/main.ts

import { resolveBaiduLink, resolveResultsCount } from "@/utils";
import { KeyValueStore, Dataset, Actor } from "apify";
import { PlaywrightCrawler } from "crawlee";

interface Link {
  url: string;
  title: string;
}

interface ResultLink extends Link {
  description: string;
}

interface DataResults {
  results: ResultLink[];
  hotSearchResults?: Link[];
  relatedSearchKeywords?: string[];
  similarSearchKeywords?: string[];
  resultsCount?: number;
}

interface Input {
  searchPhrases: string[];
  pages?: number;
}

await Actor.init();

const crawler = new PlaywrightCrawler({
  requestHandler: async ({ page }) => {
    const resultElements = await page.$$("h3");
    const results: ResultLink[] = [];
    for (const titleElement of resultElements) {
      const title = (await titleElement.textContent()) || "";
      const linkElement = await titleElement.$("a");
      const linkUrl = await linkElement?.getAttribute("href");

      const url = linkUrl ? await resolveBaiduLink(linkUrl) : "";

      results.push({ title, description: linkUrl!, url });
    }

    const hotSearchResultElements = await page.$$("div[class^='toplist1-tr']");
    const hotSearchResults: Link[] = [];
    for (const hotSearchElement of hotSearchResultElements.slice(0, 16)) {
      const title = (await hotSearchElement.textContent()) || "";
      const linkElement = await hotSearchElement.$("a");
      const url = (await linkElement?.getAttribute("href")) || "";

      hotSearchResults.push({ title, url });
    }

    const relatedKeywordsElements = await page.$$(
      "div[tpl='recommend_list'] .c-gap-top-xsmall"
    );
    const relatedSearchKeywords: string[] = [];
    for (const relatedKeywordsElement of relatedKeywordsElements) {
      const title = (await relatedKeywordsElement.textContent()) || "";
      relatedSearchKeywords.push(title);
    }

    const similarKeywordsElements = await page.$$("a[class^='rs-link']");
    const similarSearchKeywords: string[] = [];
    for (const similarKeywordsElement of similarKeywordsElements) {
      const title = (await similarKeywordsElement.getAttribute("title")) || "";
      similarSearchKeywords.push(title);
    }

    const resultsCountElement = await page.$("span[class^='hint']");
    const resultsCount = resolveResultsCount(
      (await resultsCountElement?.textContent()) || ""
    );

    const dataStructure: DataResults = {
      results,
      hotSearchResults,
      resultsCount,
      similarSearchKeywords,
      relatedSearchKeywords,
    };

    await Dataset.pushData([dataStructure]);
  },
});

const { searchPhrases = ["apify", "nic"], pages = 1 } =
  (await KeyValueStore.getInput<Input>()) ?? {};

const pageArray = [...Array(pages).keys()];

const requests = searchPhrases.flatMap((searchPhrase) =>
  pageArray.map((pageNr) => {
    const lookoutQuery = new URLSearchParams([
      ["pn", pageNr.toString()],
      ["wd", searchPhrase],
    ]);
    return `https://www.baidu.com/s?${lookoutQuery.toString()}`;
  })
);

await crawler.addRequests(requests);
await crawler.run();

await Actor.exit();

src/utils.ts

import axios from "axios";
import cheerio from "cheerio";

/**
 * There's 2 discovered baidu links behaviors
 * - redirect via server response 302,
 * - timed-out custom redirect with custom HTML (status 200)
 * @param url Baidu redirect url. Something like: http://www.baidu.com/link?url=someBasey64Chars
 */
export const resolveBaiduLink = async (url: string): Promise<string> => {
  const { data, headers } = await axios.get<string>(url, {
    maxRedirects: 0,
    validateStatus: (status) => [302, 200].includes(status), // Baidu sometimes returns redirect 302 which leads to Axios error.
  });

  // Redirect location is enough when available (for 302 response)
  if (headers.location) return headers.location;

  // Baidu sometimes returns 200 with custom html - address is available in 'noscript' tag
  const cheerioSelector = cheerio.load(data);

  const noScriptElementContent = cheerioSelector("noscript").html();

  if (!noScriptElementContent) return "";

  const fullUrl = /(')(?:(?=(\\?))\2.)*?\1/.exec(noScriptElementContent);

  return fullUrl?.[1]!;
};

/**
 * Resolves results count string to number
 * @param countText span tag text above first Baidu result, looks like '百度为您找到相关结果约740,000个'
 */
export const resolveResultsCount = (countText: string): number => {
  const digitMatches = [...countText.matchAll(/\d+/g)]; // Thousands are comma-separated, matches will be in groups - ['740', '000']

  const digitString = digitMatches.reduce(
    (digitString, digitMatch) => digitString + digitMatch,
    ""
  );

  return parseInt(digitString);
};