root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify/eslint-config-ts"
}


.eslintrc

apify_storage
node_modules
.idea
dist

apify_storage

.gitignore

# using multistage build, as we need dev deps to build the TS source code
FROM node:16 AS builder

# copy all files, install all dependencies (including dev deps) and build the project
COPY . ./
RUN npm install \
    && npm run build

# create final image, copy only package.json, readme and compiled code
FROM apify/actor-node:16
COPY --from=builder /package*.json ./
COPY --from=builder /README.md ./
COPY --from=builder /dist ./dist

# install only prod deps
RUN npm --quiet set progress=false \
    && npm install --only=prod --no-optional \
    && echo "Installed NPM packages:" \
    && (npm list --only=prod --no-optional --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version

# run compiled code
CMD npm run start:prod


Dockerfile

Actor running max concurrency, which helps you to not getting blocked.

Actor max concurrency

Select proxies to be used by your crawler.

Proxy configuration

Start URLs of Farfetch to start the actor. Category page, product page urls are all supported.

Farfetch start urls

Farfetch Scraper

INPUT_SCHEMA.json

# Farfetch Scraper

**Farfetch Scraper** provide a way to crawl product details of site Farfetch.

## Features

1. Support fetch products of category;
2. Support fetch product detail with prices,descriptions,images and sku info;

> The price values are multiplied by 100 to avoiding floating point calculations.


## Input Parameters

The input of this scraper shoule be JSON formated. Fields are:

| Field          | Type   | Description                                                                                                                |
| :------------- | :----- | :----------------------------------------------------------------------------------------------------|
| startUrls      | Array  | Start URLs of Farfetch site to start the scraper. Category page, product page urls are all supported. |
| proxy          | Object | Select proxies to be used by your crawler.                                                           |
| maxConcurrency | Object | Actor running max concurrency, which helps you to not getting blocked.                               |


## Supported URL types

* https://www.farfetch.com/shopping/women/dresses-1/items.aspx
* https://www.farfetch.com/shopping/women/isa-boulder-map-crochet-mesh-dress-item-17650965.aspx?storeid=12841


## Data storage

Farfetch scraper stores the product data to default data set in JSON format.

```json
{
  "source": {
    "id": 17053461,
    "crawlUrl": "https://www.farfetch.com/shopping/women/giuseppe-zanotti-dimitra-item-17053461.aspx?storeid=11909",
    "canonicalUrl": "https://www.farfetch.com/shopping/women/giuseppe-zanotti-dimitra-item-17053461.aspx",
    "retailer": "FARFETCH",
    "currency": "USD"
  },
  "brand": "Giuseppe Zanotti",
  "title": "Dimitra textured mules",
  "description": "Employing his signature use of Italian allure and contemporary glamour, Giuseppe Zanotti has designed footwear for over 25 years. Finished with a multicolour shearling exterior, these Dimitra textured mules are brilliantly bold.",
  "categories": [
    "Shoes",
    "Mules"
  ],
  "options": [
    {
      "type": "Colour",
      "values": [
        {
          "id": "MULTICOLOR",
          "name": "MULTICOLOR"
        }
      ]
    },
    {
      "type": "Size",
      "values": [
        {
          "id": "19",
          "name": "35"
        },
        {
          "id": "21",
          "name": "36"
        },
        {
          "id": "23",
          "name": "37"
        },
        {
          "id": "25",
          "name": "38"
        },
        {
          "id": "27",
          "name": "39"
        },
        {
          "id": "29",
          "name": "40"
        },
        {
          "id": "31",
          "name": "41"
        },
        {
          "id": "33",
          "name": "42"
        }
      ]
    }
  ],
  "variants": [
    {
      "id": "",
      "options": [
        "19",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 1,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "21",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 3,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "23",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 3,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "25",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 5,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "27",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 6,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "29",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 3,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "31",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 2,
        "stockStatus": "InStock"
      }
    },
    {
      "id": "",
      "options": [
        "33",
        "MULTICOLOR"
      ],
      "price": {
        "current": 495000,
        "original": 825000,
        "stockCount": 1,
        "stockStatus": "InStock"
      }
    }
  ],
  "medias": [
    {
      "type": "Image",
      "url": "https://cdn-images.farfetch-contents.com/17/05/34/61/17053461_34334698_480.jpg",
      "alt": "Giuseppe Zanotti Dimitra",
      "index": 1
    },
    {
      "type": "Image",
      "url": "https://cdn-images.farfetch-contents.com/17/05/34/61/17053461_34334696_480.jpg",
      "alt": "Giuseppe Zanotti Dimitra",
      "index": 2
    },
    {
      "type": "Image",
      "url": "https://cdn-images.farfetch-contents.com/17/05/34/61/17053461_34334697_480.jpg",
      "alt": "Giuseppe Zanotti Dimitra",
      "index": 3
    },
    {
      "type": "Image",
      "url": "https://cdn-images.farfetch-contents.com/17/05/34/61/17053461_34336058_480.jpg",
      "alt": "Giuseppe Zanotti Dimitra",
      "index": 4
    },
    {
      "type": "Image",
      "url": "https://cdn-images.farfetch-contents.com/17/05/34/61/17053461_34336057_480.jpg",
      "alt": "Giuseppe Zanotti Dimitra",
      "index": 5
    }
  ],
  "price": {
    "current": 495000,
    "original": 825000,
    "stockStatus": "InStock"
  }
}
```



README.md

farfetch

apify.json

farfetch-scraper

package.json

tsconfig.json

// Crawler
import { URL } from 'url';
import Apify, { CheerioCrawlerOptions, Dataset, ProxyConfiguration, Request, RequestQueue } from 'apify';
import { Log, LogLevel } from '@apify/log';
import { sleep } from 'apify/build/utils';
import _ from 'lodash';
import axios from 'axios';

import { Image, MediaType, Price, Product, Source, StockStatus, Variant, VariantOption, VariantOptionType, VariantOptionValue } from './spec';

const { log } = Apify.utils;

export interface Options {
    proxy:ProxyConfiguration | undefined
    logLevel:LogLevel,
    maxConcurrency:number,
}

export class Crawler {
    private id = 'FARFETCH'

    private startUrls:{ url:string }[]

    private proxyConf:ProxyConfiguration | undefined

    private requestQueue:RequestQueue | undefined

    private dataset:Dataset | undefined

    private options:Options

    private logger:Log

    constructor(startUrls:{ url:string }[], options:Options) {
        this.logger = log.child({ prefix: `${this.id} Scraper` });
        this.logger.setOptions({ level: options.logLevel || log.LEVELS.DEBUG });

        this.startUrls = [];
        (startUrls || []).forEach((item) => {
            if (!(item.url || '').startsWith('https://') && !(item.url || '').startsWith('http://')) {
                throw new Error('Require start url for crawler');
            }
            this.startUrls.push(item);
        });
        this.proxyConf = options.proxy;

        if (!options.maxConcurrency) {
            options.maxConcurrency = 5;
        }
        this.options = options;
        this.requestQueue = undefined;
        this.dataset = undefined;
    }

    async Parse($:any, request:Request):Promise<void> {
        this.logger.info(`access ${request.url}`);

        const u = new URL(request.url);
        if (u.pathname.match(/\/items\.aspx$/gi)) {
            await this.parseCategoryProduct($, request);
        } else if (u.pathname.match(/-\d+\.aspx$/gi)) {
            await this.parseProduct($, request);
        } else {
            throw Error(`unsupported url ${request.url}`);
        }
    }

    async parseCategoryProduct($:any, request:Request):Promise<void> {
        const rawPageInfo = />\s*window.__HYDRATION_STATE__\s*=\s*(".+?")\s*<\/script>/gi.exec($.html());
        if (!rawPageInfo || rawPageInfo.length === 0) {
            throw Error('load product page html failed');
        }
        const pageInfo = JSON.parse(JSON.parse(rawPageInfo[1]));
        const prods = _.get(pageInfo, ['initialStates', 'slice-listing', 'listingItems', 'items']);

        for (let i = 0; i < prods.length; i++) {
            const prod = prods[i];
            if (prod.url) {
                const subReqUrl = `https://www.farfetch.com${prod.url}`;
                await this.pubSubRequest(subReqUrl);
            }
        }
        const pagination = _.get(pageInfo, ['initialStates', 'slice-listing', 'listingPagination']);
        if (prods.length < (pagination.view || 90)) { return; }

        const u = new URL(request.url);
        const page = parseInt(u.searchParams.get('page') || '1', 10) || 1;
        if (page < pagination.totalPages) {
            u.searchParams.set('page', (page + 1).toString());
            u.searchParams.set('view', (pagination.view || 90).toString());
            await this.pubSubRequest(u.toString());
        }
    };

    async parseProduct($:any, request:Request):Promise<void> {
        const rawPageInfo = />\s*window.__HYDRATION_STATE__\s*=\s*(".+?")\s*<\/script>/gi.exec($.html());
        if (!rawPageInfo || rawPageInfo.length === 0) {
            throw Error('load product page html failed');
        }
        const pageInfo = JSON.parse(JSON.parse(rawPageInfo[1]));
        const states = _.get(pageInfo, ['apolloInitialState', 'ROOT_QUERY']);
        let detail;
        Object.keys(states).every((key:string):boolean => {
            if (key.startsWith('initialStates(')) {
                detail = _.get(states, [key, 'data', 'slice-product', 'productViewModel']);
                return false;
            }
            return true;
        });

        let canonicalUrl = $('link[rel="canonical"]').attr('href');
        if (canonicalUrl.startsWith('/')) {
            canonicalUrl = `https://www.farfetch.com${canonicalUrl}`;
        }

        const prod = {
            source: {
                id: _.get(detail, ['details', 'productId'], ''),
                crawlUrl: request.url,
                canonicalUrl: canonicalUrl || '',
                retailer: this.id,
                currency: $('meta[itemProp="priceCurrency"]').attr('content') || '',
            } as Source,
            brand: _.get(detail, ['designerDetails', 'name'], ''),
            title: _.get(detail, ['details', 'shortDescription'], ''),
            description: _.get(detail, ['details', 'description'], ''),
            categories: Object.values(_.get(detail, ['categories'], {})).map((item:any):string => {
                return _.get(item, 'name');
            }).filter((name:string):boolean => {
                return !!name;
            }),
            options: [],
            variants: [],
            medias: _.get(detail, ['images', 'main'], []).map((item:any):Image => {
                const img = {
                    type: MediaType.Image,
                    url: item.large || item.zoom || item.medium,
                    alt: item.alt || '',
                    index: item.index || 0,
                } as Image;
                if (img.url.startsWith('/')) {
                    img.url = `https://cdn-images.farfetch-contents.com${img.url}`;
                }
                return img;
            }),
            price: { stockStatus: StockStatus.OutofStock } as Price,
        } as Product;

        const colorVal = _.get(detail, ['details', 'colors']);
        if (colorVal) {
            const colorVariant = {
                type: VariantOptionType.Colour,
                values: [] as VariantOptionValue[],
            };
            colorVariant.values.push({ id: colorVal, name: colorVal });
            prod.options!.push(colorVariant);
        }

        const priceInfo = _.get(detail, 'priceInfo');
        if (_.get(priceInfo, 'default')) {
            prod.price = {
                current: _.get(priceInfo, ['default', 'finalPrice'], 0) * 100,
                original: _.get(priceInfo, ['default', 'initialPrice'], 0) * 100,
                stockStatus: StockStatus.OutofStock,
            } as Price;
        }

        const sizeOptions = Object.values(_.get(detail, ['sizes', 'available'], {}));
        sizeOptions.sort((a:any, b:any):number => {
            return a.order <= b.order ? -1 : 1;
        });
        const sizeVariant = {
            type: VariantOptionType.Size,
            values: [] as VariantOptionValue[],
        } as VariantOption;

        sizeOptions.forEach((item:any) => {
            sizeVariant.values.push({
                id: item.sizeId.toString(),
                name: item.description,
            } as VariantOptionValue);

            const price = _.get(priceInfo, item.sizeId.toString(), _.get(priceInfo, 'default')) as any;
            const variant = {
                id: '',
                options: [item.sizeId.toString()],
                price: {
                    current: (price!.finalPrice || 0) * 100,
                    original: (price!.initialPrice || 0) * 100,
                    stockCount: item.quantity || 0,
                    stockStatus: item.quantity > 0 ? StockStatus.InStock : StockStatus.OutofStock,
                } as Price,
            } as Variant;

            if (prod.price!.stockStatus !== StockStatus.InStock && variant.price.stockStatus === StockStatus.InStock) {
                prod.price!.stockStatus = StockStatus.InStock;
            }
            if (colorVal) {
                variant.options.push(colorVal);
            }
            prod.variants.push(variant);
        });
        prod.options!.push(sizeVariant);

        return this.saveItem(prod);
    }

    async pubSubRequest(url:any):Promise<void> {
        let u = url;
        if (typeof url === 'string') {
            u = { url };
        }
        await this.requestQueue?.addRequest(u);
    }

    async saveItem(item:any):Promise<void> {
        if (item) {
            if (!this.dataset) {
                this.dataset = await Apify.openDataset();
            }
            await this.dataset.pushData(item);
        }
    }

    async run():Promise<void> {
        this.requestQueue = await Apify.openRequestQueue();

        for (let i = 0; i < this.startUrls.length; i++) {
            await this.requestQueue.addRequest(this.startUrls[i]);
        }

        const runner = new Apify.CheerioCrawler({
            requestQueue: this.requestQueue,
            proxyConfiguration: this.proxyConf,
            useSessionPool: true,
            sessionPoolOptions: { maxPoolSize: this.options.maxConcurrency },
            persistCookiesPerSession: true,
            minConcurrency: 1,
            maxConcurrency: this.options.maxConcurrency || 10,
            maxRequestRetries: 3,
            ignoreSslErrors: true,
            preNavigationHooks: [
                async (requestAsBrowserOptions:any) => {
                    requestAsBrowserOptions.forceUrlEncoding = true;
                    requestAsBrowserOptions.headerGeneratorOptions = {
                        browsers: [{ name: 'firefox', minVersion: 80 }, { name: 'chrome', minVersion: 70 }],
                        devices: ['desktop'],
                        operatingSystems: ['windows', 'macos', 'linux'],
                    };
                },
            ],
            handlePageFunction: async ({ $, request, session }) => {
                try {
                    await this.Parse($, request);
                    await sleep(100);
                } catch (err) {
                    session.markBad();
                    this.logger.error(`parse url ${request.url} failed with error ${err}`);
                }
            },
            handleFailedRequestFunction: async ({ error, request }) => {
                await axios.post('https://open.feishu.cn/open-apis/bot/v2/hook/f328cb5c-be3d-4cd1-9519-024eafbc46b8', {
                    msg_type: 'text',
                    content: { text: `${this.id} fetch ${request.url} failed` },
                }).catch(() => null);
                this.logger.error(`try parse url ${request.url} with ${request.retryCount} times failed with error ${error}`);
            },
        } as CheerioCrawlerOptions);
        await runner.run();
    };
};


src/crawler.ts

/* eslint-disable max-len */
// This is the main Node.js source code file of your actor.
// It is referenced from the "scripts" section of the package.json file,
// so that it can be started by running "npm start".

// Include Apify SDK. For more information, see https://sdk.apify.com/
import Apify, { ProxyConfiguration } from 'apify';
import { Crawler, Options } from './crawler';

const { log } = Apify.utils;

interface InputSchema {
    startUrls: any
    proxy: any
    maxConcurrency: number
}

Apify.main(async () => {
    // Get input of the actor.
    // If you'd like to have your input checked and have Apify display
    // a user interface for it, add INPUT_SCHEMA.json file to your actor.
    // For more information, see https://apify.com/docs/actor/input-schema

    const input = (await Apify.getInput() || {}) as InputSchema;

    log.info('Input:', input);

    let proxyConf:ProxyConfiguration | undefined;

    if (input.proxy && input.proxy.useApifyProxy) {
        proxyConf = await Apify.createProxyConfiguration({
            groups: (input.proxy || {}).apifyProxyGroups || undefined,
            countryCode: (input.proxy || {}).apifyProxyCountry || undefined,
            proxyUrls: (input.proxy || {}).proxyUrls || undefined,
        });
    }

    const buildMsg = (typ:string, urls:any):string => {
        return JSON.stringify({ msg_type: 'text', content: { text: `fetch ${typ} with urls ${JSON.stringify(urls || [])}` } });
    };

    await Apify.addWebhook({
        eventTypes: ['ACTOR.RUN.SUCCEEDED'],
        requestUrl: 'https://open.feishu.cn/open-apis/bot/v2/hook/f328cb5c-be3d-4cd1-9519-024eafbc46b8',
        payloadTemplate: buildMsg('succeed', input.startUrls),
    });
    await Apify.addWebhook({
        eventTypes: ['ACTOR.RUN.FAILED'],
        requestUrl: 'https://open.feishu.cn/open-apis/bot/v2/hook/f328cb5c-be3d-4cd1-9519-024eafbc46b8',
        payloadTemplate: buildMsg('failed', input.startUrls),
    });
    await Apify.addWebhook({
        eventTypes: ['ACTOR.RUN.ABORTED'],
        requestUrl: 'https://open.feishu.cn/open-apis/bot/v2/hook/f328cb5c-be3d-4cd1-9519-024eafbc46b8',
        payloadTemplate: buildMsg('aborted', input.startUrls),
    });

    const crawler = new Crawler(input.startUrls || [
        { url: 'https://www.farfetch.com/shopping/women/mules-1/items.aspx' },
    ], {
        proxy: proxyConf,
        maxConcurrency: input.maxConcurrency,
    } as Options);
    await crawler.run();
});


src/main.ts

export interface Source {
    id:string
    crawlUrl?:string
    canonicalUrl:string
    retailer:string
    language?:string
    currency:string
    bundleIds?:string[],
    createdUTC?:number
    updatedUTC?:number
    publishedUTC?:number
}

export enum StockStatus {
    InStock = 'InStock',
    LowinStock = 'LowInStock',
    OutofStock = 'OutOfStock'
}

export interface Price {
    current:number
    previous:number
    original?:number

    stockStatus:StockStatus
    stockCount:number
}

export interface Stats {
    rating?:number
    reviewCount?:number
    lovesCount?:number // Sephora
}

export enum MediaType {
    Image = 'Image',
    Video = 'Video'
}

export interface Image {
    id?:string
    type:MediaType
    url:string
    alt?:string
    index?:number
    width?:number
    height?:number
    variantIds?:string[]
}

export interface Video {
    id?:string
    type:MediaType
    videoUrl:string
    alt?:string
    index?:number
    width?:number
    height?:number
    cover?:Image
    variantIds?:string[]
}

export interface VariantOptionValue {
    id:string
    name:string
    icon?:string
}

export enum VariantOptionType {
    Size = 'Size',
    Colour = 'Colour'
}

export interface VariantOption {
    type:VariantOptionType | string
    values:VariantOptionValue[]
}

export interface Variant {
    id:string
    title?:string
    sku?:string
    options:string[]
    price:Price
    extraInfo?:Record<string, any>
}

export interface Product {
    source:Source
    title:string
    description: string | {
        detail:string
        sizeAndFit:string
        careInfo:string
        aboutMe:string
    }
    brand:string
    categories?:string[]
    tags?: string[]
    medias:(Image|Video)[]
    variants:Variant[]
    options?:VariantOption[]
    price?:Price
    stats?:Stats
    extraInfo?:Record<string, any>
    subProducts?:Product[]
}

export interface Collection {
    id:string
    title:string
    handle:string
    description:string
    publishedUTC:number
    updatedUTC:number
    productsCount:number
}


src/spec.ts

// Util

export const parsePrice = function (text:string):number {
    const matched = /[\d]+(?:[,.][\d]+)?/g.exec(text) || [];
    if (matched.length > 0) {
        return parseFloat(matched[0]);
    }
    return 0;
};


src/util.ts

Farfetch web scraper to crawl product information including price and sale price, color, and images.

Farfetch web scraper to crawl product information including price and sale price, color, size, images, stock.

Farfetch start urls

Proxy configuration

Actor max concurrency

Where next?