
LD+JSON Schema scraper
Pricing
Pay per usage
Go to Store

LD+JSON Schema scraper
Extract all LD+JSON tags from the given URLs.
0.0 (0)
Pricing
Pay per usage
7
Total users
324
Monthly users
8
Runs succeeded
92%
Last modified
3 years ago
.editorconfig
root = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf
.eslintrc
{ "extends": "@apify"}
.gitignore
# This file tells Git which files shouldn't be added to source control
.ideanode_modules
Dockerfile
# First, specify the base Docker image. You can read more about# the available images at https://sdk.apify.com/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node:16
# Second, copy just package.json and package-lock.json since it should be# the only file that affects "npm install" in the next step, to speed up the buildCOPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --only=prod --no-optional \ && echo "Installed NPM packages:" \ && (npm list --only=prod --no-optional --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Optionally, specify how to launch the source code of your actor.# By default, Apify's base Docker images define the CMD instruction# that runs the Node.js source code using the command specified# in the "scripts.start" section of the package.json file.# In short, the instruction looks something like this:## CMD npm start
INPUT_SCHEMA.json
{ "title": "LD JSON Schema scraper", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "title": "Start Urls", "type": "array", "description": "The URLs to extract all LD+JSON data", "default": [], "prefill": [{ "url": "https://blog.apify.com/" }], "editor": "requestListSources" }, "proxyConfiguration": { "title": "Proxy configuration", "description": "A proxy required for scraping", "type": "object", "default": { "useApifyProxy": true }, "prefill": { "useApifyProxy": true }, "editor": "proxy" }, "customData": { "title": "Custom data", "description": "Provide some custom data to output", "type": "object", "default": {}, "prefill": {}, "editor": "json" } }, "required": [ "startUrls", "proxyConfiguration" ]}
apify.json
{ "env": { "npm_config_loglevel": "silent" }}
main.js
1const Apify = require('apify');2
3const pageFunction = async (context) => {4 const { request, $, log, customData } = context;5
6 const { url } = request;7
8 const lds = $('script[type="application/ld+json"]');9
10 if (!lds.length) {11 log.warning('No LD+JSON found on page', { url });12 return {13 data: {},14 url,15 customData,16 };17 }18
19 return lds20 .map((_, el) => $(el).html().trim())21 .get()22 .map((html) => {23 try {24 return JSON.parse(html);25 } catch (e) {26 log.exception(e, 'Invalid JSON', { url });27 }28 })29 .filter(Boolean)30 .map((data) => {31 return {32 data,33 url,34 customData,35 }36 });37};38
39Apify.main(async () => {40 const { proxyConfiguration, startUrls, customData } = await Apify.getInput();41
42 if (!proxyConfiguration) {43 throw new Error('You require a proxy to run');44 }45
46 // test proxy47 const proxy = await Apify.createProxyConfiguration(proxyConfiguration);48
49 if (!proxy) {50 throw new Error('Invalid proxy configuration');51 }52
53 if (!startUrls?.length) {54 throw new Error('Provide a RequestList sources array on "startUrls" input'); 55 }56
57 await Apify.metamorph('apify/cheerio-scraper', {58 startUrls,59 pageFunction: pageFunction.toString(), 60 proxyConfiguration, 61 customData,62 ignoreSslErrors: true,63 });64});
package.json
{ "name": "project-empty", "version": "0.0.1", "description": "This is a boilerplate of an Apify actor.", "dependencies": { "apify": "^2.2.1" }, "scripts": { "start": "node main.js", "lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx", "lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix", "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" }, "author": "It's not you it's me", "license": "ISC"}