24 Hour Stats
Try for free
No credit card required
View all Actors
24 Hour Stats
mtrunkat/24-hour-stats
Try for free
No credit card required
This act can be used as synchronous API. Returns a JSON containing actor runs finished in the last 24 hours along with information about their default datasets and request queues. Actors might be filtered via input array "actIds".
Dockerfile
1# This is a template for a Dockerfile used to run acts in Actor system.
2# The base image name below is set during the act build, based on user settings.
3# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
4FROM apify/actor-node-basic:v0.21.10
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Copy source code to container
23# Do this in the last step, to have fast build if only the source code changed
24COPY . ./
25
26# NOTE: The CMD is already defined by the base image.
27# Uncomment this for local node inspector debugging:
28# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]package.json
1{
2 "name": "apify-project",
3 "version": "0.0.1",
4 "description": "",
5 "author": "It's not you it's me",
6 "license": "ISC",
7 "dependencies": {
8 "apify": "0.21.10",
9 "moment": "latest",
10 "bluebird": "latest",
11 "underscore": "latest"
12 },
13 "scripts": {
14 "start": "node main.js"
15 }
16}main.js
1const Apify = require('apify');
2const moment = require('moment');
3const Promise = require('bluebird');
4const _ = require('underscore');
5
6
7const utils = require('apify-client/build/utils');
8const originalRequestPromise = utils.requestPromise;
9utils.requestPromise = (options, iteration = 0) => {
10 const startedAt = new Date();
11 //console.log(`${options.method} ${iteration} ${options.url} ${options.body ? options.body.url : 'null'}`);
12 return originalRequestPromise(options, iteration)
13 .then((response) => {
14 //console.log(JSON.stringify(response));
15 console.log(`${options.url} ${(Date.now() - startedAt) / 1000}`);
16 return response;
17 });
18};
19
20const MAX_OFFSET_LIMIT = 30000;
21const RUN_FIELDS = [
22 'id',
23 'actId',
24 'startedAt',
25 'finishedAt',
26 'status',
27 'defaultDatasetId',
28 'defaultRequestQueueId',
29 'defaultKeyValueStoreId',
30];
31
32const getRunsFromPeriod = async (actId, from, to) => {
33 const runs = [];
34 let offset = 0;
35
36 while (true) {
37 let allFetched = true;
38 const response = await Apify.client.acts.listRuns({ actId, offset, desc: true });
39
40 response
41 .items
42 .filter(run => !run.finishedAt || run.finishedAt > from)
43 .forEach((run) => {
44 run.actId = actId;
45 runs.push(run);
46 allFetched = false;
47 });
48
49 offset += response.items.length;
50
51 if (!response.items.length || allFetched || offset > MAX_OFFSET_LIMIT) return runs;
52 }
53};
54
55const fetchStorageObjects = async (ids, collection, listMethod) => {
56 const storages = {};
57 let offset = 0;
58
59 while (true) {
60 const response = await Apify.client[collection][listMethod]({ offset, desc: true, unnamed: true });
61
62 response.items.forEach((obj) => {
63 storages[obj.id] = obj;
64 });
65
66 offset += response.items.length;
67
68 if (!response.items.length || _.size(storages) === ids.length || offset > MAX_OFFSET_LIMIT * 1.1) return storages;
69 }
70};
71
72Apify.main(async () => {
73 let input;
74
75 try {
76 input = await Apify.getValue('INPUT');
77 } catch (err) {
78 // Swallow this, input is empty.
79 }
80
81 const to = new Date();
82 const from = moment(to).subtract(1, 'day').toDate();
83 const getRunsFromPeriodPartial = _.partial(getRunsFromPeriod, _, from, to);
84
85 const acts = (await Apify.client.acts.listActs())
86 .items
87 .filter((act) => {
88 if (!input || !input.actIds) return true;
89
90 return input.actIds.includes(act.id);
91 });
92
93 const actNames = _.chain(acts).indexBy('id').mapObject(act => act.name).value();
94 const actIds = acts.map(act => act.id);
95 const runs = _.flatten(await Promise.mapSeries(actIds, getRunsFromPeriodPartial));
96 const datasetIds = _.pluck(runs, 'defaultDatasetId');
97 const queueIds = _.pluck(runs, 'defaultRequestQueueId');
98 const storeIds = _.pluck(runs, 'defaultKeyValueStoreId');
99
100 const queues = await fetchStorageObjects(storeIds, 'requestQueues', 'listQueues');
101 const datasets = await fetchStorageObjects(storeIds, 'datasets', 'listDatasets');
102
103 const actorRuns = runs
104 .map((run) => {
105 const dataset = datasets[run.defaultDatasetId];
106 const queue = queues[run.defaultRequestQueueId];
107
108 return Object.assign(_.pick(run, ...RUN_FIELDS), {
109 actName: actNames[run.actId],
110 datasetItems: dataset ? dataset.itemCount : 'N/A',
111 requestQueueItems: queue ? queue.totalRequestCount : 'N/A',
112 requestQueuePending: queue ? queue.pendingRequestCount : 'N/A',
113 requestQueueHandled: queue ? queue.handledRequestCount : 'N/A',
114 });
115 });
116
117 await Apify.setValue('OUTPUT', {
118 createdAt: to,
119 period: { from, to },
120 actorRuns,
121 });
122});Developer
Marek TrunkátMaintained by Community
Actor Metrics
1 monthly user
-
2 stars
Created in Aug 2018
Modified 2 years ago
Categories
