24 Hour Stats
Try for free
No credit card required
Go to Store
24 Hour Stats
mtrunkat/24-hour-stats
Try for free
No credit card required
This act can be used as synchronous API. Returns a JSON containing actor runs finished in the last 24 hours along with information about their default datasets and request queues. Actors might be filtered via input array "actIds".
Developer
Marek TrunkátMaintained by Community
Actor Metrics
1 monthly user
No reviews yet
2 bookmarks
>99% runs succeeded
Created in Aug 2018
Modified 3 years ago
Categories
Dockerfile
1# This is a template for a Dockerfile used to run acts in Actor system.
2# The base image name below is set during the act build, based on user settings.
3# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
4FROM apify/actor-node-basic:v0.21.10
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Copy source code to container
23# Do this in the last step, to have fast build if only the source code changed
24COPY . ./
25
26# NOTE: The CMD is already defined by the base image.
27# Uncomment this for local node inspector debugging:
28# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]package.json
1{
2 "name": "apify-project",
3 "version": "0.0.1",
4 "description": "",
5 "author": "It's not you it's me",
6 "license": "ISC",
7 "dependencies": {
8 "apify": "0.21.10",
9 "moment": "latest",
10 "bluebird": "latest",
11 "underscore": "latest"
12 },
13 "scripts": {
14 "start": "node main.js"
15 }
16}main.js
1const Apify = require('apify');
2const moment = require('moment');
3const Promise = require('bluebird');
4const _ = require('underscore');
5
6
7const utils = require('apify-client/build/utils');
8const originalRequestPromise = utils.requestPromise;
9utils.requestPromise = (options, iteration = 0) => {
10 const startedAt = new Date();
11 //console.log(`${options.method} ${iteration} ${options.url} ${options.body ? options.body.url : 'null'}`);
12 return originalRequestPromise(options, iteration)
13 .then((response) => {
14 //console.log(JSON.stringify(response));
15 console.log(`${options.url} ${(Date.now() - startedAt) / 1000}`);
16 return response;
17 });
18};
19
20const MAX_OFFSET_LIMIT = 30000;
21const RUN_FIELDS = [
22 'id',
23 'actId',
24 'startedAt',
25 'finishedAt',
26 'status',
27 'defaultDatasetId',
28 'defaultRequestQueueId',
29 'defaultKeyValueStoreId',
30];
31
32const getRunsFromPeriod = async (actId, from, to) => {
33 const runs = [];
34 let offset = 0;
35
36 while (true) {
37 let allFetched = true;
38 const response = await Apify.client.acts.listRuns({ actId, offset, desc: true });
39
40 response
41 .items
42 .filter(run => !run.finishedAt || run.finishedAt > from)
43 .forEach((run) => {
44 run.actId = actId;
45 runs.push(run);
46 allFetched = false;
47 });
48
49 offset += response.items.length;
50
51 if (!response.items.length || allFetched || offset > MAX_OFFSET_LIMIT) return runs;
52 }
53};
54
55const fetchStorageObjects = async (ids, collection, listMethod) => {
56 const storages = {};
57 let offset = 0;
58
59 while (true) {
60 const response = await Apify.client[collection][listMethod]({ offset, desc: true, unnamed: true });
61
62 response.items.forEach((obj) => {
63 storages[obj.id] = obj;
64 });
65
66 offset += response.items.length;
67
68 if (!response.items.length || _.size(storages) === ids.length || offset > MAX_OFFSET_LIMIT * 1.1) return storages;
69 }
70};
71
72Apify.main(async () => {
73 let input;
74
75 try {
76 input = await Apify.getValue('INPUT');
77 } catch (err) {
78 // Swallow this, input is empty.
79 }
80
81 const to = new Date();
82 const from = moment(to).subtract(1, 'day').toDate();
83 const getRunsFromPeriodPartial = _.partial(getRunsFromPeriod, _, from, to);
84
85 const acts = (await Apify.client.acts.listActs())
86 .items
87 .filter((act) => {
88 if (!input || !input.actIds) return true;
89
90 return input.actIds.includes(act.id);
91 });
92
93 const actNames = _.chain(acts).indexBy('id').mapObject(act => act.name).value();
94 const actIds = acts.map(act => act.id);
95 const runs = _.flatten(await Promise.mapSeries(actIds, getRunsFromPeriodPartial));
96 const datasetIds = _.pluck(runs, 'defaultDatasetId');
97 const queueIds = _.pluck(runs, 'defaultRequestQueueId');
98 const storeIds = _.pluck(runs, 'defaultKeyValueStoreId');
99
100 const queues = await fetchStorageObjects(storeIds, 'requestQueues', 'listQueues');
101 const datasets = await fetchStorageObjects(storeIds, 'datasets', 'listDatasets');
102
103 const actorRuns = runs
104 .map((run) => {
105 const dataset = datasets[run.defaultDatasetId];
106 const queue = queues[run.defaultRequestQueueId];
107
108 return Object.assign(_.pick(run, ...RUN_FIELDS), {
109 actName: actNames[run.actId],
110 datasetItems: dataset ? dataset.itemCount : 'N/A',
111 requestQueueItems: queue ? queue.totalRequestCount : 'N/A',
112 requestQueuePending: queue ? queue.pendingRequestCount : 'N/A',
113 requestQueueHandled: queue ? queue.handledRequestCount : 'N/A',
114 });
115 });
116
117 await Apify.setValue('OUTPUT', {
118 createdAt: to,
119 period: { from, to },
120 actorRuns,
121 });
122});
