Github Trendings

# This is a template for a Dockerfile used to run acts in Actor system.
# The base image name below is set during the act build, based on user settings.
# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
FROM apify/actor-node-chrome:v0.21.10

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Copy source code to container
# Do this in the last step, to have fast build if only the source code changed
COPY --chown=myuser:myuser . ./

# NOTE: The CMD is already defined by the base image.
# Uncomment this for local node inspector debugging:
# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]

{
    "name": "apify-project",
    "version": "0.0.1",
    "description": "",
    "author": "It's not you it's me",
    "license": "ISC",
    "dependencies": {
        "apify": "0.21.10",
        "mongodb": "latest"
    },
    "scripts": {
        "start": "node main.js"
    }
}

1const Apify = require('apify');
2const MongoClient = require('mongodb').MongoClient;
3
4Apify.main(async () => {
5    const input = await Apify.getValue('INPUT');
6
7    const requestList = new Apify.RequestList({
8        sources: input
9    });
10
11    // This call loads and parses the URLs from the remote file.
12    await requestList.initialize();
13
14    const crawler = new Apify.PuppeteerCrawler({
15        requestList,
16        handlePageFunction: async ({ page, request }) => {
17            await Apify.utils.puppeteer.injectJQuery(page);
18            const reposList = await page.evaluate(() => {
19                const reposList = []
20                $('.Box-row').each(function() {
21                    reposList.push({
22                        name: $(this).find('h1').text().trim(),
23                        description: $(this).find('p').text().trim(),
24                        todayStars: parseInt($(this).find('.float-sm-right').text()),
25                        url: $(this).find('a').get(0).href,
26                    })
27                });
28                return reposList;
29            });
30            await Apify.pushData({ language: request.userData.language, reposList });
31        },
32        handleFailedRequestFunction: async ({ request }) => {
33            // This function is called when the crawling of a request failed too many times
34            await Apify.pushData({
35                url: request.url,
36                succeeded: false,
37                errors: request.errorMessages,
38            })
39        },
40    });
41
42    await crawler.run();
43
44    const dataset = await Apify.openDataset();
45    const data = await dataset.getData({ clean: true });
46    const results = data.items;
47
48    const client = await MongoClient.connect(process.env.MONGO_URL);
49    const db = client.db()
50    const todayCollection = db.collection('trendingToday');
51    for (let result of results) {
52        const language = result.language;
53        for (let repo of result.reposList) {
54            repo.createdAt = new Date();
55            repo.language = language;
56            const repoInCollection = await todayCollection.findOne({ name: repo.name });
57            if (!repoInCollection) await todayCollection.insert(repo);
58        }
59    }
60    await client.close();
61
62
63    await Apify.setValue('OUTPUT', results);
64    console.log('done');
65});