Actor picture

Stack Overflow Search Scraper

drobnikj/stackoverflow-search-to-slack

Searches on StackOverflow site for defined queries and sends them to your slack channel. Create Slack app, set up App name and Slack channel as a workspace, set up name in Bot Users, copy OAuth Access Token.

No credit card required

Author's avatarJakub Drobník
  • Modified
  • Users6
  • Runs455
Actor picture

Stack Overflow Search Scraper

Based on the apify/actor-node-puppeteer Docker image (see docs).

const Apify = require('apify');
const Slack = require('slack');

Apify.main(async () => {
    // Get input of your act
    const input = await Apify.getValue('INPUT');
    console.log('My input:');
    console.dir(input);
    
    const slackBotToken = input.slackBotToken || process.env.SLACK_BOT_TOKEN;
   
    if (!input) throw new Error('Invalid input, must be JSON object!');
    if (!slackBotToken) throw new Error('Invalid input, slackBotToken is missing in input or env var!');
    if (!input.searchStrings) throw new Error('Invalid input, searchStrings is missing!');
    
    // Crawl pages for data
    console.log('Launching Puppeteer...');
    const browser = await Apify.launchPuppeteer();
    const results = [];

    for (const search of input.searchStrings) {
        const url = `https://stackoverflow.com/search?q=${encodeURIComponent(search)}`;
        console.log(`Opening URL: ${url}`);
        const page = await browser.newPage();
        await page.goto(url);
        const questions = await page.$$eval('.search-result', (results) => {
            return results.map(el => `- <${el.querySelector('.result-link a').href}|${el.querySelector('.result-link a').innerText}>`).join('\n');
        });
        if (questions) results.push({ title: `search: ${search}`, text: questions })
        await new Promise(resolve => setTimeout(resolve, 5000))
    }
    
    // Send results to slack
    if (results.length) {
        const bot = new Slack({ token: slackBotToken });
        await bot.chat.postMessage({
            channel: input.channel,
            response_type: 'in_channel',
            text: 'Interesting stackoverflow questions:',
            attachments: results,
        });
    }
    
    console.log('My output:');
    console.dir(results);
    await Apify.setValue('OUTPUT', results);
});