# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

# dist folder
dist


.dockerignore

storage
apify_storage
crawlee_storage
node_modules
dist
tsconfig.tsbuildinfo
storage/*
!storage/key_value_stores
storage/key_value_stores/*
!storage/key_value_stores/default
storage/key_value_stores/default/*
!storage/key_value_stores/default/INPUT.json

# Added by Apify CLI
.venv


.gitignore

gh-issue-notifier

package.json

tsconfig.json

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:18 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node:18

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm run start:prod --silent


.actor/Dockerfile

Scrape data from single page with provided URL.

Scrape single page in TypeScript

.actor/actor.json

The keywords that will be looked for in the titles and bodies of new issues.

Keywords to search for

The URLs or repository names of repositories you want to monitor.

URL or names of repositories

Channel where the notification with Github issues information will be sent (e.g. #general)

Slack channel

Slack API token in a format xoxp-xxxxxxxxx-xxxx.

Slack token

Scrape data from a web page

.actor/input_schema.json

import crypto from 'crypto'

/**
 * Create crypto hash using SHA-256.
 * If object includes array fields, those fields are sorted using `.sort()`
 * @param object to hash
 * @returns hash string
 */
export const createObjectDigest = async (object: any) => {
  for (let field in object) {
    if (object[field] instanceof Array) {
      object[field] = object[field].sort()
    }
  }
  const objectAsMessage = JSON.stringify(object)
  return digestMessage(objectAsMessage)
}

async function digestMessage(message: string) {
  const msgUint8 = new TextEncoder().encode(message) // encode as (utf-8) Uint8Array
  const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) // hash the message
  const hashArray = Array.from(new Uint8Array(hashBuffer)) // convert buffer to byte array
  const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
  return hashHex
}


src/hash.ts

// Cheerio - The fast, flexible & elegant library for parsing and manipulating HTML and XML (Read more at https://cheerio.js.org/).
import * as cheerio from 'cheerio'
// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/).
import { Actor, log } from 'apify'
import { Octokit } from 'octokit'
import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods'
import { sendSlackNotification } from './slackMessage.js'
import { createSlackMessageBlocks } from './slackBlocks.js'
import { parseRepoNameOrUrl } from './utils.js'
import { createObjectDigest } from './hash.js'

// Extracts element type from array type
export type ArrElement<ArrType> = ArrType extends readonly (infer ElementType)[]
  ? ElementType
  : never
export type Issue = RestEndpointMethodTypes['issues']['listForRepo']['response']

interface Input {
  searchRepos: string[]
  searchKeywords: string[]
  slackToken?: string
  slackChannel?: string
}

const KV_STORE_NAME = 'gh-issues-notifier-state'

/**
 * Checks whether or not the issue is relevant.
 * Looks for search keywords in the title or body of the issue.
 * @param issue to check
 * @param searchKeywords keywords to look for
 * @returns true if the issue is relevant
 */
const checkIssueRelevant = (issue: ArrElement<Issue['data']>, searchKeywords: string[]) => {
  if (!issue || issue.state !== 'open') {
    return false
  }

  for (let keyword of searchKeywords) {
    if (issue.title.indexOf(keyword) > -1) {
      return true
    }
  }

  if (issue.body) {
    const $ = cheerio.load(issue.body)
    for (let keyword of searchKeywords) {
      if ($.text().indexOf(keyword) > -1) {
        return true
      }
    }
  }

  return false
}

/**
 * Discover all issues that are deemed relevant and were not reported yet.
 * Goes through all the provided GH repositories and looks at the issue text
 * @param repoUrls urls of repos to search
 * @param searchKeywords keywords to match in the search
 * @param checkedFilter lambda to filter out issues that were already reported
 * @throws when octokit issues fail to fetch
 * @returns list of repos, each as a list of issues
 */
const findRelevantNewIssues = async (
  repoUrls: string[],
  searchKeywords: string[],
  checkedFilter: (potentialNewIssue: ArrElement<Issue['data']>) => boolean
) => {
  let outputs: Issue['data'] = []
  for (let url of repoUrls) {
    let repoInfo
    try {
      repoInfo = parseRepoNameOrUrl(url)
    } catch (e) {
      if (e instanceof Error) {
        log.error(e.message)
      }
      continue
    }
    const repoIssues = await octokit.rest.issues.listForRepo({
      owner: repoInfo.user,
      repo: repoInfo.repoName,
    })

    const relevantIssues = repoIssues.data
      .filter(checkedFilter)
      .filter((issue) => checkIssueRelevant(issue, searchKeywords))

    if (relevantIssues.length === 0) {
      continue // no relevant issues, next repo
    }
    outputs = outputs.concat(relevantIssues)
  }
  return outputs
}

/**
 * Checks the data store for whether or not the issue was already reported.
 * @param potentialNewIssue Issue to check
 * @param lastCheckedAt timestamp of when the issues were last checked
 * @returns true if issue was not reported yet
 */
const checkIssueNotReported = (
  potentialNewIssue: ArrElement<Issue['data']>,
  lastCheckedAt: number
) => {
  return Date.parse(potentialNewIssue.created_at) > lastCheckedAt
}

const findAndReportIssues = async (input: Input) => {
  const reportedIssuesStore = await Actor.openKeyValueStore(KV_STORE_NAME)
  const actorInstanceHash = await createObjectDigest(input)
  const lastCheckedAt = +((await reportedIssuesStore.getValue(actorInstanceHash)) || '0')
  const currentCheckedAt = Date.now()

  // Closure filter function with last checked timestamp
  const issueAlreadyCheckedFilter = (potentialNewIssue: ArrElement<Issue['data']>) => {
    return checkIssueNotReported(potentialNewIssue, lastCheckedAt)
  }

  let newIssues: Awaited<ReturnType<typeof findRelevantNewIssues>>
  try {
    newIssues = await findRelevantNewIssues(searchRepos, searchKeywords, issueAlreadyCheckedFilter)
  } catch (_e) {
    log.error(
      'Error fetching issues from GitHub. One of your selected repos is likely misspelled or private.'
    )
    return
  }

  log.info(`Saving current actor information under hash [${actorInstanceHash}]`)
  reportedIssuesStore.setValue(actorInstanceHash, currentCheckedAt)

  if (newIssues.length > 0) {
    log.info(`Discovered ${newIssues.length} new issues.`)
  } else {
    log.info('No new issues discovered.')
    return
  }

  if (input.slackChannel && input.slackToken) {
    log.info(`Sending Slack notification with new issues.`)
    const messagePromises = newIssues.map((newIssue) => {
      const slackMessageBlocks = createSlackMessageBlocks(newIssue)
      return sendSlackNotification({
        token: input.slackToken!,
        channel: input.slackChannel!,
        blocks: slackMessageBlocks,
      })
    })
    Promise.all(messagePromises)
  }

  // Save headings to Dataset - a table-like storage.
  await Actor.pushData(newIssues)
}

// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init().
await Actor.init()

// Structure of input is defined in input_schema.json
const input = await Actor.getInput<Input>()
if (!input) throw new Error('Input is missing!')
const { searchRepos, searchKeywords } = input

if (!searchRepos || searchRepos.length === 0) {
  log.error('No repositories provided!')
}
if (!searchKeywords || searchKeywords.length === 0) {
  log.error('No search keywords provided!')
}

const octokit = new Octokit({})

await findAndReportIssues(input)

// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit().
await Actor.exit()


src/main.ts

import type { Issue, ArrElement } from './main.js'

/**
 * Create Slack blocks detailing the newly discovered issue.
 * https://api.slack.com/block-kit/building
 * @param newIssue issue to make message for, as returned by octokit
 * @returns Slack message Blocks summarising the new issue
 */
export const createSlackMessageBlocks = (newIssue: ArrElement<Issue['data']>) => {
  let blocks: object[] = createTitleBlocks()
  blocks = blocks.concat(createIssueBlocks(newIssue))
  return blocks
}

/**
 * Create Slack blocks for the title of the Slack message
 * @returns slack blocks
 */
const createTitleBlocks = () => {
  return [
    {
      type: 'header',
      text: {
        type: 'plain_text',
        text: 'New GitHub issue was discovered! 🔎',
        emoji: true,
      },
    },
  ]
}

/**
 * Create Slack blocks for an individual issue.
 * @returns slack blocks
 */
const createIssueBlocks = (issue: ArrElement<Issue['data']>) => {
  return [
    {
      type: 'section',
      fields: [
        {
          type: 'mrkdwn',
          text: `<${issue.url}|${issue.title}>`,
        },
        // {
        //   type: 'mrkdwn',
        //   text: '*Keywords:*\nTODO',
        // },
      ],
    },
    {
      type: 'section',
      fields: [
        {
          type: 'mrkdwn',
          text: `*Created:*\n${issue.created_at}`,
        },
        {
          type: 'mrkdwn',
          text: `*Author:*\n<${issue.user?.url}|${issue.user?.login}>`,
        },
      ],
    },
  ]
}


src/slackBlocks.ts

import { ApifyClient } from 'apify'

const SLACK_ACTOR_ID = 'katerinahronik/slack-message'
const MESSAGE_TEXT = 'New GitHub issue was discovered!'

type SlackIntegration = {
  channel: string
  token: string
  blocks: object[]
}

/**
 * Send a slack notification containing the provided message data.
 * Uses the Apify Actor katerinahronik/slack-message
 * @param slackIntegration connection data for the Slack integration
 * @returns promise of slack actor call
 */
export const sendSlackNotification = async (slackIntegration: SlackIntegration) => {
  const apifyClient = new ApifyClient({ token: process.env.APIFY_TOKEN })
  const slackActorClient = apifyClient.actor(SLACK_ACTOR_ID)

  const slackActorInput = {
    token: slackIntegration.token,
    channel: slackIntegration.channel,
    text: MESSAGE_TEXT,
    blocks: slackIntegration.blocks,
  }

  return slackActorClient.call(slackActorInput)
}


src/slackMessage.ts

export const parseRepoNameOrUrl = (nameOrUrl: string) => {
  const repo = nameOrUrl.replace('https://github.com/', '').replace('https://api.github.com/repos/', '')
  const splitRepoName = repo.split('/')
  if (splitRepoName.length !== 2) {
    throw new Error('Invalid repo name provided')
  }
  const [user, repoName] = splitRepoName
  return { user, repoName }
}


GitHub Issue Notifier

Where next?