GitHub Issue Notifier

jkuzz/github-issue-notifier

No credit card required

.dockerignore

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

# dist folder
dist

.gitignore

storage
apify_storage
crawlee_storage
node_modules
dist
tsconfig.tsbuildinfo
storage/*
!storage/key_value_stores
storage/key_value_stores/*
!storage/key_value_stores/default
storage/key_value_stores/default/*
!storage/key_value_stores/default/INPUT.json

# Added by Apify CLI
.venv

package.json

{
  "name": "gh-issue-notifier",
  "version": "0.0.1",
  "type": "module",
  "description": "This is an example of an Apify actor.",
  "engines": {
    "node": ">=18.0.0"
  },
  "dependencies": {
    "@octokit/openapi-types": "^18.0.0",
    "apify": "^3.1.4",
    "axios": "^1.4.0",
    "cheerio": "^1.0.0-rc.12",
    "octokit": "^3.1.0"
  },
  "devDependencies": {
    "@apify/tsconfig": "^0.1.0",
    "@types/node": "^20.5.9",
    "ts-node": "^10.9.1",
    "typescript": "^5.0.4"
  },
  "scripts": {
    "start": "npm run start:dev",
    "start:prod": "node dist/main.js",
    "start:dev": "node --loader ts-node/esm src/main.ts",
    "build": "tsc",
    "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
  },
  "author": "It's not you it's me",
  "license": "ISC"
}

tsconfig.json

{
  "extends": "@apify/tsconfig",
  "compilerOptions": {
    "module": "ES2022",
    "target": "ES2022",
    "outDir": "dist",
    "noUnusedLocals": false,
    "lib": ["DOM"]
  },
  "ts-node": { "esm": true },
  "include": ["./src/**/*"]
}

.actor/Dockerfile

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:18 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node:18

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm run start:prod --silent

.actor/actor.json

{
	"actorSpecification": 1,
	"name": "gh-issue-notifier",
	"title": "Scrape single page in TypeScript",
	"description": "Scrape data from single page with provided URL.",
	"version": "0.0",
	"meta": {
		"templateId": "ts-start"
	},
	"input": "./input_schema.json",
	"dockerfile": "./Dockerfile"
}

.actor/input_schema.json

{
  "title": "Scrape data from a web page",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "searchRepos": {
      "title": "URL or names of repositories",
      "type": "array",
      "description": "The URLs or repository names of repositories you want to monitor.",
      "editor": "stringList",
      "prefill": []
    },
    "searchKeywords": {
      "title": "Keywords to search for",
      "type": "array",
      "description": "The keywords that will be looked for in the titles and bodies of new issues.",
      "editor": "stringList",
      "prefill": []
    },
    "slackToken": {
      "title": "Slack token",
      "type": "string",
      "description": "Slack API token in a format xoxp-xxxxxxxxx-xxxx.",
      "editor": "textfield",
      "isSecret": true
    },
    "slackChannel": {
      "title": "Slack channel",
      "type": "string",
      "description": "Channel where the notification with Github issues information will be sent (e.g. #general)",
      "prefill": "#general",
      "editor": "textfield"
    }
  },
  "required": ["searchRepos", "searchKeywords"]
}

src/hash.ts

import crypto from 'crypto'

/**
 * Create crypto hash using SHA-256.
 * If object includes array fields, those fields are sorted using `.sort()`
 * @param object to hash
 * @returns hash string
 */
export const createObjectDigest = async (object: any) => {
  for (let field in object) {
    if (object[field] instanceof Array) {
      object[field] = object[field].sort()
    }
  }
  const objectAsMessage = JSON.stringify(object)
  return digestMessage(objectAsMessage)
}

async function digestMessage(message: string) {
  const msgUint8 = new TextEncoder().encode(message) // encode as (utf-8) Uint8Array
  const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) // hash the message
  const hashArray = Array.from(new Uint8Array(hashBuffer)) // convert buffer to byte array
  const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
  return hashHex
}

src/main.ts

// Cheerio - The fast, flexible & elegant library for parsing and manipulating HTML and XML (Read more at https://cheerio.js.org/).
import * as cheerio from 'cheerio'
// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/).
import { Actor, log } from 'apify'
import { Octokit } from 'octokit'
import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods'
import { sendSlackNotification } from './slackMessage.js'
import { createSlackMessageBlocks } from './slackBlocks.js'
import { parseRepoNameOrUrl } from './utils.js'
import { createObjectDigest } from './hash.js'

// Extracts element type from array type
export type ArrElement<ArrType> = ArrType extends readonly (infer ElementType)[]
  ? ElementType
  : never
export type Issue = RestEndpointMethodTypes['issues']['listForRepo']['response']

interface Input {
  searchRepos: string[]
  searchKeywords: string[]
  slackToken?: string
  slackChannel?: string
}

const KV_STORE_NAME = 'gh-issues-notifier-state'

/**
 * Checks whether or not the issue is relevant.
 * Looks for search keywords in the title or body of the issue.
 * @param issue to check
 * @param searchKeywords keywords to look for
 * @returns true if the issue is relevant
 */
const checkIssueRelevant = (issue: ArrElement<Issue['data']>, searchKeywords: string[]) => {
  if (!issue || issue.state !== 'open') {
    return false
  }

  for (let keyword of searchKeywords) {
    if (issue.title.indexOf(keyword) > -1) {
      return true
    }
  }

  if (issue.body) {
    const $ = cheerio.load(issue.body)
    for (let keyword of searchKeywords) {
      if ($.text().indexOf(keyword) > -1) {
        return true
      }
    }
  }

  return false
}

/**
 * Discover all issues that are deemed relevant and were not reported yet.
 * Goes through all the provided GH repositories and looks at the issue text
 * @param repoUrls urls of repos to search
 * @param searchKeywords keywords to match in the search
 * @param checkedFilter lambda to filter out issues that were already reported
 * @throws when octokit issues fail to fetch
 * @returns list of repos, each as a list of issues
 */
const findRelevantNewIssues = async (
  repoUrls: string[],
  searchKeywords: string[],
  checkedFilter: (potentialNewIssue: ArrElement<Issue['data']>) => boolean
) => {
  let outputs: Issue['data'] = []
  for (let url of repoUrls) {
    let repoInfo
    try {
      repoInfo = parseRepoNameOrUrl(url)
    } catch (e) {
      if (e instanceof Error) {
        log.error(e.message)
      }
      continue
    }
    const repoIssues = await octokit.rest.issues.listForRepo({
      owner: repoInfo.user,
      repo: repoInfo.repoName,
    })

    const relevantIssues = repoIssues.data
      .filter(checkedFilter)
      .filter((issue) => checkIssueRelevant(issue, searchKeywords))

    if (relevantIssues.length === 0) {
      continue // no relevant issues, next repo
    }
    outputs = outputs.concat(relevantIssues)
  }
  return outputs
}

/**
 * Checks the data store for whether or not the issue was already reported.
 * @param potentialNewIssue Issue to check
 * @param lastCheckedAt timestamp of when the issues were last checked
 * @returns true if issue was not reported yet
 */
const checkIssueNotReported = (
  potentialNewIssue: ArrElement<Issue['data']>,
  lastCheckedAt: number
) => {
  return Date.parse(potentialNewIssue.created_at) > lastCheckedAt
}

const findAndReportIssues = async (input: Input) => {
  const reportedIssuesStore = await Actor.openKeyValueStore(KV_STORE_NAME)
  const actorInstanceHash = await createObjectDigest(input)
  const lastCheckedAt = +((await reportedIssuesStore.getValue(actorInstanceHash)) || '0')
  const currentCheckedAt = Date.now()

  // Closure filter function with last checked timestamp
  const issueAlreadyCheckedFilter = (potentialNewIssue: ArrElement<Issue['data']>) => {
    return checkIssueNotReported(potentialNewIssue, lastCheckedAt)
  }

  let newIssues: Awaited<ReturnType<typeof findRelevantNewIssues>>
  try {
    newIssues = await findRelevantNewIssues(searchRepos, searchKeywords, issueAlreadyCheckedFilter)
  } catch (_e) {
    log.error(
      'Error fetching issues from GitHub. One of your selected repos is likely misspelled or private.'
    )
    return
  }

  log.info(`Saving current actor information under hash [${actorInstanceHash}]`)
  reportedIssuesStore.setValue(actorInstanceHash, currentCheckedAt)

  if (newIssues.length > 0) {
    log.info(`Discovered ${newIssues.length} new issues.`)
  } else {
    log.info('No new issues discovered.')
    return
  }

  if (input.slackChannel && input.slackToken) {
    log.info(`Sending Slack notification with new issues.`)
    const messagePromises = newIssues.map((newIssue) => {
      const slackMessageBlocks = createSlackMessageBlocks(newIssue)
      return sendSlackNotification({
        token: input.slackToken!,
        channel: input.slackChannel!,
        blocks: slackMessageBlocks,
      })
    })
    Promise.all(messagePromises)
  }

  // Save headings to Dataset - a table-like storage.
  await Actor.pushData(newIssues)
}

// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init().
await Actor.init()

// Structure of input is defined in input_schema.json
const input = await Actor.getInput<Input>()
if (!input) throw new Error('Input is missing!')
const { searchRepos, searchKeywords } = input

if (!searchRepos || searchRepos.length === 0) {
  log.error('No repositories provided!')
}
if (!searchKeywords || searchKeywords.length === 0) {
  log.error('No search keywords provided!')
}

const octokit = new Octokit({})

await findAndReportIssues(input)

// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit().
await Actor.exit()

src/slackBlocks.ts

import type { Issue, ArrElement } from './main.js'

/**
 * Create Slack blocks detailing the newly discovered issue.
 * https://api.slack.com/block-kit/building
 * @param newIssue issue to make message for, as returned by octokit
 * @returns Slack message Blocks summarising the new issue
 */
export const createSlackMessageBlocks = (newIssue: ArrElement<Issue['data']>) => {
  let blocks: object[] = createTitleBlocks()
  blocks = blocks.concat(createIssueBlocks(newIssue))
  return blocks
}

/**
 * Create Slack blocks for the title of the Slack message
 * @returns slack blocks
 */
const createTitleBlocks = () => {
  return [
    {
      type: 'header',
      text: {
        type: 'plain_text',
        text: 'New GitHub issue was discovered! 🔎',
        emoji: true,
      },
    },
  ]
}

/**
 * Create Slack blocks for an individual issue.
 * @returns slack blocks
 */
const createIssueBlocks = (issue: ArrElement<Issue['data']>) => {
  return [
    {
      type: 'section',
      fields: [
        {
          type: 'mrkdwn',
          text: `<${issue.url}|${issue.title}>`,
        },
        // {
        //   type: 'mrkdwn',
        //   text: '*Keywords:*\nTODO',
        // },
      ],
    },
    {
      type: 'section',
      fields: [
        {
          type: 'mrkdwn',
          text: `*Created:*\n${issue.created_at}`,
        },
        {
          type: 'mrkdwn',
          text: `*Author:*\n<${issue.user?.url}|${issue.user?.login}>`,
        },
      ],
    },
  ]
}

src/slackMessage.ts

import { ApifyClient } from 'apify'

const SLACK_ACTOR_ID = 'katerinahronik/slack-message'
const MESSAGE_TEXT = 'New GitHub issue was discovered!'

type SlackIntegration = {
  channel: string
  token: string
  blocks: object[]
}

/**
 * Send a slack notification containing the provided message data.
 * Uses the Apify Actor katerinahronik/slack-message
 * @param slackIntegration connection data for the Slack integration
 * @returns promise of slack actor call
 */
export const sendSlackNotification = async (slackIntegration: SlackIntegration) => {
  const apifyClient = new ApifyClient({ token: process.env.APIFY_TOKEN })
  const slackActorClient = apifyClient.actor(SLACK_ACTOR_ID)

  const slackActorInput = {
    token: slackIntegration.token,
    channel: slackIntegration.channel,
    text: MESSAGE_TEXT,
    blocks: slackIntegration.blocks,
  }

  return slackActorClient.call(slackActorInput)
}

src/utils.ts

export const parseRepoNameOrUrl = (nameOrUrl: string) => {
  const repo = nameOrUrl.replace('https://github.com/', '').replace('https://api.github.com/repos/', '')
  const splitRepoName = repo.split('/')
  if (splitRepoName.length !== 2) {
    throw new Error('Invalid repo name provided')
  }
  const [user, repoName] = splitRepoName
  return { user, repoName }
}