GitHub Issue Notifier

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:18 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node:18

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm run start:prod --silent

{
	"actorSpecification": 1,
	"name": "gh-issue-notifier",
	"title": "Scrape single page in TypeScript",
	"description": "Scrape data from single page with provided URL.",
	"version": "0.0",
	"meta": {
		"templateId": "ts-start"
	},
	"input": "./input_schema.json",
	"dockerfile": "./Dockerfile"
}

{
  "title": "Scrape data from a web page",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "searchRepos": {
      "title": "URL or names of repositories",
      "type": "array",
      "description": "The URLs or repository names of repositories you want to monitor.",
      "editor": "stringList",
      "prefill": []
    },
    "searchKeywords": {
      "title": "Keywords to search for",
      "type": "array",
      "description": "The keywords that will be looked for in the titles and bodies of new issues.",
      "editor": "stringList",
      "prefill": []
    },
    "slackToken": {
      "title": "Slack token",
      "type": "string",
      "description": "Slack API token in a format xoxp-xxxxxxxxx-xxxx.",
      "editor": "textfield",
      "isSecret": true
    },
    "slackChannel": {
      "title": "Slack channel",
      "type": "string",
      "description": "Channel where the notification with Github issues information will be sent (e.g. #general)",
      "prefill": "#general",
      "editor": "textfield"
    }
  },
  "required": ["searchRepos", "searchKeywords"]
}

1import crypto from 'crypto'
2
3/**
4 * Create crypto hash using SHA-256.
5 * If object includes array fields, those fields are sorted using `.sort()`
6 * @param object to hash
7 * @returns hash string
8 */
9export const createObjectDigest = async (object: any) => {
10  for (let field in object) {
11    if (object[field] instanceof Array) {
12      object[field] = object[field].sort()
13    }
14  }
15  const objectAsMessage = JSON.stringify(object)
16  return digestMessage(objectAsMessage)
17}
18
19async function digestMessage(message: string) {
20  const msgUint8 = new TextEncoder().encode(message) // encode as (utf-8) Uint8Array
21  const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) // hash the message
22  const hashArray = Array.from(new Uint8Array(hashBuffer)) // convert buffer to byte array
23  const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
24  return hashHex
25}

1// Cheerio - The fast, flexible & elegant library for parsing and manipulating HTML and XML (Read more at https://cheerio.js.org/).
2import * as cheerio from 'cheerio'
3// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/).
4import { Actor, log } from 'apify'
5import { Octokit } from 'octokit'
6import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods'
7import { sendSlackNotification } from './slackMessage.js'
8import { createSlackMessageBlocks } from './slackBlocks.js'
9import { parseRepoNameOrUrl } from './utils.js'
10import { createObjectDigest } from './hash.js'
11
12// Extracts element type from array type
13export type ArrElement<ArrType> = ArrType extends readonly (infer ElementType)[]
14  ? ElementType
15  : never
16export type Issue = RestEndpointMethodTypes['issues']['listForRepo']['response']
17
18interface Input {
19  searchRepos: string[]
20  searchKeywords: string[]
21  slackToken?: string
22  slackChannel?: string
23}
24
25const KV_STORE_NAME = 'gh-issues-notifier-state'
26
27/**
28 * Checks whether or not the issue is relevant.
29 * Looks for search keywords in the title or body of the issue.
30 * @param issue to check
31 * @param searchKeywords keywords to look for, case is ignored
32 * @returns true if the issue is relevant
33 */
34const checkIssueRelevant = (issue: ArrElement<Issue['data']>, searchKeywords: string[]) => {
35  if (!issue || issue.state !== 'open') {
36    return false
37  }
38
39  const lowerCaseKeywords = searchKeywords.map((keyword: string) => keyword.toLowerCase())
40
41  for (let keyword of lowerCaseKeywords) {
42    if (issue.title.toLowerCase().includes(keyword)) {
43      return true
44    }
45  }
46
47  if (issue.body) {
48    const $ = cheerio.load(issue.body)
49    for (let keyword of lowerCaseKeywords) {
50      if ($.text().toLowerCase().includes(keyword)) {
51        return true
52      }
53    }
54  }
55
56  return false
57}
58
59/**
60 * Discover all issues that are deemed relevant and were not reported yet.
61 * Goes through all the provided GH repositories and looks at the issue text
62 * @param repoUrls urls of repos to search
63 * @param searchKeywords keywords to match in the search
64 * @param checkedFilter lambda to filter out issues that were already reported
65 * @throws when octokit issues fail to fetch
66 * @returns list of repos, each as a list of issues
67 */
68const findRelevantNewIssues = async (
69  repoUrls: string[],
70  searchKeywords: string[],
71  checkedFilter: (potentialNewIssue: ArrElement<Issue['data']>) => boolean
72) => {
73  let outputs: Issue['data'] = []
74  for (let url of repoUrls) {
75    let repoInfo
76    try {
77      repoInfo = parseRepoNameOrUrl(url)
78    } catch (e) {
79      if (e instanceof Error) {
80        log.error(e.message)
81      }
82      continue
83    }
84    const repoIssues = await octokit.rest.issues.listForRepo({
85      owner: repoInfo.user,
86      repo: repoInfo.repoName,
87    })
88
89    const relevantIssues = repoIssues.data
90      .filter(checkedFilter)
91      .filter((issue) => checkIssueRelevant(issue, searchKeywords))
92
93    if (relevantIssues.length === 0) {
94      continue // no relevant issues, next repo
95    }
96    outputs = outputs.concat(relevantIssues)
97  }
98  return outputs
99}
100
101/**
102 * Checks the data store for whether or not the issue was already reported.
103 * @param potentialNewIssue Issue to check
104 * @param lastCheckedAt timestamp of when the issues were last checked
105 * @returns true if issue was not reported yet
106 */
107const checkIssueNotReported = (
108  potentialNewIssue: ArrElement<Issue['data']>,
109  lastCheckedAt: number
110) => {
111  return Date.parse(potentialNewIssue.created_at) > lastCheckedAt
112}
113
114const findAndReportIssues = async (input: Input) => {
115  const reportedIssuesStore = await Actor.openKeyValueStore(KV_STORE_NAME)
116  const actorInstanceHash = await createObjectDigest(input)
117  const lastCheckedAt = +((await reportedIssuesStore.getValue(actorInstanceHash)) || '0')
118  const currentCheckedAt = Date.now()
119
120  // Closure filter function with last checked timestamp
121  const issueAlreadyCheckedFilter = (potentialNewIssue: ArrElement<Issue['data']>) => {
122    return checkIssueNotReported(potentialNewIssue, lastCheckedAt)
123  }
124
125  let newIssues: Awaited<ReturnType<typeof findRelevantNewIssues>>
126  try {
127    newIssues = await findRelevantNewIssues(searchRepos, searchKeywords, issueAlreadyCheckedFilter)
128  } catch (_e) {
129    log.error(
130      'Error fetching issues from GitHub. One of your selected repos is likely misspelled or private.'
131    )
132    return
133  }
134
135  log.info(`Saving current actor information under hash [${actorInstanceHash}]`)
136  reportedIssuesStore.setValue(actorInstanceHash, currentCheckedAt)
137
138  if (newIssues.length > 0) {
139    log.info(`Discovered ${newIssues.length} new issues.`)
140  } else {
141    log.info('No new issues discovered.')
142    return
143  }
144
145  if (input.slackChannel && input.slackToken) {
146    log.info(`Sending Slack notification with new issues.`)
147    const messagePromises = newIssues.map((newIssue) => {
148      const slackMessageBlocks = createSlackMessageBlocks(newIssue)
149      return sendSlackNotification({
150        token: input.slackToken!,
151        channel: input.slackChannel!,
152        blocks: slackMessageBlocks,
153      })
154    })
155    await Promise.all(messagePromises)
156  }
157
158  await Actor.pushData(newIssues)
159}
160
161// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init().
162await Actor.init()
163
164// Structure of input is defined in input_schema.json
165const input = await Actor.getInput<Input>()
166if (!input) throw new Error('Input is missing!')
167const { searchRepos, searchKeywords } = input
168
169if (!searchRepos || searchRepos.length === 0) {
170  log.error('No repositories provided!')
171}
172if (!searchKeywords || searchKeywords.length === 0) {
173  log.error('No search keywords provided!')
174}
175
176const octokit = new Octokit({})
177
178await findAndReportIssues(input)
179
180// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit().
181await Actor.exit()

1import type { Issue, ArrElement } from './main.js'
2
3/**
4 * Create Slack blocks detailing the newly discovered issue.
5 * https://api.slack.com/block-kit/building
6 * @param newIssue issue to make message for, as returned by octokit
7 * @returns Slack message Blocks summarising the new issue
8 */
9export const createSlackMessageBlocks = (newIssue: ArrElement<Issue['data']>) => {
10  let blocks: object[] = createTitleBlocks()
11  blocks = blocks.concat(createIssueBlocks(newIssue))
12  return blocks
13}
14
15/**
16 * Create Slack blocks for the title of the Slack message
17 * @returns slack blocks
18 */
19const createTitleBlocks = () => {
20  return [
21    {
22      type: 'header',
23      text: {
24        type: 'plain_text',
25        text: 'New GitHub issue was discovered! 🔎',
26        emoji: true,
27      },
28    },
29  ]
30}
31
32/**
33 * Create Slack blocks for an individual issue.
34 * @returns slack blocks
35 */
36const createIssueBlocks = (issue: ArrElement<Issue['data']>) => {
37  return [
38    {
39      type: 'section',
40      fields: [
41        {
42          type: 'mrkdwn',
43          text: `<${issue.html_url ?? issue.url}|${issue.title}>`,
44        },
45        // {
46        //   type: 'mrkdwn',
47        //   text: '*Keywords:*\nTODO',
48        // },
49      ],
50    },
51    {
52      type: 'section',
53      fields: [
54        {
55          type: 'mrkdwn',
56          text: `*Created:*\n${issue.created_at}`,
57        },
58        {
59          type: 'mrkdwn',
60          text: `*Author:*\n<${issue.user?.url}|${issue.user?.login}>`,
61        },
62      ],
63    },
64  ]
65}

1import { ApifyClient } from 'apify'
2
3const SLACK_ACTOR_ID = 'katerinahronik/slack-message'
4const MESSAGE_TEXT = 'New GitHub issue was discovered!'
5
6type SlackIntegration = {
7  channel: string
8  token: string
9  blocks: object[]
10}
11
12/**
13 * Send a slack notification containing the provided message data.
14 * Uses the Apify Actor katerinahronik/slack-message
15 * @param slackIntegration connection data for the Slack integration
16 * @returns promise of slack actor call
17 */
18export const sendSlackNotification = async (slackIntegration: SlackIntegration) => {
19  const apifyClient = new ApifyClient({ token: process.env.APIFY_TOKEN })
20  const slackActorClient = apifyClient.actor(SLACK_ACTOR_ID)
21
22  const slackActorInput = {
23    token: slackIntegration.token,
24    channel: slackIntegration.channel,
25    text: MESSAGE_TEXT,
26    blocks: slackIntegration.blocks,
27  }
28
29  return slackActorClient.call(slackActorInput)
30}

1export const parseRepoNameOrUrl = (nameOrUrl: string) => {
2  const repo = nameOrUrl.replace('https://github.com/', '').replace('https://api.github.com/repos/', '')
3  const splitRepoName = repo.split('/')
4  if (splitRepoName.length !== 2) {
5    throw new Error('Invalid repo name provided')
6  }
7  const [user, repoName] = splitRepoName
8  return { user, repoName }
9}

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

# dist folder
dist

storage
apify_storage
crawlee_storage
node_modules
dist
tsconfig.tsbuildinfo
storage/*
!storage/key_value_stores
storage/key_value_stores/*
!storage/key_value_stores/default
storage/key_value_stores/default/*
!storage/key_value_stores/default/INPUT.json

# Added by Apify CLI
.venv

{
  "name": "gh-issue-notifier",
  "version": "0.0.1",
  "type": "module",
  "description": "This is an example of an Apify actor.",
  "engines": {
    "node": ">=18.0.0"
  },
  "dependencies": {
    "@octokit/openapi-types": "^18.0.0",
    "apify": "^3.1.4",
    "axios": "^1.4.0",
    "cheerio": "^1.0.0-rc.12",
    "octokit": "^3.1.0"
  },
  "devDependencies": {
    "@apify/tsconfig": "^0.1.0",
    "@types/node": "^20.5.9",
    "ts-node": "^10.9.1",
    "typescript": "^5.0.4"
  },
  "scripts": {
    "start": "npm run start:dev",
    "start:prod": "node dist/main.js",
    "start:dev": "node --loader ts-node/esm src/main.ts",
    "build": "tsc",
    "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
  },
  "author": "It's not you it's me",
  "license": "ISC"
}

{
  "extends": "@apify/tsconfig",
  "compilerOptions": {
    "module": "ES2022",
    "target": "ES2022",
    "outDir": "dist",
    "noUnusedLocals": false,
    "lib": ["DOM"]
  },
  "ts-node": { "esm": true },
  "include": ["./src/**/*"]
}