# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:18 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node:18

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm run start:prod --silent

.actor/actor.json

{
	"actorSpecification": 1,
	"name": "gh-issue-notifier",
	"title": "Scrape single page in TypeScript",
	"description": "Scrape data from single page with provided URL.",
	"version": "0.0",
	"meta": {
		"templateId": "ts-start"
	},
	"input": "./input_schema.json",
	"dockerfile": "./Dockerfile"
}

.actor/input_schema.json

{
  "title": "Scrape data from a web page",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "searchRepos": {
      "title": "URL or names of repositories",
      "type": "array",
      "description": "The URLs or repository names of repositories you want to monitor.",
      "editor": "stringList",
      "prefill": []
    },
    "searchKeywords": {
      "title": "Keywords to search for",
      "type": "array",
      "description": "The keywords that will be looked for in the titles and bodies of new issues.",
      "editor": "stringList",
      "prefill": []
    },
    "slackToken": {
      "title": "Slack token",
      "type": "string",
      "description": "Slack API token in a format xoxp-xxxxxxxxx-xxxx.",
      "editor": "textfield",
      "isSecret": true
    },
    "slackChannel": {
      "title": "Slack channel",
      "type": "string",
      "description": "Channel where the notification with Github issues information will be sent (e.g. #general)",
      "prefill": "#general",
      "editor": "textfield"
    }
  },
  "required": ["searchRepos", "searchKeywords"]
}

src/hash.ts

1import crypto from 'crypto'
2
3/**
4 * Create crypto hash using SHA-256.
5 * If object includes array fields, those fields are sorted using `.sort()`
6 * @param object to hash
7 * @returns hash string
8 */
9export const createObjectDigest = async (object: any) => {
10  for (let field in object) {
11    if (object[field] instanceof Array) {
12      object[field] = object[field].sort()
13    }
14  }
15  const objectAsMessage = JSON.stringify(object)
16  return digestMessage(objectAsMessage)
17}
18
19async function digestMessage(message: string) {
20  const msgUint8 = new TextEncoder().encode(message) // encode as (utf-8) Uint8Array
21  const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) // hash the message
22  const hashArray = Array.from(new Uint8Array(hashBuffer)) // convert buffer to byte array
23  const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join('') // convert bytes to hex string
24  return hashHex
25}

src/main.ts

1// Cheerio - The fast, flexible & elegant library for parsing and manipulating HTML and XML (Read more at https://cheerio.js.org/).
2import * as cheerio from 'cheerio'
3// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/).
4import { Actor, log } from 'apify'
5import { Octokit } from 'octokit'
6import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods'
7import { sendSlackNotification } from './slackMessage.js'
8import { createSlackMessageBlocks } from './slackBlocks.js'
9import { parseRepoNameOrUrl } from './utils.js'
10import { createObjectDigest } from './hash.js'
11
12// Extracts element type from array type
13export type ArrElement<ArrType> = ArrType extends readonly (infer ElementType)[]
14  ? ElementType
15  : never
16export type Issue = RestEndpointMethodTypes['issues']['listForRepo']['response']
17
18interface Input {
19  searchRepos: string[]
20  searchKeywords: string[]
21  slackToken?: string
22  slackChannel?: string
23}
24
25const KV_STORE_NAME = 'gh-issues-notifier-state'
26
27/**
28 * Checks whether or not the issue is relevant.
29 * Looks for search keywords in the title or body of the issue.
30 * @param issue to check
31 * @param searchKeywords keywords to look for, case is ignored
32 * @returns true if the issue is relevant
33 */
34const checkIssueRelevant = (issue: ArrElement<Issue['data']>, searchKeywords: string[]) => {
35  if (!issue || issue.state !== 'open') {
36    return false
37  }
38
39  const lowerCaseKeywords = searchKeywords.map((keyword: string) => keyword.toLowerCase())
40
41  for (let keyword of lowerCaseKeywords) {
42    if (issue.title.toLowerCase().includes(keyword)) {
43      return true
44    }
45  }
46
47  if (issue.body) {
48    const $ = cheerio.load(issue.body)
49    for (let keyword of lowerCaseKeywords) {
50      if ($.text().toLowerCase().includes(keyword)) {
51        return true
52      }
53    }
54  }
55
56  return false
57}
58
59/**
60 * Discover all issues that are deemed relevant and were not reported yet.
61 * Goes through all the provided GH repositories and looks at the issue text
62 * @param repoUrls urls of repos to search
63 * @param searchKeywords keywords to match in the search
64 * @param checkedFilter lambda to filter out issues that were already reported
65 * @throws when octokit issues fail to fetch
66 * @returns list of repos, each as a list of issues
67 */
68const findRelevantNewIssues = async (
69  repoUrls: string[],
70  searchKeywords: string[],
71  checkedFilter: (potentialNewIssue: ArrElement<Issue['data']>) => boolean
72) => {
73  let outputs: Issue['data'] = []
74  for (let url of repoUrls) {
75    let repoInfo
76    try {
77      repoInfo = parseRepoNameOrUrl(url)
78    } catch (e) {
79      if (e instanceof Error) {
80        log.error(e.message)
81      }
82      continue
83    }
84    const repoIssues = await octokit.rest.issues.listForRepo({
85      owner: repoInfo.user,
86      repo: repoInfo.repoName,
87    })
88
89    const relevantIssues = repoIssues.data
90      .filter(checkedFilter)
91      .filter((issue) => checkIssueRelevant(issue, searchKeywords))
92
93    if (relevantIssues.length === 0) {
94      continue // no relevant issues, next repo
95    }
96    outputs = outputs.concat(relevantIssues)
97  }
98  return outputs
99}
100
101/**
102 * Checks the data store for whether or not the issue was already reported.
103 * @param potentialNewIssue Issue to check
104 * @param lastCheckedAt timestamp of when the issues were last checked
105 * @returns true if issue was not reported yet
106 */
107const checkIssueNotReported = (
108  potentialNewIssue: ArrElement<Issue['data']>,
109  lastCheckedAt: number
110) => {
111  return Date.parse(potentialNewIssue.created_at) > lastCheckedAt
112}
113
114const findAndReportIssues = async (input: Input) => {
115  const reportedIssuesStore = await Actor.openKeyValueStore(KV_STORE_NAME)
116  const actorInstanceHash = await createObjectDigest(input)
117  const lastCheckedAt = +((await reportedIssuesStore.getValue(actorInstanceHash)) || '0')
118  const currentCheckedAt = Date.now()
119
120  // Closure filter function with last checked timestamp
121  const issueAlreadyCheckedFilter = (potentialNewIssue: ArrElement<Issue['data']>) => {
122    return checkIssueNotReported(potentialNewIssue, lastCheckedAt)
123  }
124
125  let newIssues: Awaited<ReturnType<typeof findRelevantNewIssues>>
126  try {
127    newIssues = await findRelevantNewIssues(searchRepos, searchKeywords, issueAlreadyCheckedFilter)
128  } catch (_e) {
129    log.error(
130      'Error fetching issues from GitHub. One of your selected repos is likely misspelled or private.'
131    )
132    return
133  }
134
135  log.info(`Saving current actor information under hash [${actorInstanceHash}]`)
136  reportedIssuesStore.setValue(actorInstanceHash, currentCheckedAt)
137
138  if (newIssues.length > 0) {
139    log.info(`Discovered ${newIssues.length} new issues.`)
140  } else {
141    log.info('No new issues discovered.')
142    return
143  }
144
145  if (input.slackChannel && input.slackToken) {
146    log.info(`Sending Slack notification with new issues.`)
147    const messagePromises = newIssues.map((newIssue) => {
148      const slackMessageBlocks = createSlackMessageBlocks(newIssue)
149      return sendSlackNotification({
150        token: input.slackToken!,
151        channel: input.slackChannel!,
152        blocks: slackMessageBlocks,
153      })
154    })
155    await Promise.all(messagePromises)
156  }
157
158  await Actor.pushData(newIssues)
159}
160
161// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init().
162await Actor.init()
163
164// Structure of input is defined in input_schema.json
165const input = await Actor.getInput<Input>()
166if (!input) throw new Error('Input is missing!')
167const { searchRepos, searchKeywords } = input
168
169if (!searchRepos || searchRepos.length === 0) {
170  log.error('No repositories provided!')
171}
172if (!searchKeywords || searchKeywords.length === 0) {
173  log.error('No search keywords provided!')
174}
175
176const octokit = new Octokit({})
177
178await findAndReportIssues(input)
179
180// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit().
181await Actor.exit()

src/slackBlocks.ts

1import type { Issue, ArrElement } from './main.js'
2
3/**
4 * Create Slack blocks detailing the newly discovered issue.
5 * https://api.slack.com/block-kit/building
6 * @param newIssue issue to make message for, as returned by octokit
7 * @returns Slack message Blocks summarising the new issue
8 */
9export const createSlackMessageBlocks = (newIssue: ArrElement<Issue['data']>) => {
10  let blocks: object[] = createTitleBlocks()
11  blocks = blocks.concat(createIssueBlocks(newIssue))
12  return blocks
13}
14
15/**
16 * Create Slack blocks for the title of the Slack message
17 * @returns slack blocks
18 */
19const createTitleBlocks = () => {
20  return [
21    {
22      type: 'header',
23      text: {
24        type: 'plain_text',
25        text: 'New GitHub issue was discovered! 🔎',
26        emoji: true,
27      },
28    },
29  ]
30}
31
32/**
33 * Create Slack blocks for an individual issue.
34 * @returns slack blocks
35 */
36const createIssueBlocks = (issue: ArrElement<Issue['data']>) => {
37  return [
38    {
39      type: 'section',
40      fields: [
41        {
42          type: 'mrkdwn',
43          text: `<${issue.html_url ?? issue.url}|${issue.title}>`,
44        },
45        // {
46        //   type: 'mrkdwn',
47        //   text: '*Keywords:*\nTODO',
48        // },
49      ],
50    },
51    {
52      type: 'section',
53      fields: [
54        {
55          type: 'mrkdwn',
56          text: `*Created:*\n${issue.created_at}`,
57        },
58        {
59          type: 'mrkdwn',
60          text: `*Author:*\n<${issue.user?.url}|${issue.user?.login}>`,
61        },
62      ],
63    },
64  ]
65}

src/slackMessage.ts

1import { ApifyClient } from 'apify'
2
3const SLACK_ACTOR_ID = 'katerinahronik/slack-message'
4const MESSAGE_TEXT = 'New GitHub issue was discovered!'
5
6type SlackIntegration = {
7  channel: string
8  token: string
9  blocks: object[]
10}
11
12/**
13 * Send a slack notification containing the provided message data.
14 * Uses the Apify Actor katerinahronik/slack-message
15 * @param slackIntegration connection data for the Slack integration
16 * @returns promise of slack actor call
17 */
18export const sendSlackNotification = async (slackIntegration: SlackIntegration) => {
19  const apifyClient = new ApifyClient({ token: process.env.APIFY_TOKEN })
20  const slackActorClient = apifyClient.actor(SLACK_ACTOR_ID)
21
22  const slackActorInput = {
23    token: slackIntegration.token,
24    channel: slackIntegration.channel,
25    text: MESSAGE_TEXT,
26    blocks: slackIntegration.blocks,
27  }
28
29  return slackActorClient.call(slackActorInput)
30}

src/utils.ts

1export const parseRepoNameOrUrl = (nameOrUrl: string) => {
2  const repo = nameOrUrl.replace('https://github.com/', '').replace('https://api.github.com/repos/', '')
3  const splitRepoName = repo.split('/')
4  if (splitRepoName.length !== 2) {
5    throw new Error('Invalid repo name provided')
6  }
7  const [user, repoName] = splitRepoName
8  return { user, repoName }
9}

.dockerignore

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

# dist folder
dist

.gitignore

storage
apify_storage
crawlee_storage
node_modules
dist
tsconfig.tsbuildinfo
storage/*
!storage/key_value_stores
storage/key_value_stores/*
!storage/key_value_stores/default
storage/key_value_stores/default/*
!storage/key_value_stores/default/INPUT.json

# Added by Apify CLI
.venv

package.json

{
  "name": "gh-issue-notifier",
  "version": "0.0.1",
  "type": "module",
  "description": "This is an example of an Apify actor.",
  "engines": {
    "node": ">=18.0.0"
  },
  "dependencies": {
    "@octokit/openapi-types": "^18.0.0",
    "apify": "^3.1.4",
    "axios": "^1.4.0",
    "cheerio": "^1.0.0-rc.12",
    "octokit": "^3.1.0"
  },
  "devDependencies": {
    "@apify/tsconfig": "^0.1.0",
    "@types/node": "^20.5.9",
    "ts-node": "^10.9.1",
    "typescript": "^5.0.4"
  },
  "scripts": {
    "start": "npm run start:dev",
    "start:prod": "node dist/main.js",
    "start:dev": "node --loader ts-node/esm src/main.ts",
    "build": "tsc",
    "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
  },
  "author": "It's not you it's me",
  "license": "ISC"
}

tsconfig.json

{
  "extends": "@apify/tsconfig",
  "compilerOptions": {
    "module": "ES2022",
    "target": "ES2022",
    "outDir": "dist",
    "noUnusedLocals": false,
    "lib": ["DOM"]
  },
  "ts-node": { "esm": true },
  "include": ["./src/**/*"]
}

GitHub Stars

sauain/github-stars

Input will be the URL of any GitHub repository, and output will be GitHub Stars.

Saurav Jain

Github Search Scraper

saswave/github-search-scraper

Github search scraper. Get all data from search results list

SASWAVE

Github Profile Scraper

saswave/github-profile-scraper

GitHub User Profile Scraper. Extracts data from GitHub profiles, including followers, following, LinkedIn, Twitter, achievements and much more. Ideal for developers, researchers, and marketers. From a list of Github profile or a repository stargazers link

SASWAVE

Github Users Scraper

dtrungtin/github-users-scraper

Github Users Scraper is an Apify actor for extracting users or emails from Github. It allows you to extract all watchers, stargazers, and members from a repository page.

Tin Duong

205

4.0

Github User Profile Scraper

powerful_bachelor/Github-User-Profile-Scraper

The GitHub User Profile Scraper extracts vital info from GitHub profiles, including followers, following, LinkedIn, Twitter, achievements and much more. Ideal for developers, researchers, and marketers, it supports multiple profiles and exports data in various formats.

Powerful Bachelor

Github Marketplace Scraper

jungle_synthesizer/github-marketplace-scraper

Scrape GitHub Marketplace listings. Extracts relevant metadata, developer details, pricing information, and even hidden fields like emails.

BowTiedRacoon

Github List Scraper

janbuchar/github-list-scraper

This Actor scrapes repositories from GitHub **Awesome Lists**, **topic listings**, and **individual repositories**, collecting useful metadata for each project.

Jan Buchar

Github Repo User Scraper

inquisitive_sarangi/github-repo-scraper

Github Repo User Scraper is simple tool to extract users of a repo(s) like contributors, stargazers & watchers. You can also export listings to JSON/CSV or any other as format.

API Master

GitHub Repository Scraper

fresh_cliff/github-scraper

This actor scrapes detailed information from GitHub repositories using reliable HTTP requests and HTML parsing. It extracts repository metadata including star counts, fork counts, topics/tags, license information, primary programming language, and last updated timestamps.

Brennan Crawford

Github Repo Markdown Scraper

louisdeconinck/github-repo-markdown-scraper

Transform GitHub repositories into a single, comprehensive markdown document effortlessly. Our tool streamlines analysis and processing, offering configurable file size limits, pattern filtering, and batch processing. Perfect for LLM AI prompts, it handles large repositories with ease.