1"""GitHub Repository Search & Scraper — Apify Actor.
2
3Uses the official GitHub REST API v3 search endpoint:
4 https://api.github.com/search/repositories
5
6Field shape verified against a live API response. Relevant per-item fields:
7 full_name, name, description, html_url, stargazers_count, forks_count,
8 open_issues_count, language, topics, license{spdx_id,name},
9 owner{login,type,html_url}, created_at, updated_at, pushed_at, homepage,
10 archived, fork, default_branch, size
11NOTE: the search endpoint's watchers_count is an alias of stargazers_count; the
12real subscriber count (subscribers_count) is NOT in search results, so no
13watchers field is emitted.
14
15Auth: optional GitHub token. Unauthenticated search is limited to ~10 req/min;
16authenticated is ~30 req/min and 5000 req/hr core. The user supplies their own
17token as a secret input — we never store or transmit it anywhere but GitHub.
18"""
19
20from __future__ import annotations
21
22import asyncio
23from urllib.parse import urlencode
24
25import httpx
26from apify import Actor
27
28SEARCH_URL = "https://api.github.com/search/repositories"
29GITHUB_MAX_RESULTS = 1000
30PAGE_SIZE = 100
31
32
33def build_search_q(actor_input: dict) -> str:
34 """Assemble the GitHub search `q` string from structured inputs.
35
36 GitHub uses qualifiers like `language:python stars:>100 topic:cli`.
37 """
38 parts: list[str] = []
39
40 query = (actor_input.get("query") or "").strip()
41 if query:
42 parts.append(query)
43
44 language = (actor_input.get("language") or "").strip()
45 if language:
46 parts.append(f"language:{language}")
47
48 topic = (actor_input.get("topic") or "").strip()
49 if topic:
50
51 for t in (x.strip() for x in topic.split(",") if x.strip()):
52 parts.append(f"topic:{t}")
53
54 min_stars = actor_input.get("minStars")
55 if isinstance(min_stars, int):
56 parts.append(f"stars:>={min_stars}")
57
58 user = (actor_input.get("user") or "").strip()
59 if user:
60 parts.append(f"user:{user}")
61
62 pushed_after = (actor_input.get("pushedAfter") or "").strip()
63 if pushed_after:
64 parts.append(f"pushed:>={pushed_after}")
65
66 created_after = (actor_input.get("createdAfter") or "").strip()
67 if created_after:
68 parts.append(f"created:>={created_after}")
69
70 if actor_input.get("excludeForks"):
71 parts.append("fork:false")
72 if actor_input.get("excludeArchived"):
73 parts.append("archived:false")
74
75 return " ".join(parts).strip()
76
77
78def transform_repo(repo: dict) -> dict:
79 """Convert a raw GitHub repo object into a clean, stable output record."""
80 owner = repo.get("owner") or {}
81 license_info = repo.get("license") or {}
82 return {
83 "fullName": repo.get("full_name"),
84 "name": repo.get("name"),
85 "description": repo.get("description"),
86 "url": repo.get("html_url"),
87 "homepage": repo.get("homepage"),
88 "owner": owner.get("login"),
89 "ownerType": owner.get("type"),
90 "ownerUrl": owner.get("html_url"),
91 "stars": repo.get("stargazers_count"),
92 "forks": repo.get("forks_count"),
93 "openIssues": repo.get("open_issues_count"),
94 "language": repo.get("language"),
95 "topics": repo.get("topics") or [],
96 "license": license_info.get("spdx_id"),
97 "licenseName": license_info.get("name"),
98 "isFork": repo.get("fork"),
99 "isArchived": repo.get("archived"),
100 "defaultBranch": repo.get("default_branch"),
101 "sizeKb": repo.get("size"),
102 "createdAt": repo.get("created_at"),
103 "updatedAt": repo.get("updated_at"),
104 "pushedAt": repo.get("pushed_at"),
105 }
106
107
108async def main() -> None:
109 async with Actor:
110 actor_input = await Actor.get_input() or {}
111
112 sort = actor_input.get("sort", "stars")
113 order = actor_input.get("order", "desc")
114 try:
115 max_items = int(actor_input.get("maxItems") or 100)
116 except (TypeError, ValueError):
117 Actor.log.warning("Invalid maxItems value; using the default of 100.")
118 max_items = 100
119 max_items = max(1, max_items)
120 token = (actor_input.get("githubToken") or "").strip()
121
122 q = build_search_q(actor_input)
123 if not q:
124 Actor.log.warning(
125 "No search criteria provided. Add a query, language, topic, user, or filter."
126 )
127 await Actor.push_data([])
128 return
129
130 headers = {
131 "Accept": "application/vnd.github+json",
132 "User-Agent": "scrapeworks-github-search/0.1",
133 "X-GitHub-Api-Version": "2022-11-28",
134 }
135 if token:
136 headers["Authorization"] = f"Bearer {token}"
137 Actor.log.info("Using authenticated GitHub requests (higher rate limit).")
138 else:
139 Actor.log.info(
140 "Running unauthenticated (low rate limit ~10 req/min). "
141 "Add a free GitHub token in the input for 5000 req/hr."
142 )
143
144
145 target = min(max_items, GITHUB_MAX_RESULTS)
146 Actor.log.info(f"GitHub search q={q!r} sort={sort} order={order} maxItems={target}")
147
148 pushed = 0
149 page = 1
150 total_count: int | None = None
151
152 async with httpx.AsyncClient(timeout=30.0, headers=headers) as client:
153 while pushed < target:
154 params = {
155 "q": q,
156 "sort": sort,
157 "order": order,
158 "per_page": min(PAGE_SIZE, target - pushed),
159 "page": page,
160 }
161
162 if sort == "best-match":
163 params.pop("sort")
164 params.pop("order")
165
166 url = f"{SEARCH_URL}?{urlencode(params)}"
167
168 data = None
169 for attempt in range(1, 4):
170 try:
171 resp = await client.get(url)
172 if resp.status_code == 429 or (
173 resp.status_code == 403 and "rate limit" in resp.text.lower()
174 ):
175 wait = attempt * 10
176 Actor.log.warning(
177 f"Rate limited. Backing off {wait}s "
178 f"(add a token to avoid this)..."
179 )
180 await asyncio.sleep(wait)
181 continue
182 if resp.status_code == 422:
183 Actor.log.error(
184 f"GitHub rejected the query (422). Check your filters: {resp.text[:300]}"
185 )
186 return
187 resp.raise_for_status()
188 data = resp.json()
189 break
190 except (httpx.HTTPError, ValueError) as exc:
191 Actor.log.warning(f"Request attempt {attempt} failed: {exc}")
192 if attempt < 3:
193 await asyncio.sleep(attempt * 3)
194
195 if data is None:
196 Actor.log.error(f"Failed to fetch page {page}; stopping.")
197 break
198
199 if total_count is None:
200 total_count = data.get("total_count", 0)
201 Actor.log.info(f"GitHub reports {total_count} matching repositories.")
202
203 items = data.get("items", [])
204 if not items:
205 Actor.log.info("No more results.")
206 break
207
208 batch = []
209 for repo in items:
210 if pushed >= target:
211 break
212 batch.append(transform_repo(repo))
213 pushed += 1
214
215 if batch:
216
217
218
219 await Actor.push_data(batch)
220 Actor.log.info(f"Pushed {pushed}/{target} repositories (page {page}).")
221
222 page += 1
223
224 if total_count is not None and pushed >= total_count:
225 break
226
227 Actor.log.info(f"Done. Returned {pushed} repositories.")
