1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29import { Actor } from 'apify';
30import { CheerioCrawler, Dataset } from 'crawlee';
31import { gotScraping } from 'got-scraping';
32
33await Actor.init();
34
35const input = await Actor.getInput();
36const { startUrls, maxRequestsPerRun = 100, proxyConfiguration } = input;
37
38if (!startUrls || startUrls.length === 0) {
39 await Actor.fail('startUrls is required and must be non-empty');
40}
41
42
43const proxy = await Actor.createProxyConfiguration(
44 proxyConfiguration?.useApifyProxy === false
45 ? { ...proxyConfiguration, useApifyProxy: false }
46 : proxyConfiguration || { useApifyProxy: true }
47);
48
49const BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
50
51
52
53
54
55async function fetchUrl(url, { headers = {}, crawlerContext } = {}) {
56 if (crawlerContext?.sendRequest) {
57 return crawlerContext.sendRequest({ url, headers });
58 }
59 return gotScraping({
60 url,
61 responseType: 'text',
62 headers: { 'user-agent': BROWSER_UA, ...headers },
63 });
64}
65
66
67
68
69function parseJson(str) {
70 return JSON.parse(str);
71}
72
73
74
75
76
77function eventsToTranscript(events) {
78 if (!Array.isArray(events)) return '';
79 return events
80 .filter((e) => Array.isArray(e.segs))
81 .map((e) => e.segs
82 .map((s) => (typeof s.utf8 === 'string' ? s.utf8 : ''))
83 .join('')
84 .replace(/\s+/g, ' ')
85 .trim()
86 )
87 .filter(Boolean)
88 .join(' ')
89 .replace(/\s+/g, ' ')
90 .trim();
91}
92
93
94
95
96
97function srv3XmlToTranscript(xml) {
98 if (typeof xml !== 'string' || !xml.trim().startsWith('<')) return '';
99 const out = [];
100 const re = /<p\b[^>]*>([\s\S]*?)<\/p>/g;
101 let m;
102 while ((m = re.exec(xml)) !== null) {
103 const inside = m[1]
104 .replace(/<[^>]+>/g, '')
105 .replace(/'/g, "'")
106 .replace(/"/g, '"')
107 .replace(/&/g, '&')
108 .replace(/</g, '<')
109 .replace(/>/g, '>')
110 .replace(/\s+/g, ' ')
111 .trim();
112 if (inside) out.push(inside);
113 }
114 return out.join(' ').replace(/\s+/g, ' ').trim();
115}
116
117
118
119
120function pickTrack(tracks) {
121 if (!Array.isArray(tracks) || tracks.length === 0) return null;
122 return tracks.find((t) => (t.languageCode || '').startsWith('en')) || tracks[0];
123}
124
125
126
127
128
129
130async function innertubeAndroidTracks(videoId, { html, crawlerContext, log }) {
131
132 let apiKey;
133 if (html) {
134 const m = html.match(/"INNERTUBE_API_KEY"\s*:\s*"([^"]+)"/);
135 if (m) apiKey = m[1];
136 }
137 if (!apiKey) {
138 log?.info(`Fetching fresh watch page for InnerTube key for ${videoId}`);
139 const watch = await fetchUrl(`https://www.youtube.com/watch?v=${videoId}`, { crawlerContext });
140 const wHtml = typeof watch.body === 'string' ? watch.body : '';
141 const m = wHtml.match(/"INNERTUBE_API_KEY"\s*:\s*"([^"]+)"/);
142 if (m) apiKey = m[1];
143 }
144 if (!apiKey) return { tracks: null };
145
146
147 const endpoint = `https://www.youtube.com/youtubei/v1/player?key=${apiKey}`;
148 const player = await gotScraping({
149 url: endpoint,
150 method: 'POST',
151 headers: {
152 'content-type': 'application/json',
153 'user-agent': BROWSER_UA,
154 'accept-language': 'en-US,en;q=0.9',
155 },
156 body: JSON.stringify({
157 context: {
158 client: {
159 clientName: 'ANDROID',
160 clientVersion: '20.10.38',
161 androidSdkVersion: 34,
162 hl: 'en',
163 gl: 'US',
164 },
165 },
166 videoId,
167 }),
168 responseType: 'json',
169
170 retry: { limit: 0 },
171 });
172
173 if (!player || player.statusCode !== 200) return { tracks: null };
174 const d = player.body;
175 const tracks = d?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
176 if (!Array.isArray(tracks) || tracks.length === 0) return { tracks: null };
177 return { tracks, apiKey };
178}
179
180
181
182
183
184async function fetchTranscript(track, { crawlerContext }) {
185 const sep = track.baseUrl.includes('?') ? '&' : '?';
186 const urlJson = `${track.baseUrl}${sep}fmt=json3`;
187 const r = await fetchUrl(urlJson, { crawlerContext });
188 if (r?.statusCode !== 200) return { text: null };
189 const body = typeof r.body === 'string' ? r.body : '';
190 if (body.length === 0) {
191
192 const r2 = await fetchUrl(track.baseUrl, { crawlerContext });
193 if (r2?.statusCode !== 200) return { text: null };
194 const xml = typeof r2.body === 'string' ? r2.body : '';
195 const text = srv3XmlToTranscript(xml);
196 return text ? { text, format: 'srv3' } : { text: null };
197 }
198
199 try {
200 const parsed = parseJson(body);
201 const events = Array.isArray(parsed?.events) ? parsed.events : [];
202 const text = eventsToTranscript(events);
203 return text ? { text, format: 'json3' } : { text: null };
204 } catch {
205
206 const text = srv3XmlToTranscript(body);
207 return text ? { text, format: 'srv3' } : { text: null };
208 }
209}
210
211
212
213
214async function fetchTranscriptFromSpecPath({ html, crawlerContext }) {
215 const iprMatch = html.match(/ytInitialPlayerResponse\s*=\s*(\{[^\n]*?\});/);
216 if (!iprMatch) return null;
217 let tracks;
218 try {
219 const pr = parseJson(iprMatch[1]);
220 tracks = pr?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
221 } catch {
222 return null;
223 }
224 const chosen = pickTrack(tracks);
225 if (!chosen) return null;
226 const { text } = await fetchTranscript(chosen, { crawlerContext });
227 const gated = /\bexp=xpe\b/.test(chosen.baseUrl);
228 return { chosen, text, gated };
229}
230
231
232
233
234async function fetchTranscriptFromInnerTube(videoId, { html, crawlerContext, log }) {
235 if (!videoId) return null;
236 const { tracks } = await innertubeAndroidTracks(videoId, { html, crawlerContext, log });
237 const chosen = pickTrack(tracks);
238 if (!chosen) return null;
239 const { text } = await fetchTranscript(chosen, { crawlerContext });
240 return { chosen, text };
241}
242
243const crawler = new CheerioCrawler({
244 proxyConfiguration: proxy,
245 maxRequestsPerMinute: 60,
246 requestHandler: async (ctx) => {
247 const { request, $, log } = ctx;
248 const url = request.url;
249 const videoId = new URL(url).searchParams.get('v') || null;
250 const html = $.html();
251 const title = (
252 $('head > meta[property="og:title"]').attr('content')
253 || $('title').first().text()
254 || ''
255 ).trim();
256
257 let result = await fetchTranscriptFromSpecPath({ html, crawlerContext: ctx });
258 let source = 'spec';
259 if (result && (result.gated || (!result.text && result.chosen))) {
260 log.info(`Falling back to InnerTube Android for ${videoId} (gated=${result.gated})`);
261 result = await fetchTranscriptFromInnerTube(videoId, { html, crawlerContext: ctx, log });
262 source = 'innertube_android';
263 }
264
265 const record = {
266 url, videoId, title,
267 language: result?.chosen?.languageCode || null,
268 transcript: result?.text || null,
269 scrapedAt: new Date().toISOString(),
270 };
271 if (!result?.text) record.error = 'no_captions';
272 record.source = source;
273 await Dataset.pushData(record);
274 },
275 failedRequestHandler: async ({ request, log }) => {
276 log.error(`Request failed: ${request.url}`);
277 },
278});
279
280await crawler.run(startUrls.slice(0, maxRequestsPerRun));
281await Actor.exit();
