1
2
3
4
5
6
7
8
9import { Actor } from 'apify';
10import { execSync } from 'node:child_process';
11import {
12 createWriteStream,
13 mkdtempSync,
14 readFileSync,
15 statSync,
16 writeFileSync,
17 rmSync,
18} from 'node:fs';
19import { randomBytes } from 'node:crypto';
20import { tmpdir } from 'node:os';
21import { join, extname } from 'node:path';
22import { setTimeout as sleep } from 'node:timers/promises';
23import { Writable } from 'node:stream';
24import { safeFilename, mimeType, formatDuration, formatBytes } from './utils.js';
25
26
27
28
29
30
31
32
33async function withHeartbeat(promiseOrFn, msg, { intervalMs = 15_000, abortSignal } = {}) {
34 let done = false;
35
36 let value;
37 try {
38 value = typeof promiseOrFn === 'function' ? promiseOrFn() : promiseOrFn;
39 } catch (e) {
40 return Promise.reject(e);
41 }
42
43 const promise = Promise.resolve(value);
44 const result = promise.then(
45 (v) => { done = true; return v; },
46 (e) => { done = true; throw e; },
47 );
48
49 const hb = (async () => {
50 let count = 0;
51 while (!done) {
52 await sleep(intervalMs);
53 if (done) break;
54 count++;
55 const elapsed = count * (intervalMs / 1000);
56 console.log(` ${msg} (still working... ${elapsed}s elapsed)`);
57 }
58 })();
59
60 try {
61 return await result;
62 } finally {
63 done = true;
64
65 await sleep(0);
66 }
67}
68
69
70
71const USER_AGENTS = [
72 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
73 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
74 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
75 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15',
76];
77
78function randomUserAgent() {
79 return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
80}
81
82function parseCookiesTxt(text) {
83 if (!text || !text.trim()) return null;
84 const dir = mkdtempSync(join(tmpdir(), 'cookies-'));
85 const path = join(dir, 'cookies.txt');
86 writeFileSync(path, text);
87 return path;
88}
89
90
91let ytDlpPath = null;
92
93async function ensureYtDlp() {
94 if (ytDlpPath) return;
95 const dir = mkdtempSync(join(tmpdir(), 'ytdlp-bin-'));
96 ytDlpPath = join(dir, 'yt-dlp');
97 console.log(' Downloading latest yt-dlp...');
98 const resp = await fetch('https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp');
99 if (!resp.ok) throw new Error(`Failed to download yt-dlp: HTTP ${resp.status}`);
100 const buffer = Buffer.from(await resp.arrayBuffer());
101 writeFileSync(ytDlpPath, buffer, { mode: 0o755 });
102 const ver = execSync(`${ytDlpPath} --version`, { encoding: 'utf-8' }).trim();
103 console.log(` yt-dlp ${ver} ready (${formatBytes(buffer.length)})`);
104}
105
106
107
108
109function ytdlp(args, { maxBuffer = 10 * 1024 * 1024, timeout = 120_000 } = {}) {
110 const quoted = args.map((a) => `"${a.replace(/"/g, '\\"')}"`).join(' ');
111 return execSync(`"${ytDlpPath}" ${quoted}`, {
112 maxBuffer,
113 timeout,
114 encoding: 'utf-8',
115 shell: true,
116 }).trim();
117}
118
119
120
121
122
123
124function extractInfo(url, { proxyUrl, cookiesPath, playerClients = 'tv_embedded,tv,web,mweb,android' }) {
125 const tmpDir = mkdtempSync(join(tmpdir(), 'ytdlp-'));
126 try {
127 const args = [
128 url,
129 '--no-playlist',
130 '--quiet',
131 '--no-warnings',
132 '--print-json',
133 '--skip-download',
134 '--user-agent', randomUserAgent(),
135 '--socket-timeout', '30',
136 '--retries', '0',
137 '--fragment-retries', '0',
138 '--extractor-retries', '0',
139 '--sleep-requests', '1',
140 '--xff', 'default',
141 '--extractor-args', `youtube:player_client=${playerClients};skip=hls`,
142 '--output', join(tmpDir, '%(title)s.%(ext)s'),
143 ];
144 if (proxyUrl) args.push('--proxy', proxyUrl);
145 if (cookiesPath) args.push('--cookies', cookiesPath);
146
147 const stdout = ytdlp(args, { maxBuffer: 10 * 1024 * 1024, timeout: 120_000 });
148
149 const lines = stdout.split('\n').filter(Boolean);
150 const info = JSON.parse(lines[lines.length - 1]);
151 const title = info.title || 'Unknown';
152 const dur = info.duration || 0;
153 console.log(` Found: "${title}" — ${formatDuration(dur)}`);
154 return info;
155 } finally {
156 try { rmSync(tmpDir, { recursive: true, force: true }); } catch {}
157 }
158}
159
160
161
162
163
164
165async function streamToFile(readableStream, filePath) {
166 const fileStream = createWriteStream(filePath);
167 const writableStream = Writable.toWeb(fileStream);
168 await readableStream.pipeTo(writableStream);
169}
170
171function bestByBitrate(formats) {
172 return formats
173 .filter(Boolean)
174 .sort((a, b) => (b.abr || b.tbr || 0) - (a.abr || a.tbr || 0))[0] || null;
175}
176
177function isDirectAudioFormat(format) {
178 return format?.url
179 && format.vcodec === 'none'
180 && format.acodec
181 && format.acodec !== 'none'
182 && ['http', 'https'].includes(format.protocol || 'https');
183}
184
185function hasDirectAudioFormats(info) {
186 return (info.formats || []).some(isDirectAudioFormat) || isDirectAudioFormat(info);
187}
188
189function selectAudioFormat(info, requestedFormat) {
190 const audioFormats = (info.formats || []).filter(isDirectAudioFormat);
191 if (!audioFormats.length && isDirectAudioFormat(info)) return info;
192 if (!audioFormats.length) throw new Error('No direct audio-only formats found');
193
194 if (requestedFormat === 'm4a') {
195 return bestByBitrate(audioFormats.filter((f) => f.ext === 'm4a')) || bestByBitrate(audioFormats);
196 }
197
198 if (requestedFormat === 'opus' || requestedFormat === 'webm') {
199 return bestByBitrate(audioFormats.filter((f) => f.ext === 'webm' || f.acodec?.includes('opus')))
200 || bestByBitrate(audioFormats);
201 }
202
203 return bestByBitrate(audioFormats);
204}
205
206function mergedHeaders(info, format) {
207 return {
208 ...(info.http_headers || {}),
209 ...(format.http_headers || {}),
210 'User-Agent': format.http_headers?.['User-Agent'] || info.http_headers?.['User-Agent'] || randomUserAgent(),
211 };
212}
213
214function ffmpegAudioArgs(audioFormat, audioBitrate) {
215 if (audioFormat === 'mp3') return `-c:a libmp3lame -b:a ${audioBitrate}k`;
216 if (audioFormat === 'm4a') return `-c:a aac -b:a ${audioBitrate}k`;
217 if (audioFormat === 'flac') return '-c:a flac';
218 if (audioFormat === 'wav') return '-c:a pcm_s16le';
219 return '-c:a copy';
220}
221
222
223
224await Actor.init();
225
226Actor.on('aborting', async () => {
227 await sleep(1000);
228 await Actor.exit();
229});
230
231await ensureYtDlp();
232
233const input = (await Actor.getInput()) || {};
234const videoUrls = input.videoUrls || [];
235const audioFormat = input.audioFormat || 'm4a';
236const audioBitrate = input.audioBitrate || '192';
237const embedMetadata = input.embedMetadata !== false;
238const sleepMax = input.sleepBetweenUrls || 12;
239const cookiesText = input.cookies || '';
240const proxyInput = input.proxyConfiguration || {};
241const useApifyProxy = proxyInput.useApifyProxy !== false;
242const proxyCountry = proxyInput.apifyProxyCountry || 'US';
243const allowProxyMediaDownload = input.allowProxyMediaDownload === true;
244
245if (!videoUrls.length) {
246 console.log('No video URLs provided.');
247 await Actor.exit();
248 process.exit(0);
249}
250
251console.log(`Processing ${videoUrls.length} video(s), format=${audioFormat}, bitrate=${audioBitrate}k, embed=${embedMetadata}`);
252
253
254let cookiesPath = null;
255if (cookiesText) {
256 cookiesPath = parseCookiesTxt(cookiesText);
257 console.log('Cookies loaded from input');
258}
259
260const conversionFormats = ['mp3', 'm4a', 'flac', 'wav'];
261const kvStoreId = process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
262const kvBaseUrl = kvStoreId
263 ? `https://api.apify.com/v2/key-value-stores/${kvStoreId}/records`
264 : null;
265
266for (let i = 0; i < videoUrls.length; i++) {
267 const url = videoUrls[i];
268 console.log(`[${i + 1}/${videoUrls.length}] Processing: ${url}`);
269
270 const tmpDir = mkdtempSync(join(tmpdir(), 'ytdlp-'));
271
272 try {
273
274 let proxyUrl = null;
275 let proxyTier = 'none';
276
277 if (useApifyProxy) {
278 console.log(' Setting up residential proxy...');
279 const proxyConfig = await Actor.createProxyConfiguration({
280 groups: ['RESIDENTIAL'],
281 countryCode: proxyCountry,
282 });
283 if (proxyConfig) {
284 proxyUrl = await proxyConfig.newUrl(`session_${randomBytes(4).toString('hex')}`);
285 proxyTier = 'residential';
286 }
287 }
288
289
290
291
292
293
294 console.log(' Fetching video metadata via proxy...');
295 let info = await withHeartbeat(
296 () => extractInfo(url, { proxyUrl, cookiesPath }),
297 'Fetching video metadata',
298 );
299 if (!hasDirectAudioFormats(info)) {
300 console.log(' No audio-only formats from initial client set; retrying metadata with TV clients...');
301 info = await withHeartbeat(
302 () => extractInfo(url, { proxyUrl, cookiesPath, playerClients: 'tv_embedded,tv' }),
303 'Retrying video metadata',
304 );
305 }
306 const title = info.title || 'Unknown';
307 const videoId = info.id || 'unknown';
308 const duration = info.duration || 0;
309 const channel = info.channel || info.uploader || '';
310 const selectedFormat = selectAudioFormat(info, audioFormat);
311 const sourceExt = selectedFormat.ext || info.ext || 'm4a';
312 const sourceCodec = selectedFormat.acodec || 'unknown';
313 const sourceBitrate = selectedFormat.abr || selectedFormat.tbr || null;
314 const requestedDirectFormat = audioFormat === 'best'
315 || (audioFormat === 'opus' && (sourceExt === 'webm' || sourceCodec.includes('opus')))
316 || audioFormat === sourceExt;
317 const needsConversion = conversionFormats.includes(audioFormat) && !requestedDirectFormat;
318
319 console.log(` Selected audio-only format ${selectedFormat.format_id || 'unknown'} (${sourceExt}, ${sourceCodec}, ${sourceBitrate || '?'} kbps)`);
320
321
322 const rawPath = join(tmpDir, `${safeFilename(title)}.raw.${sourceExt}`);
323 console.log(' Downloading audio-only stream from CDN (free, no proxy cost)...');
324 let proxyMediaUsed = false;
325 try {
326 await withHeartbeat(
327 (async () => {
328 const r = await fetch(selectedFormat.url, {
329 headers: mergedHeaders(info, selectedFormat),
330 signal: AbortSignal.timeout(300_000),
331 });
332 if (!r.ok) throw new Error(`CDN returned ${r.status}`);
333 await streamToFile(r.body, rawPath);
334 })(),
335 'Downloading from CDN',
336 );
337 console.log(` Downloaded ${formatBytes(statSync(rawPath).size)} from CDN`);
338 } catch (cdErr) {
339 if (!allowProxyMediaDownload) {
340 throw new Error(`CDN audio download failed (${cdErr.message}). Proxy media fallback is disabled to avoid residential bandwidth costs.`);
341 }
342 console.log(` CDN download failed (${cdErr.message}), falling back to full yt-dlp via proxy because allowProxyMediaDownload=true...`);
343 proxyMediaUsed = true;
344 const fetchArgs = [
345 url, '--no-playlist', '--quiet', '--no-warnings',
346 '--format', selectedFormat.format_id || 'bestaudio/best',
347 '--user-agent', randomUserAgent(),
348 '--socket-timeout', '30',
349 '--output', rawPath,
350 ];
351 if (proxyUrl) fetchArgs.push('--proxy', proxyUrl);
352 if (cookiesPath) fetchArgs.push('--cookies', cookiesPath);
353 ytdlp(fetchArgs, { maxBuffer: 50 * 1024 * 1024, timeout: 600_000 });
354 }
355
356
357 let audioPath;
358 let wasConverted = false;
359 if (needsConversion) {
360 audioPath = join(tmpDir, `${safeFilename(title)}.${audioFormat}`);
361 console.log(` Converting to ${audioFormat} with ffmpeg...`);
362 execSync(
363 `ffmpeg -y -i "${rawPath}" -vn ${ffmpegAudioArgs(audioFormat, audioBitrate)} "${audioPath}"`,
364 { stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000 },
365 );
366 wasConverted = true;
367 console.log(` Converted: ${formatBytes(statSync(audioPath).size)}`);
368 } else {
369 audioPath = join(tmpDir, `${safeFilename(title)}.${audioFormat === 'opus' ? 'webm' : sourceExt}`);
370 execSync(`mv "${rawPath}" "${audioPath}"`);
371 }
372
373 if (embedMetadata && audioFormat !== 'wav') {
374 console.log(' Embedding metadata...');
375 try {
376 const safeTitle = (info.title || '').replace(/"/g, '\\"');
377 const safeArtist = (channel || '').replace(/"/g, '\\"');
378 const tmpAudio = `${audioPath}.tmp.${extname(audioPath).replace(/^\./, '')}`;
379 execSync(
380 `ffmpeg -y -i "${audioPath}" -c copy ` +
381 `-metadata title="${safeTitle}" ` +
382 `-metadata artist="${safeArtist}" ` +
383 `-id3v2_version 3 "${tmpAudio}" && mv "${tmpAudio}" "${audioPath}"`,
384 { stdio: ['ignore', 'pipe', 'pipe'], timeout: 60_000 },
385 );
386 } catch (metaErr) {
387 console.log(` Metadata embedding skipped (${metaErr.message})`);
388 }
389 }
390
391
392 const audioFile = readFileSync(audioPath);
393 const fileExt = extname(audioPath).replace(/^\./, '');
394 const kvKey = `audio-${videoId}`;
395 const contentType = mimeType(fileExt);
396
397 console.log(` Uploading ${formatBytes(audioFile.length)} to key-value store...`);
398 await Actor.setValue(kvKey, audioFile, { contentType });
399 console.log(` ✓ Done: "${title}" (${videoId}), ${formatDuration(duration)}, ${formatBytes(audioFile.length)}`);
400
401 const item = {
402 video_id: videoId,
403 video_url: url,
404 video_title: title,
405 channel,
406 duration,
407 audio_format: fileExt,
408 file_size_bytes: audioFile.length,
409 source_format_id: selectedFormat.format_id || null,
410 source_ext: sourceExt,
411 source_codec: sourceCodec,
412 source_bitrate: sourceBitrate,
413 was_converted: wasConverted,
414 proxy_media_used: proxyMediaUsed,
415 kv_store_key: kvKey,
416 proxy_tier_used: proxyTier,
417 status: 'downloaded',
418 };
419 if (kvBaseUrl) item.audio_url = `${kvBaseUrl}/${kvKey}`;
420 await Actor.pushData(item);
421
422 } catch (err) {
423 console.error(` ✗ Failed: ${url} — ${err.message}`);
424 await Actor.pushData({
425 video_url: url,
426 status: 'error',
427 error: err.message.slice(0, 500),
428 });
429 } finally {
430 try { rmSync(tmpDir, { recursive: true, force: true }); } catch {}
431 }
432
433
434 if (i < videoUrls.length - 1) {
435 const delaySec = 5 + Math.random() * (sleepMax - 5);
436 console.log(` Sleeping ${delaySec.toFixed(1)}s before next URL...`);
437 await sleep(Math.round(delaySec * 1000));
438 }
439}
440
441console.log('Finished processing all URLs');
442await Actor.exit();