1import { Actor } from 'apify';
2import { XMLParser } from 'fast-xml-parser';
3import dns from 'node:dns/promises';
4import net from 'node:net';
5
6const DEFAULT_TIMEOUT_SECONDS = 10;
7const DEFAULT_MAX_ITEMS = 20;
8const ABSOLUTE_MAX_ITEMS = 100;
9const MAX_FEEDS = 50;
10const USER_AGENT = 'RssFeedNormalizer/0.1 (+https://apify.com)';
11
12function isPrivateIPv4(ip) {
13 const parts = ip.split('.').map(Number);
14 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
15 const [a, b] = parts;
16 return a === 10
17 || (a === 172 && b >= 16 && b <= 31)
18 || (a === 192 && b === 168)
19 || a === 127
20 || a === 0
21 || (a === 169 && b === 254);
22}
23
24function isPrivateIPv6(ip) {
25 const normalized = ip.toLowerCase();
26 return normalized === '::1'
27 || normalized.startsWith('fc')
28 || normalized.startsWith('fd')
29 || normalized.startsWith('fe80:');
30}
31
32export async function normalizeAndValidateUrl(rawUrl) {
33 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('feed URL is required');
34
35 if (/^[a-z][a-z0-9+.-]*:\/\//i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
36 throw new Error('Only HTTP and HTTPS URLs are supported');
37 }
38 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
39 const url = new URL(withScheme);
40 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
41 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
42
43 const literalType = net.isIP(url.hostname);
44 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
45 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
46
47 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
48 for (const record of records) {
49 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
50 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
51 }
52
53 return url;
54}
55
56async function fetchFeed(url, timeoutSeconds) {
57 const controller = new AbortController();
58 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
59 try {
60 const response = await fetch(url, {
61 redirect: 'follow',
62 signal: controller.signal,
63 headers: { 'user-agent': USER_AGENT, accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*;q=0.1' },
64 });
65 const text = await response.text();
66 return {
67 ok: response.ok,
68 status: response.status,
69 finalUrl: response.url,
70 contentType: response.headers.get('content-type') || '',
71 body: text.slice(0, 2_000_000),
72 };
73 } catch (error) {
74 return { ok: false, status: null, finalUrl: url.href, contentType: '', body: '', error: error.message };
75 } finally {
76 clearTimeout(timeout);
77 }
78}
79
80function firstString(...values) {
81 for (const v of values) {
82 if (typeof v === 'string' && v.trim()) return v.trim();
83 if (Array.isArray(v)) {
84 for (const item of v) {
85 if (typeof item === 'string' && item.trim()) return item.trim();
86 }
87 }
88 }
89 return null;
90}
91
92function coerceArray(value) {
93 if (value === undefined || value === null) return [];
94 return Array.isArray(value) ? value : [value];
95}
96
97function extractText(node) {
98 if (node === undefined || node === null) return null;
99 if (typeof node === 'string') return node.trim() || null;
100 if (typeof node === 'object') {
101
102 const text = node['#text'];
103 if (typeof text === 'string') return text.trim() || null;
104
105 for (const key of Object.keys(node)) {
106 if (typeof node[key] === 'string' && node[key].trim()) return node[key].trim();
107 }
108 }
109 return null;
110}
111
112function parseDate(value) {
113 if (!value) return null;
114 const time = Date.parse(value);
115 if (Number.isNaN(time)) return null;
116 return new Date(time).toISOString();
117}
118
119function detectFeedFormat(parsed, rootKeys) {
120 if (rootKeys.includes('rss')) return 'rss2';
121 if (rootKeys.includes('feed')) return 'atom';
122 if (rootKeys.includes('rdf') || rootKeys.includes('rdf:RDF')) return 'rdf';
123 return 'unknown';
124}
125
126export function normalizeRss2Channel(channel) {
127 const title = firstString(channel.title);
128 const link = firstString(channel.link);
129 const description = firstString(channel.description);
130 const items = coerceArray(channel.item);
131 const normalizedItems = items.map((item) => ({
132 title: firstString(item.title),
133 link: firstString(item.link, item.guid),
134 guid: firstString(item.guid),
135 description: firstString(item.description),
136 pubDate: parseDate(firstString(item.pubDate)),
137 author: firstString(item.author, item['dc:creator']),
138 categories: coerceArray(item.category).map((c) => (typeof c === 'string' ? c.trim() : extractText(c))).filter(Boolean),
139 }));
140 return { title, link, description, items: normalizedItems };
141}
142
143export function normalizeAtomFeed(feed) {
144 const title = firstString(extractText(feed.title));
145 const link = (Array.isArray(feed.link) ? feed.link : [feed.link])
146 .map((l) => (l && typeof l === 'object' ? l.href : l))
147 .find((l) => typeof l === 'string' && l.trim()) || null;
148 const subtitle = firstString(extractText(feed.subtitle));
149 const entries = coerceArray(feed.entry);
150 const normalizedItems = entries.map((entry) => {
151 const entryLinks = coerceArray(entry.link);
152 const alternateLink = entryLinks
153 .map((l) => (l && typeof l === 'object' ? l : null))
154 .find((l) => l && (!l.rel || l.rel === 'alternate'));
155 const itemLink = alternateLink?.href
156 || entryLinks.map((l) => (typeof l === 'object' ? l.href : l)).find((l) => typeof l === 'string' && l.trim())
157 || null;
158 const content = firstString(extractText(entry.content), extractText(entry.summary));
159 return {
160 title: firstString(extractText(entry.title)),
161 link: itemLink,
162 guid: firstString(entry.id),
163 description: content,
164 pubDate: parseDate(firstString(extractText(entry.published), extractText(entry.updated))),
165 author: firstString(extractText(entry.author?.name), entry.author?.name),
166 categories: coerceArray(entry.category)
167 .map((c) => (typeof c === 'string' ? c.trim() : (c && typeof c === 'object' ? c.term : null)))
168 .filter(Boolean),
169 };
170 });
171 return { title, link, description: subtitle, items: normalizedItems };
172}
173
174export function normalizeRdfChannel(rdfRoot) {
175
176 const channel = rdfRoot.channel || {};
177 const title = firstString(channel.title);
178 const link = firstString(channel.link);
179 const description = firstString(channel.description);
180 const rawItems = coerceArray(rdfRoot.item);
181 const normalizedItems = rawItems.map((item) => ({
182 title: firstString(item.title),
183 link: firstString(item.link),
184 guid: firstString(item.link, item['rdf:about']),
185 description: firstString(item.description, item['content:encoded']),
186 pubDate: parseDate(firstString(item['dc:date'], item.pubDate)),
187 author: firstString(item['dc:creator'], item.author),
188 categories: coerceArray(item['dc:subject']).map((c) => (typeof c === 'string' ? c.trim() : extractText(c))).filter(Boolean),
189 }));
190 return { title, link, description, items: normalizedItems };
191}
192
193function findFeedRoot(parsed) {
194 if (!parsed || typeof parsed !== 'object') return { root: {}, key: '' };
195 const keys = Object.keys(parsed);
196
197 const feedKey = keys.find((k) => !k.startsWith('?')) || keys[0];
198 return { root: parsed[feedKey] || {}, key: feedKey || '' };
199}
200
201export function normalizeFeed(xmlBody, feedUrl) {
202 const parser = new XMLParser({
203 ignoreAttributes: false,
204 attributeNamePrefix: '',
205 allowBooleanAttributes: true,
206 parseAttributeValue: false,
207 parseTagValue: true,
208 trimValues: true,
209 });
210 const parsed = parser.parse(xmlBody);
211 const rootKeys = Object.keys(parsed || {});
212 const { root, key: rootKey } = findFeedRoot(parsed);
213 const format = detectFeedFormat(parsed, rootKeys);
214
215 let channel = null;
216 if (format === 'rss2') {
217 channel = normalizeRss2Channel(root.channel || {});
218 } else if (format === 'atom') {
219 channel = normalizeAtomFeed(root);
220 } else if (format === 'rdf' || rootKey.toLowerCase().includes('rdf')) {
221 channel = normalizeRdfChannel(root);
222 } else {
223
224 if (root.channel) channel = normalizeRss2Channel(root.channel);
225 else channel = normalizeAtomFeed(root);
226 }
227 return { feedUrl, format, channel };
228}
229
230export async function normalizeFeeds(input) {
231 const feedUrls = Array.isArray(input.feedUrls) ? input.feedUrls : [];
232 if (feedUrls.length === 0) throw new Error('feedUrls must contain at least one feed URL');
233 if (feedUrls.length > MAX_FEEDS) throw new Error(`A maximum of ${MAX_FEEDS} feeds can be normalized per run`);
234
235 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
236 const maxItems = Math.min(Math.max(Number(input.maxItemsPerFeed || DEFAULT_MAX_ITEMS), 1), ABSOLUTE_MAX_ITEMS);
237 const seen = new Set();
238 const outputs = [];
239
240 for (const rawUrl of feedUrls) {
241 let validated;
242 try {
243 validated = await normalizeAndValidateUrl(rawUrl);
244 } catch (error) {
245 outputs.push({ inputUrl: rawUrl, ok: false, error: error.message });
246 continue;
247 }
248 if (seen.has(validated.href)) continue;
249 seen.add(validated.href);
250
251 const fetchResult = await fetchFeed(validated.href, timeoutSeconds);
252 if (!fetchResult.ok || !fetchResult.body) {
253 outputs.push({
254 inputUrl: rawUrl,
255 feedUrl: validated.href,
256 ok: false,
257 status: fetchResult.status,
258 error: fetchResult.error || `Feed fetch failed with status ${fetchResult.status}`,
259 });
260 continue;
261 }
262
263 let normalized;
264 try {
265 normalized = normalizeFeed(fetchResult.body, validated.href);
266 } catch (error) {
267 outputs.push({
268 inputUrl: rawUrl,
269 feedUrl: validated.href,
270 ok: false,
271 status: fetchResult.status,
272 error: `Feed parsing failed: ${error.message}`,
273 });
274 continue;
275 }
276
277 const items = normalized.channel.items.slice(0, maxItems);
278 outputs.push({
279 inputUrl: rawUrl,
280 feedUrl: validated.href,
281 ok: true,
282 status: fetchResult.status,
283 checkedAt: new Date().toISOString(),
284 format: normalized.format,
285 finalUrl: fetchResult.finalUrl,
286 contentType: fetchResult.contentType,
287 channel: {
288 title: normalized.channel.title,
289 link: normalized.channel.link,
290 description: normalized.channel.description,
291 itemCount: normalized.channel.items.length,
292 itemsReturned: items.length,
293 },
294 items,
295 });
296 }
297
298 return outputs;
299}
300
301if (process.env.NODE_ENV !== 'test') {
302 await Actor.init();
303 try {
304 const input = await Actor.getInput();
305 const results = await normalizeFeeds(input || {});
306 for (const result of results) {
307 await Actor.pushData(result);
308 }
309 await Actor.setValue('OUTPUT', { count: results.length, results });
310 Actor.log.info('Feed normalization complete', { count: results.length });
311 } finally {
312 await Actor.exit();
313 }
314}
