1import { Actor } from 'apify';
2import { XMLParser } from 'fast-xml-parser';
3import { validateTarget, parseTargetUrl, fetchFeed } from './safe-fetch.js';
4
5const DEFAULT_TIMEOUT_SECONDS = 10;
6const DEFAULT_MAX_ITEMS = 20;
7const ABSOLUTE_MAX_ITEMS = 100;
8const MAX_FEEDS = 50;
9
10
11export async function normalizeAndValidateUrl(rawUrl, resolve) {
12 return (await validateTarget(rawUrl, resolve)).url;
13}
14
15function firstString(...values) {
16 for (const v of values) {
17 if (typeof v === 'string' && v.trim()) return v.trim();
18 if (Array.isArray(v)) {
19 for (const item of v) {
20 if (typeof item === 'string' && item.trim()) return item.trim();
21 }
22 }
23 }
24 return null;
25}
26
27function coerceArray(value) {
28 if (value === undefined || value === null) return [];
29 return Array.isArray(value) ? value : [value];
30}
31
32function extractText(node) {
33 if (node === undefined || node === null) return null;
34 if (typeof node === 'string') return node.trim() || null;
35 if (typeof node === 'object') {
36
37 const text = node['#text'];
38 if (typeof text === 'string') return text.trim() || null;
39
40 for (const key of Object.keys(node)) {
41 if (typeof node[key] === 'string' && node[key].trim()) return node[key].trim();
42 }
43 }
44 return null;
45}
46
47function parseDate(value) {
48 if (!value) return null;
49 const time = Date.parse(value);
50 if (Number.isNaN(time)) return null;
51 return new Date(time).toISOString();
52}
53
54function detectFeedFormat(parsed, rootKeys) {
55 if (rootKeys.includes('rss')) return 'rss2';
56 if (rootKeys.includes('feed')) return 'atom';
57 if (rootKeys.includes('rdf') || rootKeys.includes('rdf:RDF')) return 'rdf';
58 return 'unknown';
59}
60
61function rssGuidText(guid) {
62
63 return firstString(typeof guid === 'object' && guid !== null ? guid['#text'] : guid);
64}
65
66function rssGuidPermalink(guid) {
67 const flag = typeof guid === 'object' && guid !== null ? guid.isPermaLink : undefined;
68 if (flag !== undefined && flag !== 'true') return null;
69 const text = rssGuidText(guid);
70 if (!text) return null;
71 try {
72 const url = new URL(text);
73 if (['http:', 'https:'].includes(url.protocol) && !url.username && !url.password) return url.href;
74 } catch {
75
76 }
77 return null;
78}
79
80export function normalizeRss2Channel(channel) {
81 const title = firstString(channel.title);
82 const link = firstString(channel.link);
83 const description = firstString(channel.description);
84 const items = coerceArray(channel.item);
85 const normalizedItems = items.map((item) => ({
86 title: firstString(item.title),
87 link: firstString(item.link) || rssGuidPermalink(item.guid),
88 guid: rssGuidText(item.guid),
89 description: firstString(item.description),
90 pubDate: parseDate(firstString(item.pubDate)),
91 author: firstString(item.author, item['dc:creator']),
92 categories: coerceArray(item.category).map((c) => (typeof c === 'string' ? c.trim() : extractText(c))).filter(Boolean),
93 }));
94 return { title, link, description, items: normalizedItems };
95}
96
97function atomBase(node, inheritedBase) {
98 if (typeof node?.['xml:base'] !== 'string') return inheritedBase;
99 try {
100 return new URL(node['xml:base'], inheritedBase || undefined).href;
101 } catch {
102 return null;
103 }
104}
105
106function atomAlternateLink(node, inheritedBase) {
107 const candidates = coerceArray(node.link)
108 .filter((link) => link && typeof link === 'object'
109 && (!link.rel || link.rel === 'alternate' || link.rel === 'http://www.iana.org/assignments/relation/alternate'));
110 const isHtml = (link) => /^(text\/html|application\/xhtml\+xml)(?:;|$)/i.test(link.type || '');
111
112 candidates.sort((a, b) => Number(isHtml(b)) - Number(isHtml(a)));
113 for (const link of candidates) {
114 if (typeof link.href !== 'string' || !link.href.trim()) continue;
115 try {
116 const base = atomBase(link, inheritedBase);
117 const url = new URL(link.href, base || undefined);
118 if (['http:', 'https:'].includes(url.protocol) && !url.username && !url.password) return url.href;
119 } catch {
120
121 }
122 }
123 return null;
124}
125
126export function normalizeAtomFeed(feed, documentUrl) {
127 const feedBase = atomBase(feed, documentUrl);
128 const title = firstString(extractText(feed.title));
129 const link = atomAlternateLink(feed, feedBase);
130 const subtitle = firstString(extractText(feed.subtitle));
131 const entries = coerceArray(feed.entry);
132 const normalizedItems = entries.map((entry) => {
133 const itemLink = atomAlternateLink(entry, atomBase(entry, feedBase));
134 const content = firstString(extractText(entry.content), extractText(entry.summary));
135 return {
136 title: firstString(extractText(entry.title)),
137 link: itemLink,
138 guid: firstString(entry.id),
139 description: content,
140 pubDate: parseDate(firstString(extractText(entry.published), extractText(entry.updated))),
141 author: firstString(extractText(entry.author?.name), entry.author?.name),
142 categories: coerceArray(entry.category)
143 .map((c) => (typeof c === 'string' ? c.trim() : (c && typeof c === 'object' ? c.term : null)))
144 .filter(Boolean),
145 };
146 });
147 return { title, link, description: subtitle, items: normalizedItems };
148}
149
150export function normalizeRdfChannel(rdfRoot) {
151
152 const channel = rdfRoot.channel || {};
153 const title = firstString(channel.title);
154 const link = firstString(channel.link);
155 const description = firstString(channel.description);
156 const rawItems = coerceArray(rdfRoot.item);
157 const normalizedItems = rawItems.map((item) => ({
158 title: firstString(item.title),
159 link: firstString(item.link),
160 guid: firstString(item.link, item['rdf:about']),
161 description: firstString(item.description, item['content:encoded']),
162 pubDate: parseDate(firstString(item['dc:date'], item.pubDate)),
163 author: firstString(item['dc:creator'], item.author),
164 categories: coerceArray(item['dc:subject']).map((c) => (typeof c === 'string' ? c.trim() : extractText(c))).filter(Boolean),
165 }));
166 return { title, link, description, items: normalizedItems };
167}
168
169function findFeedRoot(parsed) {
170 if (!parsed || typeof parsed !== 'object') return { root: {}, key: '' };
171 const keys = Object.keys(parsed);
172
173 const feedKey = keys.find((k) => !k.startsWith('?')) || keys[0];
174 return { root: parsed[feedKey] || {}, key: feedKey || '' };
175}
176
177export function normalizeFeed(xmlBody, feedUrl) {
178 const parser = new XMLParser({
179 ignoreAttributes: false,
180 attributeNamePrefix: '',
181 allowBooleanAttributes: true,
182 parseAttributeValue: false,
183
184 parseTagValue: false,
185 trimValues: true,
186 });
187 const parsed = parser.parse(xmlBody);
188 const rootKeys = Object.keys(parsed || {});
189 const { root, key: rootKey } = findFeedRoot(parsed);
190 const format = detectFeedFormat(parsed, rootKeys);
191
192 let channel = null;
193 if (format === 'rss2') {
194 channel = normalizeRss2Channel(root.channel || {});
195 } else if (format === 'atom') {
196 channel = normalizeAtomFeed(root, feedUrl);
197 } else if (format === 'rdf' || rootKey.toLowerCase().includes('rdf')) {
198 channel = normalizeRdfChannel(root);
199 } else {
200
201 if (root.channel) channel = normalizeRss2Channel(root.channel);
202 else channel = normalizeAtomFeed(root, feedUrl);
203 }
204 return { feedUrl, format, channel };
205}
206
207export async function normalizeFeeds(input, fetchOptions) {
208 const feedUrls = Array.isArray(input.feedUrls) ? input.feedUrls : [];
209 if (feedUrls.length === 0) throw new Error('feedUrls must contain at least one feed URL');
210 if (feedUrls.length > MAX_FEEDS) throw new Error(`A maximum of ${MAX_FEEDS} feeds can be normalized per run`);
211
212 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
213 const maxItems = Math.min(Math.max(Number(input.maxItemsPerFeed || DEFAULT_MAX_ITEMS), 1), ABSOLUTE_MAX_ITEMS);
214 const seen = new Set();
215 const outputs = [];
216
217 for (const rawUrl of feedUrls) {
218 let validated;
219 try {
220 validated = parseTargetUrl(rawUrl);
221 } catch (error) {
222 outputs.push({ inputUrl: rawUrl, ok: false, error: error.message });
223 continue;
224 }
225 if (seen.has(validated.href)) continue;
226 seen.add(validated.href);
227
228 const fetchResult = await fetchFeed(validated.href, timeoutSeconds, fetchOptions);
229 if (!fetchResult.ok || !fetchResult.body) {
230 outputs.push({
231 inputUrl: rawUrl,
232 feedUrl: validated.href,
233 ok: false,
234 status: fetchResult.status,
235 error: fetchResult.error || `Feed fetch failed with status ${fetchResult.status}`,
236 });
237 continue;
238 }
239
240 let normalized;
241 try {
242 normalized = normalizeFeed(fetchResult.body, fetchResult.finalUrl);
243 } catch (error) {
244 outputs.push({
245 inputUrl: rawUrl,
246 feedUrl: validated.href,
247 ok: false,
248 status: fetchResult.status,
249 error: `Feed parsing failed: ${error.message}`,
250 });
251 continue;
252 }
253
254 const items = normalized.channel.items.slice(0, maxItems);
255 outputs.push({
256 inputUrl: rawUrl,
257 feedUrl: validated.href,
258 ok: true,
259 status: fetchResult.status,
260 checkedAt: new Date().toISOString(),
261 format: normalized.format,
262 finalUrl: fetchResult.finalUrl,
263 contentType: fetchResult.contentType,
264 channel: {
265 title: normalized.channel.title,
266 link: normalized.channel.link,
267 description: normalized.channel.description,
268 itemCount: normalized.channel.items.length,
269 itemsReturned: items.length,
270 },
271 items,
272 });
273 }
274
275 return outputs;
276}
277
278if (process.env.NODE_ENV !== 'test') {
279 await Actor.init();
280 try {
281 const input = await Actor.getInput();
282 const results = await normalizeFeeds(input || {});
283 for (const result of results) {
284 await Actor.pushData(result);
285 }
286 await Actor.setValue('OUTPUT', { count: results.length, results });
287 Actor.log.info('Feed normalization complete', { count: results.length });
288 } finally {
289 await Actor.exit();
290 }
291}
