1"""Apify Actor: Google News articles by keyword or topic, for any country and language, with publisher URLs and
2(optionally) the article page details and text."""
3from __future__ import annotations
4
5import asyncio
6import random
7from datetime import datetime, timedelta, timezone
8
9import httpx
10from apify import Actor
11
12from .article import details
13from .resolve import resolve_async
14from .news import (FEED_CAP, InputError, edition, parse_day, parse_feed, search_url, topic_url, windows)
15
16EVENT_ARTICLE = 'article'
17CONCURRENCY = 4
18ENRICH_CONCURRENCY = 8
19TIMEFRAME_DAYS = {'1h': 1, '1d': 1, '7d': 7, '30d': 30, '1y': 365}
20HEADERS = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
21 'Chrome/130.0 Safari/537.36', 'Accept-Language': 'en;q=0.8'}
22
23
24async def fetch(client: httpx.AsyncClient, url: str) -> list[dict]:
25 """One feed, with retries on rate limits and server errors."""
26 for attempt in range(5):
27 try:
28 r = await client.get(url)
29 if r.status_code == 200:
30 return parse_feed(r.content)
31 if r.status_code not in (429, 500, 502, 503, 504):
32 raise RuntimeError(f'Google News answered HTTP {r.status_code}')
33 except httpx.HTTPError as e:
34 if attempt == 4:
35 raise RuntimeError(f'network error: {e}') from e
36 await asyncio.sleep(2 ** attempt + random.random())
37 raise RuntimeError('Google News keeps answering "too many requests"; try again later or use a proxy')
38
39
40class Run:
41 def __init__(self, client: httpx.AsyncClient, max_per_query: int, include_related: bool,
42 resolve_urls: bool = True, details: bool = False, full_text: bool = False):
43 self.client, self.max_per_query, self.include_related = client, max_per_query, include_related
44 self.resolve_urls, self.details, self.full_text = resolve_urls or details or full_text, \
45 details or full_text, full_text
46 self.charging = Actor.get_charging_manager()
47 self.seen: set[str] = set()
48 self.pushed = 0
49 self.limit_hit = False
50 self.sem = asyncio.Semaphore(CONCURRENCY)
51 self.enrich_sem = asyncio.Semaphore(ENRICH_CONCURRENCY)
52
53 def _left(self) -> int | None:
54 return self.charging.calculate_max_event_charge_count_within_limit(EVENT_ARTICLE)
55
56 async def _feed(self, url: str) -> list[dict]:
57 async with self.sem:
58 return await fetch(self.client, url)
59
60 async def _enrich(self, a: dict) -> dict:
61 """Publisher URL (and page details) for one article; problems are noted on the item, never fatal."""
62 async with self.enrich_sem:
63 if self.resolve_urls and a.get('articleId'):
64 for attempt in range(3):
65 res = await resolve_async(self.client, a['articleId'])
66 if res.get('status') not in (429, 503):
67 break
68 await asyncio.sleep(2 ** attempt * 2 + random.random())
69 if res.get('url'):
70 a['url'] = res['url']
71 if res.get('ampUrl'):
72 a['ampUrl'] = res['ampUrl']
73 else:
74 a['urlError'] = res.get('error')
75 if self.details and a.get('url'):
76 got = await details(self.client, a['url'], self.full_text)
77 a.update(got.get('details') or {})
78 if got.get('detailsError'):
79 a['detailsError'] = got['detailsError']
80 return a
81
82 async def push(self, articles: list[dict], extra: dict, budget: int) -> int:
83 """Pushes new (not yet seen) articles, newest first, up to budget and the user's charge limit."""
84 articles.sort(key=lambda a: a['publishedAt'] or '', reverse=True)
85 picked = []
86 left = self._left()
87 for a in articles:
88 if len(picked) >= budget:
89 break
90 key = a['articleId'] or a['googleNewsUrl']
91 if key in self.seen:
92 continue
93 if left is not None and left <= len(picked):
94 self.limit_hit = True
95 break
96 self.seen.add(key)
97 if not self.include_related:
98 a = {k: v for k, v in a.items() if k != 'relatedCoverage'}
99 picked.append(a)
100 if self.resolve_urls:
101 picked = await asyncio.gather(*(self._enrich(a) for a in picked))
102 for n, a in enumerate(picked):
103 await Actor.push_data({**extra, 'position': n + 1, **a}, charged_event_name=EVENT_ARTICLE)
104 self.pushed += len(picked)
105 return len(picked)
106
107 async def query(self, q: str, ed_name: str, ed: dict, timeframe: str | None, day_from, day_to) -> dict:
108 extra = {'query': q, 'edition': ed['ceid']}
109 if self.max_per_query <= FEED_CAP:
110 got = await self._feed(search_url(q, ed, timeframe, day_from, day_to))
111 return {'found': await self.push(got, extra, self.max_per_query)}
112
113 today = datetime.now(timezone.utc).date()
114 end = day_to or today
115 start = day_from or end - timedelta(days=TIMEFRAME_DAYS.get(timeframe or '', 30) - 1)
116 found, days = 0, windows(start, end, 1)
117 for i in range(0, len(days), CONCURRENCY):
118 if found >= self.max_per_query or self.limit_hit:
119 break
120 batch = await asyncio.gather(*(self._feed(search_url(q, ed, None, a, b)) for a, b in days[i:i + CONCURRENCY]))
121 for got in batch:
122 found += await self.push(got, extra, self.max_per_query - found)
123 return {'found': found}
124
125 async def topic(self, t: str, ed: dict) -> dict:
126 got = await self._feed(topic_url(t, ed))
127 return {'found': await self.push(got, {'topic': t.upper(), 'edition': ed['ceid']},
128 min(self.max_per_query, FEED_CAP))}
129
130
131async def main() -> None:
132 async with Actor:
133 inp = await Actor.get_input() or {}
134 queries = [str(q).strip() for q in inp.get('queries') or [] if str(q).strip()]
135 topics = [str(t).strip() for t in inp.get('topics') or [] if str(t).strip()]
136 if not queries and not topics:
137 await Actor.fail(status_message='Add at least one search query or topic.')
138 return
139 try:
140 eds = [(e, edition(e)) for e in dict.fromkeys(inp.get('editions') or ['US:en'])]
141 day_from = parse_day(inp.get('dateFrom'), 'dateFrom')
142 day_to = parse_day(inp.get('dateTo'), 'dateTo')
143 except InputError as e:
144 await Actor.fail(status_message=str(e))
145 return
146 if day_from and day_to and day_from > day_to:
147 await Actor.fail(status_message='dateFrom is after dateTo.')
148 return
149 timeframe = inp.get('timeframe') or None
150 max_per_query = max(1, min(int(inp.get('maxArticlesPerQuery') or 100), 5000))
151
152 proxy_url = None
153 if (inp.get('proxyConfiguration') or {}).get('useApifyProxy'):
154 proxy = await Actor.create_proxy_configuration(actor_proxy_input=inp['proxyConfiguration'])
155 proxy_url = await proxy.new_url() if proxy else None
156 async with httpx.AsyncClient(timeout=httpx.Timeout(30, connect=10), headers=HEADERS, follow_redirects=True,
157 proxy=proxy_url) as client:
158 run = Run(client, max_per_query, bool(inp.get('includeRelatedCoverage', True)),
159 bool(inp.get('resolveUrls', True)), bool(inp.get('articleDetails', False)),
160 bool(inp.get('fullText', False)))
161 failed = []
162 for name, ed in eds:
163 for q in queries:
164 if run.limit_hit:
165 break
166 try:
167 res = await run.query(q, name, ed, timeframe, day_from, day_to)
168 Actor.log.info(f'"{q}" [{ed["ceid"]}]: {res["found"]} articles')
169 except (RuntimeError, InputError) as e:
170 failed.append(f'"{q}" [{ed["ceid"]}]: {e}')
171 for t in topics:
172 if run.limit_hit:
173 break
174 try:
175 res = await run.topic(t, ed)
176 Actor.log.info(f'topic {t} [{ed["ceid"]}]: {res["found"]} articles')
177 except (RuntimeError, InputError) as e:
178 failed.append(f'topic {t} [{ed["ceid"]}]: {e}')
179 for f in failed:
180 Actor.log.warning(f)
181 msg = f'{run.pushed} articles'
182 if run.limit_hit:
183 msg += '; stopped at the maximum charge per run'
184 if failed:
185 msg += f'; {len(failed)} feed(s) failed: {failed[0]}'
186 await Actor.set_status_message(msg, is_terminal=True)
187 if failed and not run.pushed:
188 await Actor.fail(status_message=msg)