1from apify import Actor
2from os import getenv
3import requests
4from .models import ProtocolFilter, ProxyFetcherInput, ProxyFetcherOutput
5from .utils import map_anonymity_level
6
7
8async def main():
9 async with Actor:
10 try:
11 input_data = await Actor.get_input()
12 params = (
13 ProxyFetcherInput(**input_data) if input_data else ProxyFetcherInput()
14 )
15
16 url = getenv("PROXIES_SOURCE_URL")
17 Actor.log.info(f"Fetching proxies")
18 response = requests.get(url, timeout=10)
19 response.raise_for_status()
20 raw_proxies = response.json()
21
22
23 processed = []
24 for proxy in raw_proxies:
25 if not proxy.get("country_code"):
26 proxy["country_code"] = "Unknown"
27 try:
28 processed_proxy = {
29 "ip": proxy["ip"],
30 "port": int(proxy["port"]),
31 "http": proxy["http"] == "1",
32 "ssl": proxy["ssl"] == "1",
33 "socks4": proxy["socks4"] == "1",
34 "socks5": proxy["socks5"] == "1",
35 "anon": map_anonymity_level(proxy["anon"]),
36 "country_code": proxy.get("country_code").upper(),
37 "delay": float(proxy.get("delay", 0)),
38 "lastseen": proxy.get("lastseen"),
39 "checks_up": proxy.get("checks_up"),
40 "checks_down": proxy.get("checks_down"),
41 }
42 processed.append(processed_proxy)
43 except (KeyError, ValueError) as e:
44 Actor.log.warning(
45 f"Skipping invalid proxy entry: {proxy}. Error: {str(e)}"
46 )
47
48
49 filtered = [
50 p
51 for p in processed
52 if (
53 params.country_codes is None
54 or (
55 isinstance(p.get("country_code"), str)
56 and p["country_code"] in params.country_codes
57 )
58 )
59 and (
60 params.http == ProtocolFilter.UNSET
61 or p["http"] == (params.http == ProtocolFilter.TRUE)
62 )
63 and (
64 params.ssl == ProtocolFilter.UNSET
65 or p["ssl"] == (params.ssl == ProtocolFilter.TRUE)
66 )
67 and (
68 params.socks4 == ProtocolFilter.UNSET
69 or p["socks4"] == (params.socks4 == ProtocolFilter.TRUE)
70 )
71 and (
72 params.socks5 == ProtocolFilter.UNSET
73 or p["socks5"] == (params.socks5 == ProtocolFilter.TRUE)
74 )
75 ]
76
77
78 if params.sort_by:
79 reverse = params.sort_order.lower() == "desc"
80 try:
81 filtered.sort(
82 key=lambda x: x.get(params.sort_by, 0), reverse=reverse
83 )
84 except Exception as e:
85 Actor.log.error(f"Sorting failed: {str(e)}")
86
87 await Actor.push_data(
88 ProxyFetcherOutput(proxies=filtered).model_dump(
89 exclude_none=True, exclude_unset=True
90 )
91 )
92
93 except Exception as e:
94 error_msg = f"Actor failed: {str(e)}"
95 Actor.log.error(error_msg)
96 await Actor.fail(exception=e, status_message=error_msg)