1from apify import Actor
2import asyncio
3import httpx
4import re
5import json
6
7async def main():
8 async with Actor() as actor:
9 input_data = await actor.get_input() or {}
10 urls = input_data.get("urls", [])
11 search_queries = input_data.get("search_queries", [])
12 domain = input_data.get("domain", "amazon.com")
13 max_results = input_data.get("max_results", 20)
14
15 results = []
16 headers = {
17 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
18 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
19 "Accept-Language": "en-US,en;q=0.5",
20 "Accept-Encoding": "gzip, deflate, br",
21 }
22
23 async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
24
25 for url in urls:
26 Actor.log.info(f"Scraping product: {url}")
27 try:
28 resp = await client.get(url, headers=headers)
29 text = resp.text
30
31 data = {"url": url, "domain": domain}
32
33
34 title_match = re.search(r'<span[^>]*id="productTitle"[^>]*>([^<]+)</span>', text)
35 if title_match:
36 data["title"] = title_match.group(1).strip()
37
38
39 price_patterns = [
40 r'<span[^>]*class="a-price-whole"[^>]*>([^<]+)',
41 r'<span[^>]*id="priceblock_ourprice"[^>]*>([^<]+)',
42 r'<span[^>]*id="priceblock_dealprice"[^>]*>([^<]+)',
43 ]
44 for pattern in price_patterns:
45 price_match = re.search(pattern, text)
46 if price_match:
47 data["price"] = price_match.group(1).strip()
48 break
49
50
51 rating_match = re.search(r'<span[^>]*class="a-icon-alt"[^>]*>([\d.]+ out of [\d.]+ stars?)', text)
52 if rating_match:
53 data["rating"] = rating_match.group(1).strip()
54
55
56 rating_num = re.search(r'data-average-rating="([\d.]+)"', text)
57 if rating_num:
58 data["rating_value"] = rating_num.group(1)
59
60
61 ratings_count = re.search(r'id="acrCustomerReviewText"[^>]*>([^<]+)', text)
62 if ratings_count:
63 data["ratings_count"] = ratings_count.group(1).strip()
64
65
66 bsr_match = re.search(r'Best Sellers Rank[^<]*<span[^>]*>([^<]+)</span>', text)
67 if bsr_match:
68 data["best_sellers_rank"] = bsr_match.group(1).strip()
69
70
71 avail_match = re.search(r'id="availability"[^>]*>.*?<span[^>]*>([^<]+)</span>', text, re.DOTALL)
72 if avail_match:
73 data["availability"] = avail_match.group(1).strip()
74
75
76 features = re.findall(r'<span[^>]*class="a-list-item"[^>]*>([^<]+)</span>', text)
77 if features:
78 data["features"] = [f.strip() for f in features[:10] if len(f.strip()) > 10]
79
80
81 images = re.findall(r'"hiRes"\s*:\s*"([^"]+)"', text)
82 if images:
83 data["images"] = images[:10]
84
85
86 asin_match = re.search(r'"ASIN"\s*:\s*"([^"]+)"', text)
87 if not asin_match:
88 asin_match = re.search(r'/dp/([A-Z0-9]{10})', url)
89 if asin_match:
90 data["asin"] = asin_match.group(1)
91
92
93 brand_match = re.search(r'"brand"\s*:\s*"([^"]+)"', text)
94 if brand_match:
95 data["brand"] = brand_match.group(1)
96
97
98 desc_section = re.search(r'<div[^>]*id="feature-bullets"[^>]*>(.*?)</div>', text, re.DOTALL)
99 if desc_section:
100 bullets = re.findall(r'<span[^>]*class="a-list-item"[^>]*>(.*?)</span>', desc_section.group(1), re.DOTALL)
101 data["description_bullets"] = [re.sub(r'<[^>]+>', '', b).strip() for b in bullets if len(re.sub(r'<[^>]+>', '', b).strip()) > 5]
102
103 results.append(data)
104 await actor.push_data(data)
105 Actor.log.info(f" Scraped: {data.get('title', 'Unknown')[:50]}")
106
107 except Exception as e:
108 Actor.log.error(f"Error scraping product: {e}")
109 await actor.push_data({"url": url, "error": str(e)})
110
111 await asyncio.sleep(2)
112
113
114 for query in search_queries:
115 Actor.log.info(f"Searching: {query}")
116 try:
117 search_url = f"https://www.{domain}/s?k={query}"
118 resp = await client.get(search_url, headers=headers)
119 text = resp.text
120
121
122 products = re.findall(r'data-asin="([A-Z0-9]{10})"', text)
123 seen = set()
124
125 for asin in products[:max_results]:
126 if asin not in seen:
127 seen.add(asin)
128 product_url = f"https://www.{domain}/dp/{asin}"
129
130 product_data = {"asin": asin, "url": product_url, "search_query": query}
131
132
133 title_pattern = re.compile(
134 rf'data-asin="{asin}"[\s\S]*?<h2[^>]*class="a-size-base[^"]*"[^>]*>[\s\S]*?<a[^>]*>([\s\S]*?)</a>',
135 re.DOTALL
136 )
137 title_match = title_pattern.search(text)
138 if title_match:
139 product_data["title"] = re.sub(r'<[^>]+>', '', title_match.group(1)).strip()
140
141
142 price_s = re.compile(
143 rf'data-asin="{asin}"[\s\S]*?a-price-whole[^>]*>([\d,.]+)',
144 re.DOTALL
145 )
146 price_match = price_s.search(text)
147 if price_match:
148 product_data["price"] = price_match.group(1).strip()
149
150 results.append(product_data)
151 await actor.push_data(product_data)
152
153 Actor.log.info(f" Found {len(seen)} products for '{query}'")
154
155 except Exception as e:
156 Actor.log.error(f"Error searching '{query}': {e}")
157 await actor.push_data({"query": query, "error": str(e)})
158
159 await asyncio.sleep(2)
160
161 Actor.log.info(f"Done! Processed {len(results)} items.")
162
163
164if __name__ == "__main__":
165 asyncio.run(main())