1import asyncio
2import json
3import random
4import re
5import sys
6from datetime import datetime, timezone
7from urllib.parse import urlencode, urljoin
8
9import httpx
10
11try:
12 from apify import Actor
13except ImportError:
14 Actor = None
15
16BASE_URL = "https://komehyo.jp/"
17SEARCH_URL = "https://komehyo.jp/search/"
18MAX_PAGES = 200
19
20
21CARD_RE = re.compile(
22 r'<a\b(?=[^>]*\bclass=["\'][^"\']*p-link--card[^"\']*["\'])(?=[^>]*\bhref="/product/([\d-]+)/")[^>]*>'
23)
24
25FIELD_CLASSES = {
26 "title": "p-link__txt--productsname",
27 "brand": "p-link__txt--brand",
28 "size": "p-link__txt--size",
29 "rank": "p-link__txt--rank",
30 "material": "p-link__txt--material",
31 "store": "p-link__txt--store",
32 "reference": "p-link__txt--reference",
33 "price": "p-link__txt--price",
34}
35
36
37def extract_text(segment: str, class_name: str):
38 """指定クラスを持つ span の中身からタグを除去してテキストを返す。"""
39 pattern = (
40 r'<span[^>]*class=["\'][^"\']*\b'
41 + re.escape(class_name)
42 + r'\b[^"\']*["\'][^>]*>(.*?)</span>'
43 )
44 match = re.search(pattern, segment, re.S)
45 if not match:
46 return None
47
48 text = re.sub(r'<[^>]+>', '', match.group(1))
49 text = (
50 text.replace(' ', ' ')
51 .replace('¥', '¥')
52 .replace('&', '&')
53 .replace(''', "'")
54 .replace('"', '"')
55 )
56 return re.sub(r'\s+', ' ', text).strip() or None
57
58
59def extract_rating(segment: str, class_name: str):
60 """親span(例: p-link__txt--rank)の内側にある rating の値を返す。"""
61 parent = extract_text(segment, class_name)
62 if not parent:
63 return None
64
65 for prefix in ["ランク：", "サイズ：", "素材：", "在庫店舗：", "ランク:", "サイズ:", "素材:", "在庫店舗:"]:
66 if parent.startswith(prefix):
67 return parent[len(prefix):].strip()
68 return parent
69
70
71def extract_image_url(segment: str):
72 match = re.search(r'<img[^>]+src="([^"]+)"', segment)
73 if not match:
74 match = re.search(r'<img[^>]+data-src="([^"]+)"', segment)
75 if match:
76 return urljoin(BASE_URL, match.group(1))
77 return None
78
79
80def parse_price(text):
81 if not text:
82 return None
83 clean = (
84 text.replace('￥', '')
85 .replace('¥', '')
86 .replace('円', '')
87 .replace('税込', '')
88 .replace('参考上代', '')
89 .replace(':', '')
90 .replace('：', '')
91 )
92 match = re.search(r'\d[\d,]*', clean)
93 if not match:
94 return None
95 return int(match.group(0).replace(',', ''))
96
97
98def iter_product_cards(html):
99 for match in CARD_RE.finditer(html):
100 product_id = match.group(1)
101 start = match.end()
102 end = html.find('</a>', start)
103 if end == -1:
104 end = len(html)
105 yield html[start:end], product_id
106
107
108def make_item(segment: str, product_id: str):
109 fields = {
110 key: extract_text(segment, class_name)
111 for key, class_name in FIELD_CLASSES.items()
112 }
113
114 item = {
115 "productId": str(product_id),
116 "title": fields["title"],
117 "brand": fields["brand"],
118 "price": parse_price(fields["price"]),
119 "referencePrice": parse_price(fields["reference"]),
120 "rank": extract_rating(segment, "p-link__txt--rank"),
121 "size": extract_rating(segment, "p-link__txt--size"),
122 "material": extract_rating(segment, "p-link__txt--material"),
123 "store": extract_rating(segment, "p-link__txt--store"),
124 "imageUrl": extract_image_url(segment),
125 "productUrl": f"{BASE_URL}product/{product_id}/",
126 "scrapedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
127 }
128
129 if not item["title"]:
130 return None
131 return item
132
133
134async def fetch_page(client, keyword: str, page: int, headers: dict):
135 url = f"{SEARCH_URL}?{urlencode({'q': keyword, 'page': page})}"
136 last_error = None
137
138 for attempt in range(3):
139 try:
140 response = await client.get(url, headers=headers)
141 if response.status_code == 200:
142 return response.text
143 last_error = RuntimeError(f"Unexpected HTTP status {response.status_code}")
144 except httpx.HTTPError as exc:
145 last_error = exc
146
147 await asyncio.sleep(min(2 ** attempt, 8) + random.uniform(0, 1))
148
149 raise RuntimeError(f"Failed to fetch {url}: {last_error}")
150
151
152async def main():
153 if Actor is not None:
154 await Actor.init()
155
156 results = []
157
158 try:
159
160
161 actor_input = await Actor.get_input() if Actor is not None else None
162 if not actor_input:
163 raw_input = sys.stdin.read()
164 actor_input = json.loads(raw_input) if raw_input.strip() else {}
165
166 keyword = str(actor_input.get("keyword") or "").strip()
167 if not keyword:
168 raise ValueError("keyword is required")
169
170 max_items = int(actor_input.get("maxItems", 100))
171 max_items = max(1, min(max_items, 500))
172
173 proxy_url = None
174 if Actor is not None:
175 proxy_input = actor_input.get("proxyConfiguration")
176 if proxy_input:
177 proxy_config = await Actor.create_proxy_configuration(
178 actor_proxy_input=proxy_input
179 )
180 if proxy_config:
181 proxy_url = await proxy_config.new_url()
182
183 headers = {
184 "User-Agent": (
185 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
186 "AppleWebKit/537.36 (KHTML, like Gecko) "
187 "Chrome/120.0 Safari/537.36"
188 )
189 }
190
191
192
193 async with httpx.AsyncClient(
194 timeout=30,
195 follow_redirects=True,
196 proxy=proxy_url,
197 trust_env=False,
198 ) as client:
199 page = 1
200 seen_ids = set()
201 collected = 0
202
203 while collected < max_items and page <= MAX_PAGES:
204 html = await fetch_page(client, keyword, page, headers)
205 cards = list(iter_product_cards(html))
206
207 if not cards:
208 break
209
210 for segment, product_id in cards:
211 if collected >= max_items:
212 break
213 if product_id in seen_ids:
214 continue
215 seen_ids.add(product_id)
216
217 item = make_item(segment, product_id)
218 if not item:
219 continue
220
221 if Actor is not None:
222 await Actor.push_data(item)
223 else:
224 results.append(item)
225 collected += 1
226
227 if len(cards) < 50:
228 break
229
230 page += 1
231 if collected < max_items and page <= MAX_PAGES:
232 await asyncio.sleep(random.uniform(1.0, 3.0))
233
234 if Actor is None:
235 print(json.dumps(results, ensure_ascii=False, indent=2))
236
237 finally:
238 if Actor is not None:
239 await Actor.exit()
240
241
242if __name__ == "__main__":
243 asyncio.run(main())
