1import asyncio
2import json
3import re
4from datetime import datetime, timezone
5
6from bs4 import BeautifulSoup
7
8
9def extract_price(text):
10 if not text:
11 return None
12 text_clean = text.replace('¥', '').replace('円', '').replace(',', '')
13 m = re.search(r'(\d+)', text_clean)
14 if m:
15 try:
16 return int(m.group(1))
17 except ValueError:
18 return None
19 return None
20
21
22def split_top_level(s, sep=','):
23 parts = []
24 depth = 0
25 in_str = False
26 quote = ''
27 cur = []
28 for ch in s:
29 if in_str:
30 cur.append(ch)
31 if ch == '\\':
32 continue
33 elif ch == quote:
34 in_str = False
35 continue
36 if ch in ("'", '"'):
37 in_str = True
38 quote = ch
39 cur.append(ch)
40 continue
41 if ch in '({[':
42 depth += 1
43 elif ch in ')}]':
44 depth -= 1
45 if ch == sep and depth == 0:
46 parts.append(''.join(cur).strip())
47 cur = []
48 else:
49 cur.append(ch)
50 if ''.join(cur).strip():
51 parts.append(''.join(cur).strip())
52 return parts
53
54
55def parse_js_value(val):
56 val = val.strip()
57 if not val:
58 return None
59 if val.startswith('{') and val.endswith('}'):
60 return parse_js_object(val)
61 if val.startswith('[') and val.endswith(']'):
62 inner = val[1:-1].strip()
63 if not inner:
64 return []
65 parts = split_top_level(inner, ',')
66 return [parse_js_value(p) for p in parts]
67 if val in ('true', 'True'):
68 return True
69 if val in ('false', 'False'):
70 return False
71 if val in ('null', 'None'):
72 return None
73 try:
74 return json.loads(val)
75 except json.JSONDecodeError:
76 if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'):
77 return val[1:-1]
78 return val
79
80
81def find_colon(s):
82 depth = 0
83 in_str = False
84 quote = ''
85 for i, ch in enumerate(s):
86 if in_str:
87 if ch == '\\':
88 continue
89 elif ch == quote:
90 in_str = False
91 continue
92 if ch in ("'", '"'):
93 in_str = True
94 quote = ch
95 continue
96 if ch in '({[':
97 depth += 1
98 continue
99 if ch in ')}]':
100 depth -= 1
101 continue
102 if ch == ':' and depth == 0:
103 return i
104 return -1
105
106
107def parse_js_object(s):
108 s = s.strip()
109 if not s.startswith('{') or not s.endswith('}'):
110 return {}
111 inner = s[1:-1].strip()
112 if not inner:
113 return {}
114 fields = split_top_level(inner, ',')
115 obj = {}
116 for f in fields:
117 idx = find_colon(f)
118 if idx == -1:
119 continue
120 key = f[:idx].strip().strip('"\'')
121 val = f[idx + 1:].strip()
122 obj[key] = parse_js_value(val)
123 return obj
124
125
126def parse_js_array(s):
127 s = s.strip()
128 if not s.startswith('[') or not s.endswith(']'):
129 return []
130 inner = s[1:-1].strip()
131 if not inner:
132 return []
133 parts = split_top_level(inner, ',')
134 return [parse_js_value(p) for p in parts]
135
136
137def extract_data_items(html):
138
139
140 marker = 'dataLayer.push({'
141 search_from = 0
142 while True:
143 start = html.find(marker, search_from)
144 if start == -1:
145 return []
146
147 i = start + len(marker)
148 depth = 0
149 in_str = False
150 quote = ''
151 end = -1
152 for idx in range(i, len(html)):
153 ch = html[idx]
154 if in_str:
155 if ch == '\\':
156 continue
157 elif ch == quote:
158 in_str = False
159 continue
160 if ch in ("'", '"'):
161 in_str = True
162 quote = ch
163 continue
164 if ch == '(':
165 depth += 1
166 elif ch == ')':
167 depth -= 1
168 if depth == 0:
169 end = idx
170 break
171 if end == -1:
172 return []
173 block = html[i:end]
174 if 'view_item_list' in block and re.search(r'items\s*:\s*\[', block):
175 break
176 search_from = start + 1
177 m = re.search(r'items\s*:\s*\[', block)
178 if not m:
179 return []
180 start_arr = m.end() - 1
181 depth = 0
182 in_str = False
183 quote = ''
184 end_arr = -1
185 for idx in range(start_arr, len(block)):
186 ch = block[idx]
187 if in_str:
188 if ch == '\\':
189 continue
190 elif ch == quote:
191 in_str = False
192 continue
193 if ch in ("'", '"'):
194 in_str = True
195 quote = ch
196 continue
197 if ch == '[':
198 depth += 1
199 elif ch == ']':
200 depth -= 1
201 if depth == 0:
202 end_arr = idx
203 break
204 if end_arr == -1:
205 return []
206 arr_str = block[start_arr:end_arr + 1]
207 return parse_js_array(arr_str)
208
209
210def parse_page(html):
211 items = extract_data_items(html)
212 soup = BeautifulSoup(html, 'html.parser')
213 cards = soup.select('div.itemSearchBlock.itemSearchListItem')
214 html_items = []
215 for card in cards:
216 product_id = card.get('data-instrument-cd')
217 title_elem = card.select_one('p.ttl a')
218 title = title_elem.get_text(strip=True) if title_elem else ''
219 product_url = title_elem.get('href') if title_elem else ''
220 shop_elem = card.select_one('p.itemShopInfo a')
221 shop_name = shop_elem.get_text(strip=True) if shop_elem else ''
222 fixed_price_elem = card.select_one('p.fixedPrice span')
223 list_price = extract_price(fixed_price_elem.get_text()) if fixed_price_elem else None
224 price_elem = card.select_one('p.price')
225 price = extract_price(price_elem.get_text()) if price_elem else None
226 cond_elem = card.select_one('p.state span.tooltip')
227 condition = cond_elem.get_text(strip=True) if cond_elem else ''
228 reg_elem = card.select_one('ul.itemDateInfo li:nth-of-type(2)')
229 registered = None
230 if reg_elem:
231 txt = reg_elem.get_text(strip=True)
232 m = re.search(r'登録：(.+)', txt)
233 if m:
234 registered = m.group(1).strip()
235 img_elem = card.select_one('div.pic img')
236 image_url = img_elem.get('src') if img_elem else ''
237 if image_url.startswith('//'):
238 image_url = 'https:' + image_url
239 if product_url.startswith('/'):
240 product_url = 'https://www.digimart.net' + product_url
241 html_items.append({
242 'productId': product_id,
243 'title': title,
244 'shopName': shop_name,
245 'listPrice': list_price,
246 'price': price,
247 'condition': condition,
248 'registeredAt': registered,
249 'imageUrl': image_url,
250 'productUrl': product_url,
251 })
252
253 data_map = {}
254 for it in items:
255 pid = it.get('item_id')
256 if pid:
257 data_map[str(pid)] = it
258 html_map = {}
259 for it in html_items:
260 pid = it.get('productId')
261 if pid:
262 html_map[str(pid)] = it
263
264 all_ids = set(data_map.keys()) | set(html_map.keys())
265 results = []
266 instru_labels = {
267 '1': '新品',
268 '2': '中古',
269 '3': 'ビンテージ',
270 '4': '新品アウトレット',
271 '5': 'B-stock',
272 }
273 now = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')
274
275 for pid in all_ids:
276 d = data_map.get(pid, {})
277 h = html_map.get(pid, {})
278 product = {}
279 product['productId'] = pid
280 product['title'] = d.get('item_name') or h.get('title')
281 product['brand'] = d.get('item_brand')
282 product['category'] = d.get('item_category')
283 product['price'] = d.get('price') or h.get('price')
284 product['listPrice'] = h.get('listPrice')
285 product['condition'] = h.get('condition')
286 instru = d.get('instru_type')
287 product['instruType'] = instru
288 if instru is not None:
289 product['instruTypeLabel'] = instru_labels.get(str(instru))
290 else:
291 product['instruTypeLabel'] = None
292 product['inStock'] = d.get('in_stock') if 'in_stock' in d else True
293 product['shopName'] = h.get('shopName')
294 product['registeredAt'] = h.get('registeredAt')
295 product['imageUrl'] = h.get('imageUrl')
296 product['productUrl'] = h.get('productUrl')
297 product['scrapedAt'] = now
298 results.append(product)
299
300 return results
301
302
303async def fetch_page(client, url):
304 for attempt in range(3):
305 try:
306 resp = await client.get(url)
307 if resp.status_code == 200:
308 return resp.text
309 if resp.status_code in (429, 500, 502, 503, 504):
310 await asyncio.sleep(2 ** attempt)
311 continue
312 return None
313 except Exception:
314 await asyncio.sleep(2 ** attempt)
315 return None
