Eventbrite Event Scraper
Pricing
$3.50 / 1,000 results
Scrape upcoming Eventbrite events by city and category: title, dates, venue, price range and organizer name. No personal data collected, robots.txt respected, rate-limited crawling.
Pricing
$3.50 / 1,000 results
Rating
0.0
(0)
Developer
Ferit Töngemen
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
a day ago
Last modified
Categories
Share
testattacktoolsdocsstorage*.logprobe.mjsprobe-urls.jsonprobe-b-urls.jsonDEVELOPMENT.mdpatch*.mjsnode_modulesstorage.git.githubtestattacktoolsdocs*.log*.md!README.mdprobe.mjsprobe-urls.jsonprobe-b-urls.jsonDEVELOPMENT.mdpatch*.mjsnode_modules/storage/apify_storage/.env*.log{ "city": "San Francisco", "category": "music", "maxItems": 20, "futureOnly": true}1import { Actor, log } from 'apify';2import { CheerioCrawler } from 'crawlee';3
4import { RobotsGate } from './src/core/robots.js';5import { BlockTracker, hasBlockMarker } from './src/core/blockTracker.js';6import { crawlerLimits, jitterDelay, fetchWithRetry, LIMITS } from './src/core/politeness.js';7import { EVENTBRITE_POLICY, eventIdFromUrl } from './src/targets/eventbrite/urlPolicy.js';8import { parseEventPage, isPastEvent } from './src/targets/eventbrite/parser.js';9import { EVENT_SCHEMA } from './src/targets/eventbrite/schema.js';10import { validateRecord } from './src/core/schema.js';11import {12 PROBE_URL, parseBrowserInfo, proxyVerdict, chargeVerdict, reconcileCounters,13} from './src/core/preflight.js';14import {15 shardUrlsFromIndex, extractLocs, gunzipMaybe, parseBrowseUrl, couldMatchCity, resolveCity,16 selectBrowseUrls, listPagesNeeded, estimateRequests, parseDateInput, inDateRange,17 shortfallMessage, slugify, linkStockTarget, listsToTopUp, orderBrowseUrls, availableCategories,18 discoveryVerdict,19} from './src/targets/eventbrite/discovery.js';20
21const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36';22const BASE = 'https://www.eventbrite.com';23const SITEMAP_INDEX = `${BASE}/sitemap_xml/sitemap_index.xml`;24
25await Actor.init();26
27const input = (await Actor.getInput()) ?? {};28const {29 startUrls = [],30 city = null,31 category = null,32 subcategory = null,33 onlineOnly = false,34 maxItems = 50,35 futureOnly = true,36 dateFrom = null,37 dateTo = null,38} = input;39
40// --- tarih filtresi: cekilen sonuclari suzer, arama kapsamini degistirmez -------41const dateFromD = parseDateInput(dateFrom);42const dateToD = /^\d{4}-\d{2}-\d{2}$/.test(String(dateTo ?? ''))43 ? parseDateInput(`${dateTo}T23:59:59.999Z`) // "dahil": gunun sonuna kadar44 : parseDateInput(dateTo);45if (dateFrom && !dateFromD) await Actor.fail(`dateFrom ayristirilamadi: "${dateFrom}" (YYYY-MM-DD bekleniyor).`);46if (dateTo && !dateToD) await Actor.fail(`dateTo ayristirilamadi: "${dateTo}" (YYYY-MM-DD bekleniyor).`);47if (dateFromD && dateToD && dateFromD > dateToD) await Actor.fail('dateFrom, dateTo`dan sonra olamaz.');48const dateFilter = Boolean(dateFromD || dateToD);49
50// --- mod: startUrls verilmisse kesif ATLANIR ----------------------------------51const seeds = [];52for (const s of startUrls) {53 const url = typeof s === 'string' ? s : s?.url;54 if (url) seeds.push(url);55}56const cityInput = onlineOnly ? 'online' : (city ? String(city).trim() : null);57let mode;58if (seeds.length > 0) mode = 'startUrls';59else if (cityInput) mode = 'discovery';60else await Actor.fail('Girdi yok: city (ya da onlineOnly) veya startUrls verin.');61
62// --- plan ve istek tavani --------------------------------------------------------63// Kesif: 6 sabit (1 indeks + 5 parca) + liste sayfalari + etkinlik sayfalari.64const linkTarget = linkStockTarget(maxItems, { dateFilter, futureOnly });65const plan = mode === 'discovery'66 ? estimateRequests(maxItems, { linkTarget })67 : { fixed: 0, lists: seeds.filter((u) => !/\/e\//.test(u)).length, events: maxItems, total: seeds.length + maxItems };68const hardCap = Math.max(plan.total * 2, plan.total + 10);69log.info(`Mod: ${mode}. Plan: ${JSON.stringify(plan)}. Istek tavani: ${hardCap}.`);70
71// --- politika denetimi: kuyruga girmeden once ------------------------------------72const allowedSeeds = [];73for (const u of seeds) {74 const c = EVENTBRITE_POLICY.check(u);75 if (c.ok) allowedSeeds.push(u);76 else log.warning(`URL politikasi reddetti, atlaniyor: ${u} (${c.reason})`);77}78if (mode === 'startUrls' && allowedSeeds.length === 0) {79 await Actor.fail('Tum giris URL`leri politika tarafindan reddedildi.');80}81
82// --- cekirdek bilesenler ---------------------------------------------------------83// robots.txt tek istektir ve butun kosuyu belirler: gecici ag hatasinda uc deneme,84// hepsi basarisizsa RobotsGate failClosed ile kapali kalir.85const robots = new RobotsGate({86 userAgent: UA,87 failClosed: true,88 log,89 fetchFn: (u) => fetchWithRetry(u, { init: { headers: { 'user-agent': UA } }, log }),90});91
92let crawler; // asagida atanir93const blocks = new BlockTracker({94 maxConsecutive: 5,95 log,96 onAbort: async (stats) => {97 log.error(`Engel esigi asildi, tarama durduruluyor: ${JSON.stringify(stats)}`);98 await crawler?.autoscaledPool?.abort();99 },100});101
102const counters = {103 pushed: 0, skippedPast: 0, dateFiltered: 0, dropped: 0, forbidden: 0, parseFail: 0,104 policyBlocked: 0, robotsBlocked: 0, piiMasked: 0, truncated: 0, organizerRejected: 0,105 stripped: 0, depthTrimmed: 0, charged: 0, schemaFail: 0, pushFail: 0,106 eventPages: 0, duplicate: 0, overMaxItems: 0,107 sitemapIndex: 0, sitemapIndexFail: 0, sitemapShards: 0, sitemapShardFail: 0, listPages: 0, listPagesEmpty: 0,108 eventLinksFound: 0, eventLinksEnqueued: 0, dBlocked: 0,109};110const schemaViolations = new Set();111let chargeLimitReached = false;112const egress = { direct: null, proxied: null };113
114// Kesif durumu115const disc = {116 citySlug: cityInput ? slugify(cityInput) : null,117 shardsTotal: 0,118 shardsDone: 0,119 allCities: new Set(), // aday listeleri icin (sehir bulunamazsa)120 candidates: [], // couldMatchCity ile on-elenmis /b/ URL'leri (sitemap sirasi)121 resolved: null,122 matched: [], // secilen /b/ URL'leri123 matchedEnqueued: 0,124 listsEnqueued: 0, // gercekten kuyruga giren liste sayfasi125 finalized: false,126};127const seenIds = new Set(); // yazilan etkinlik kimlikleri128const eventLinks = new Set(); // kuyruga alinan etkinlik URL'leri129const eventLinksNeeded = () => linkTarget;130
131/** Politika + robots birlikte; ikisi de gecerse true. */132async function mayVisit(url) {133 const pol = EVENTBRITE_POLICY.check(url);134 if (!pol.ok) { counters.policyBlocked += 1; return false; }135 if (!(await robots.isAllowed(url))) { counters.robotsBlocked += 1; return false; }136 return true;137}138
139function labelFor(url) {140 const p = new URL(url).pathname;141 if (/^\/e\//.test(p)) return 'EVENT';142 if (/^\/sitemap_xml\//.test(p)) return 'SITEMAP_INDEX';143 return 'LIST'; // /b/, /d/, /o/, /directory/sitemap/144}145
146async function enqueueLists(n) {147 let eklendi = 0;148 while (eklendi < n && disc.matchedEnqueued < disc.matched.length) {149 const u = disc.matched[disc.matchedEnqueued];150 disc.matchedEnqueued += 1;151 if (!(await mayVisit(u))) continue;152 await crawler.addRequests([{ url: u, label: 'LIST' }]);153 eklendi += 1;154 }155 disc.listsEnqueued += eklendi;156 return eklendi;157}158
159/**160 * Bag stogu yetmeyecekse liste sayfasi ekle. Kuyrukta bekleyen sayfalarin beklenen161 * verimi dusulur; verim gozlenen ortalamadir. (Kor +2, 5 yerine 13 sayfa cekmisti.)162 */163async function topUpLists() {164 const pending = disc.listsEnqueued - counters.listPages;165 const observedYield = counters.listPages ? counters.eventLinksFound / counters.listPages : 0;166 const n = listsToTopUp({167 stock: eventLinks.size,168 needed: eventLinksNeeded(),169 pending,170 observedYield,171 available: disc.matched.length - disc.matchedEnqueued,172 });173 if (n > 0) await enqueueLists(n);174}175
176/** Tum parcalar indikten sonra: sehri coz, /b/ sayfalarini sec, ilk partiyi kuyruga al. */177async function finalizeDiscovery() {178 if (disc.finalized) return;179 disc.finalized = true;180
181 if (counters.sitemapShards === 0) {182 await Actor.fail(`KESIF BASLAYAMADI: ${disc.shardsTotal} sitemap parcasinin hicbiri alinamadi (${counters.sitemapShardFail} basarisiz).`);183 }184 const r = resolveCity(cityInput, disc.allCities);185 if (!r.ok) {186 await Actor.fail(187 `Sehir cozulemedi: ${r.reason}. ` +188 (r.candidates.length ? `Adaylar: ${r.candidates.join(', ')}. Tam slug verin.` : 'Aday yok.'),189 );190 }191 disc.resolved = r;192 // Deterministik: ana sehir once, mahalleler sonra, kendi icinde URL sirasi (parca inis sirasindan bagimsiz).193 disc.matched = orderBrowseUrls(194 selectBrowseUrls(disc.candidates, { citySlugs: r.slugs, category, subcategory }),195 r.base,196 );197 const kapsam = `${r.base}${category ? `/${slugify(category)}` : ''}${subcategory ? `/${slugify(subcategory)}` : ''}`;198 log.info(`Sehir: "${cityInput}" -> ${r.base} (${r.slugs.length} slug). ${kapsam} icin eslesen /b/ sayfasi: ${disc.matched.length}.`);199 if (disc.matched.length === 0) {200 // Kullanici ne yazacagini bilsin: bu sehir icin sitemap'te var olan slug'lar.201 const kategoriler = availableCategories(disc.candidates, { citySlugs: r.slugs });202 let ipucu;203 if (category && kategoriler.includes(slugify(category))) {204 const altlar = availableCategories(disc.candidates, { citySlugs: r.slugs, category });205 ipucu = `"${slugify(category)}" gecerli ama alt kategori degil. Gecerli alt kategoriler (${altlar.length}): ${altlar.join(', ') || 'yok'}.`;206 } else {207 ipucu = `Gecerli kategoriler (${kategoriler.length}): ${kategoriler.join(', ') || 'yok'}.`;208 }209 await Actor.fail(`${kapsam} icin sitemap'te /b/ sayfasi yok. ${ipucu}`);210 }211 const ilkParti = listPagesNeeded(eventLinksNeeded());212 const eklendi = await enqueueLists(ilkParti);213 log.info(`Ilk parti: ${eklendi} /b/ sayfasi kuyruga alindi (plan ${ilkParti}, eslesen ${disc.matched.length}).`);214}215
216const proxyConfiguration = await Actor.createProxyConfiguration();217const isAtHome = Actor.isAtHome();218log.info(`Ortam: isAtHome=${isAtHome}, proxyConfiguration=${proxyConfiguration ? 'var' : 'YOK'}`);219
220// Dogrudan (proxysiz) cikis IP'si -- proxy karsilastirmasinin kontrol olcumu.221try {222 const res = await fetch(PROBE_URL, { headers: { 'user-agent': UA } });223 egress.direct = parseBrowserInfo(await res.text());224} catch (e) {225 log.warning(`Dogrudan cikis IP'si okunamadi: ${e.message}`);226}227
228crawler = new CheerioCrawler({229 ...crawlerLimits(),230 proxyConfiguration,231 maxRequestsPerCrawl: hardCap,232 maxSessionRotations: 2, // varsayilan 10: engele karsi oturum dondurmek zorlamadir, iki deneme yeter233 requestHandlerTimeoutSecs: 60,234 navigationTimeoutSecs: 45, // varsayilan 30; proxy uzerinden tek deneme 30 sn'yi asti235 additionalMimeTypes: [236 'application/xml', 'text/xml',237 'application/gzip', 'application/x-gzip', 'application/octet-stream', 'binary/octet-stream',238 ],239 preNavigationHooks: [240 async ({ request }) => {241 const sitemap = request.label === 'SITEMAP_INDEX' || request.label === 'SITEMAP_SHARD';242 request.headers = {243 ...request.headers,244 'user-agent': UA,245 accept: sitemap ? '*/*' : 'text/html,application/xhtml+xml,*/*',246 'accept-language': 'en-US,en;q=0.9',247 };248 await jitterDelay(LIMITS.minDelayMs, LIMITS.maxDelayMs);249 },250 ],251
252 async requestHandler({ request, body, $, proxyInfo }) {253 // ---- proxy olcum istegi (crawler'in gercek cikis yolu) --------------------254 if (request.label === 'PROBE') {255 egress.proxied = parseBrowserInfo(body?.toString?.() ?? '');256 log.info(`Proxy olcumu: ${JSON.stringify(egress.proxied)} | proxyInfo.hostname=${proxyInfo?.hostname ?? 'yok'} sessionId=${proxyInfo?.sessionId ?? 'yok'}`);257 return;258 }259
260 // ---- sitemap parcasi (gz, ikili): engel isareti aranmaz, acilip ayristirilir --261 if (request.label === 'SITEMAP_SHARD') {262 const xml = gunzipMaybe(body);263 const locs = extractLocs(xml);264 counters.sitemapShards += 1;265 disc.shardsDone += 1;266 if (locs.length === 0) log.warning(`Parca bos ya da acilamadi: ${request.url}`);267 let aday = 0;268 for (const loc of locs) {269 const p = parseBrowseUrl(loc);270 if (!p) continue;271 disc.allCities.add(p.city);272 if (couldMatchCity(p.city, disc.citySlug)) { disc.candidates.push(loc); aday += 1; }273 }274 log.info(`Parca ${disc.shardsDone}/${disc.shardsTotal}: ${locs.length} URL, ${aday} sehir adayi.`);275 if (disc.shardsDone >= disc.shardsTotal) await finalizeDiscovery();276 return;277 }278
279 const bodyText = body?.toString?.() ?? '';280 if (blocks.recordResponse({ status: 200, body: bodyText, url: request.url })) {281 return; // govdede challenge: engel sayildi, islenmez282 }283
284 // ---- sitemap indeksi --------------------------------------------------------285 if (request.label === 'SITEMAP_INDEX') {286 const shards = shardUrlsFromIndex(bodyText);287 counters.sitemapIndex += 1;288 if (shards.length === 0) {289 await Actor.fail('Sitemap indeksinde category_browse_pages parcasi yok; kesif yapilamiyor.');290 }291 const istekler = [];292 for (const u of shards) if (await mayVisit(u)) istekler.push({ url: u, label: 'SITEMAP_SHARD' });293 disc.shardsTotal = istekler.length;294 await crawler.addRequests(istekler);295 log.info(`Sitemap indeksi: ${shards.length} category_browse_pages parcasi, ${istekler.length} kuyruga alindi.`);296 return;297 }298
299 // ---- liste sayfasi: /b/ (kesif), /d/ ve /o/ (yalniz startUrls) -----------------300 if (request.label === 'LIST') {301 const path = new URL(request.url).pathname;302 const hrefs = new Set();303 $('a[href]').each((_, el) => {304 const href = $(el).attr('href');305 if (!href) return;306 let abs;307 try { abs = new URL(href, request.url).href.split('#')[0].split('?')[0]; } catch { return; }308 if (/\/e\/[^/?#]+/.test(new URL(abs).pathname)) hrefs.add(abs);309 });310 counters.listPages += 1;311 if (hrefs.size === 0) counters.listPagesEmpty += 1;312 if (/^\/o\//.test(path)) {313 log.warning(`Organizator sayfasi statik HTML'de etkinlik tasimiyor (${hrefs.size} bag): ${request.url}`);314 }315 let queued = 0;316 for (const href of hrefs) {317 if (eventLinks.has(href)) continue;318 counters.eventLinksFound += 1;319 if (eventLinks.size >= eventLinksNeeded()) continue;320 if (!(await mayVisit(href))) continue;321 eventLinks.add(href);322 // izlenebilirlik: kayit hangi liste sayfasindan bulundu323 await crawler.addRequests([{ url: href, label: 'EVENT', userData: { sourceListUrl: request.url } }]);324 queued += 1;325 }326 counters.eventLinksEnqueued += queued;327 log.info(`Liste ${path}: ${hrefs.size} etkinlik bagi, ${queued} kuyruga alindi (stok ${eventLinks.size}/${eventLinksNeeded()}).`);328 if (mode === 'discovery') await topUpLists();329 return;330 }331
332 // ---- etkinlik sayfasi ----------------------------------------------------------333 // Her cikis yolu bir sayaca yazmali; toplamlari eventPages'e esit olmali.334 counters.eventPages += 1;335 const parsed = parseEventPage(bodyText, request.url);336 if (!parsed.ok) {337 counters.parseFail += 1;338 log.warning(`Ayristirilamadi (${parsed.reason}): ${request.url}`);339 return;340 }341
342 counters.dropped += parsed.dropped.length;343 counters.truncated += parsed.truncated.length;344 counters.piiMasked += parsed.meta.piiMasked;345 if (parsed.forbiddenHits.length) {346 counters.forbidden += parsed.forbiddenHits.length;347 log.error(`YASAKLI ALAN: ${parsed.forbiddenHits.join(', ')} @ ${request.url}`);348 }349 // O1: derin taramada yakalanan alanlar kayittan dusuruldu; ne dusuruldugu loglanir.350 if (parsed.stripped.length) {351 counters.stripped += parsed.stripped.length;352 log.warning(`KAYITTAN DUSURULDU (yasakli, derin): ${parsed.stripped.join(', ')} @ ${request.url}`);353 }354 if (parsed.depthTrimmed.length) {355 counters.depthTrimmed += parsed.depthTrimmed.length;356 log.warning(`KAYITTAN DUSURULDU (derinlik siniri): ${parsed.depthTrimmed.join(', ')} @ ${request.url}`);357 }358 if (parsed.meta.piiMasked > 0) {359 log.info(`PII maskelendi (${JSON.stringify(parsed.meta.piiKinds)}) @ ${request.url}`);360 }361 if (parsed.meta.organizerRejected) {362 counters.organizerRejected += 1;363 log.info(`Organizator @type=${parsed.meta.organizerRejectedType}, ad alinmadi: ${request.url}`);364 }365
366 // sourceListUrl sayfadan degil kuyruktan gelir; dogrudan /e/ tohumunda null kalir.367 const rec = { ...parsed.record, sourceListUrl: request.userData?.sourceListUrl ?? null };368 if (futureOnly && isPastEvent(rec)) { counters.skippedPast += 1; return; }369 if (!inDateRange(rec, dateFromD, dateToD)) { counters.dateFiltered += 1; return; }370
371 const id = rec.id ?? eventIdFromUrl(request.url) ?? request.url;372 if (seenIds.has(id)) { counters.duplicate += 1; return; }373 seenIds.add(id); // yazma basarisiz olursa asagida geri alinir374
375 if (counters.pushed >= maxItems) { counters.overMaxItems += 1; return; }376
377 // P1: platform her kaydi dataset semasina gore dogruluyor, yerel depolama378 // dogrulamiyor. Ayni denetimi ISTEK YAPMADAN once, alan adi vererek yap.379 const violations = validateRecord(rec, EVENT_SCHEMA);380 if (violations.length) {381 counters.schemaFail += 1;382 violations.forEach((v) => schemaViolations.add(v));383 seenIds.delete(id); // tekrar denenirse sessizce atlanmasin384 log.error(`SEMA UYUSMAZLIGI (${request.url}): ${violations.join('; ')}`);385 return; // determinist hata: yeniden denemek ayni sonucu verir386 }387
388 // Actor.pushData(item, eventName): yazma ve ucretlendirme tek cagri.389 let charge;390 try {391 charge = await Actor.pushData(rec, 'result');392 } catch (e) {393 // P2: seenIds'e push'tan ONCE eklenirse, hata sonrasi yeniden deneme394 // 'zaten gorduk' deyip sessizce doner ve istek BASARILI sayilir.395 seenIds.delete(id);396 const semaHatasi = /schema validation/i.test(e?.message ?? '');397 if (semaHatasi) {398 counters.schemaFail += 1;399 schemaViolations.add(e.message);400 log.error(`PLATFORM SEMA HATASI (${request.url}): ${e.message}`);401 return; // determinist: yeniden deneme bos yere istek harcar402 }403 counters.pushFail += 1;404 log.error(`DATASET YAZMA HATASI (${request.url}): ${e?.message ?? e}`);405 throw e; // gecici olabilir, Crawlee yeniden denesin406 }407 counters.pushed += 1;408 counters.charged += charge?.chargedCount ?? 0;409 if (charge?.eventChargeLimitReached) {410 chargeLimitReached = true;411 log.warning('Kullanicinin harcama tavani doldu, tarama durduruluyor.');412 await crawler.autoscaledPool?.abort();413 return;414 }415
416 if (counters.pushed >= maxItems) {417 log.info(`maxItems (${maxItems}) doldu, durduruluyor.`);418 await crawler.autoscaledPool?.abort();419 }420 },421
422 async failedRequestHandler({ request, response }) {423 const status = response?.statusCode ?? 0;424 const marker = hasBlockMarker(response?.body);425 blocks.recordResponse({ status, body: response?.body, url: request.url });426 const path = new URL(request.url).pathname;427 if (status === 405 || marker === 'human verification') {428 if (/^\/d\//.test(path)) {429 counters.dBlocked += 1;430 log.warning(`Liste yolu (/d/) engellendi, yeniden DENENMEYECEK: ${request.url}. Bu yol guvenilir degil; sehir/kategori girdisini ya da /b/ kullanin.`);431 } else {432 log.warning(`405/insan dogrulamasi, yeniden DENENMEYECEK: ${request.url}`);433 }434 return;435 }436 if (request.label === 'SITEMAP_SHARD') {437 disc.shardsDone += 1;438 counters.sitemapShardFail += 1;439 log.error(`Sitemap parcasi alinamadi (${status}): ${request.url}`);440 if (disc.shardsDone >= disc.shardsTotal) await finalizeDiscovery();441 return;442 }443 if (request.label === 'SITEMAP_INDEX') {444 counters.sitemapIndexFail += 1;445 log.error(`Sitemap indeksi alinamadi (${status}): ${request.url} -- kesif baslayamiyor`);446 return;447 }448 log.warning(`Istek basarisiz (${status}): ${request.url}`);449 },450});451
452// --- tohumlar --------------------------------------------------------------------453const finalSeeds = [];454if (mode === 'discovery') {455 if (await mayVisit(SITEMAP_INDEX)) finalSeeds.push({ url: SITEMAP_INDEX, label: 'SITEMAP_INDEX' });456 else await Actor.fail('robots.txt sitemap indeksine izin vermiyor; kesif yapilamiyor.');457} else {458 for (const u of allowedSeeds) {459 if (!(await mayVisit(u))) { log.warning(`robots.txt reddetti, atlaniyor: ${u}`); continue; }460 const label = labelFor(u);461 const req = { url: u, label };462 if (/^\/d\//.test(new URL(u).pathname)) req.noRetry = true; // tek deneme, 405'i zorlama463 if (label === 'EVENT') eventLinks.add(u.split('#')[0].split('?')[0]);464 finalSeeds.push(req);465 }466 if (finalSeeds.length === 0) await Actor.fail('Tum giris URL`leri robots.txt ya da politika tarafindan reddedildi.');467}468
469log.info(`Baslaniyor: mod=${mode}, ${finalSeeds.length} tohum, maxItems=${maxItems}, futureOnly=${futureOnly}, tarih=${dateFrom ?? '-'}..${dateTo ?? '-'}`);470await crawler.run([{ url: PROBE_URL, label: 'PROBE' }, ...finalSeeds]);471
472// --- butce: plan vs gercek ---------------------------------------------------------473const st = crawler.stats?.state ?? {};474const actualRequests = (st.requestsFinished ?? 0) + (st.requestsFailed ?? 0) - 1; // PROBE plana dahil degil475const hardCapHit = actualRequests + 1 >= hardCap;476const budget = {477 plan,478 hardCap,479 actual: actualRequests,480 actualPerRecord: counters.pushed ? Number((actualRequests / counters.pushed).toFixed(2)) : null,481 planPerRecord: Number((plan.total / plan.events).toFixed(2)),482 hardCapHit,483};484
485// --- hukumler ------------------------------------------------------------------------486const proxyCheck = proxyVerdict({487 direct: egress.direct, proxied: egress.proxied, isAtHome, proxyConfigured: Boolean(proxyConfiguration),488});489const chargeCheck = chargeVerdict({490 isAtHome, pushed: counters.pushed, chargedCount: counters.charged, limitReached: chargeLimitReached,491});492
493// Kesif hic baslayamadiysa "sayfa yok" uyarisi yaniltici olur: once bu, gurultulu.494const kesifHukmu = mode === 'discovery'495 ? discoveryVerdict({496 sitemapIndex: counters.sitemapIndex,497 indexFailed: counters.sitemapIndexFail,498 shardsTotal: disc.shardsTotal,499 shardsDone: disc.shardsDone,500 shardsFailed: counters.sitemapShardFail,501 })502 : { ok: true, partial: false, reason: '-' };503if (!kesifHukmu.ok) {504 await Actor.setValue('RUN_STATS', { mode, ...counters, kesifHukmu, egress });505 await Actor.fail(`KESIF BASLAYAMADI: ${kesifHukmu.reason}. Sehir/kategori sonucu degil, ag/engel sorunu.`);506}507if (kesifHukmu.partial) log.warning(`KESIF EKSIK: ${kesifHukmu.reason}`);508
509let shortfall = null;510if (counters.pushed < maxItems) {511 shortfall = mode === 'discovery'512 ? shortfallMessage({513 requested: maxItems,514 found: counters.pushed,515 listPagesScanned: counters.listPages,516 listPagesAvailable: disc.matched.length,517 city: disc.resolved?.base ?? disc.citySlug,518 category: category ? slugify(category) : null,519 dateFiltered: counters.dateFiltered,520 pastFiltered: counters.skippedPast,521 hardCapHit,522 blocked: blocks.stats().totalBlocks ?? 0,523 })524 : `${maxItems} istendi, ${counters.pushed} bulundu; ${finalSeeds.length} tohum, ${counters.listPages} liste sayfasi, ${counters.dBlocked} /d/ engeli, ${counters.skippedPast} gecmis, ${counters.dateFiltered} tarih disi.`;525 log.warning(`maxItems KARSILANAMADI: ${shortfall}`);526}527
528const stats = {529 mode,530 ...counters,531 ...blocks.stats(),532 semaAlanSayisi: Object.keys(EVENT_SCHEMA.fields).length,533 isAtHome,534 egress,535 proxy: proxyCheck,536 ucretlendirme: chargeCheck,537 semaIhlalleri: [...schemaViolations],538 muhasebe: reconcileCounters(counters),539 kesif: mode === 'discovery' ? {540 sehir: disc.resolved?.base ?? null,541 slugSayisi: disc.resolved?.slugs?.length ?? 0,542 eslesenListe: disc.matched.length,543 taranan: counters.listPages,544 } : null,545 butce: budget,546 eksik: shortfall,547};548log.info(`Bitti: ${JSON.stringify(stats)}`);549log.info(`PROXY: ${proxyCheck.verdict} -- ${proxyCheck.detail}`);550log.info(`UCRETLENDIRME: ${chargeCheck.verdict} -- ${chargeCheck.detail}`);551log.info(`BUTCE: plan ${plan.total} istek, gercek ${actualRequests}; kayit basina plan ${budget.planPerRecord}, gercek ${budget.actualPerRecord ?? '-'}${hardCapHit ? ' [TAVAN DOLDU]' : ''}`);552if (counters.forbidden > 0) log.error('DIKKAT: yasakli alan sayaci sifir degil, beyaz liste gozden gecirilmeli.');553if (isAtHome && !proxyCheck.ok) log.error(`DIKKAT: ${proxyCheck.detail}`);554if (isAtHome && !chargeCheck.ok) log.error(`DIKKAT: ${chargeCheck.detail}`);555await Actor.setValue('RUN_STATS', stats);556
557// Sema uyusmazligi tek basina bile kosuyu dusurur: 'basarili ama bos' bitis yok.558if (counters.schemaFail > 0) {559 await Actor.fail(560 `SEMA UYUSMAZLIGI: ${counters.schemaFail} kayit datasete yazilamadi. ` +561 `Ihlaller: ${[...schemaViolations].join(' | ')}. ` +562 '.actor/dataset_schema.json ile parser ciktisi ayristi.',563 );564}565if (counters.pushFail > 0) {566 await Actor.fail(`DATASET YAZMA HATASI: ${counters.pushFail} kayit yazilamadi.`);567}568
569// Bos dataset: gercekten bos bir sehirden ayirt etmek icin sadece hata varken duser.570const engelVar = (blocks.stats().totalBlocks ?? 0) > 0 || counters.parseFail > 0;571if (counters.pushed === 0 && engelVar) {572 await Actor.fail(573 `Hic kayit yazilamadi; ${blocks.stats().totalBlocks} engel, ${counters.parseFail} ayristirma hatasi. ` +574 'Datacenter IP adresi Eventbrite tarafindan dogrulamaya dusuruyor olabilir.',575 );576}577// Sayac muhasebesi: etkinlik sayfasina giren her istek bir cikis yoluna yazilmali.578const muhasebe = reconcileCounters(counters);579if (!muhasebe.ok) {580 await Actor.fail(`SAYAC TUTMUYOR: ${muhasebe.detail}. Sayilmayan bir dal var; sessiz kayip.`);581}582
583await Actor.exit();{ "name": "apify-actors", "version": "0.1.0", "type": "module", "description": "Ortak cekirdek + Eventbrite actor", "engines": { "node": ">=22" }, "scripts": { "start": "node main.js", "test": "node --test", "parse": "node tools/parse-file.mjs", "robots": "node tools/check-robots.mjs" }, "dependencies": { "apify": "^3.2.6", "cheerio": "^1.0.0", "crawlee": "^3.11.5" }}{ "actorSpecification": 1, "name": "eventbrite-events-scraper", "title": "Eventbrite Event Scraper", "description": "Sehir ve kategoriye gore Eventbrite etkinlikleri: baslik, tarih, mekan, fiyat, organizator adi. Sitemap + /b/ uzerinden kesif; kisisel veri yok; sonuc basina odeme.", "version": "0.1", "meta": { "templateId": "js-crawlee-cheerio" }, "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": "./dataset_schema.json" }, "minMemoryMbytes": 512, "defaultMemoryMbytes": 1024, "maxMemoryMbytes": 4096}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "id": { "type": [ "string", "null" ] }, "url": { "type": [ "string", "null" ] }, "title": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "startDate": { "type": [ "string", "null" ] }, "endDate": { "type": [ "string", "null" ] }, "timezone": { "type": [ "string", "null" ] }, "venueName": { "type": [ "string", "null" ] }, "city": { "type": [ "string", "null" ] }, "country": { "type": [ "string", "null" ] }, "isOnline": { "type": [ "boolean", "null" ] }, "price": { "type": [ "object", "null" ] }, "category": { "type": [ "string", "null" ] }, "organizerName": { "type": [ "string", "null" ] }, "organizerUrl": { "type": [ "string", "null" ] }, "attendeeCount": { "type": [ "number", "null" ] }, "sourceListUrl": { "type": [ "string", "null" ] }, "scrapedAt": { "type": [ "string", "null" ] } } }, "views": { "overview": { "title": "Etkinlikler", "transformation": { "fields": [ "title", "startDate", "city", "isOnline", "price", "organizerName", "url", "sourceListUrl" ] }, "display": { "component": "table", "properties": { "title": { "label": "Etkinlik" }, "startDate": { "label": "Baslangic", "format": "date" }, "city": { "label": "Sehir" }, "isOnline": { "label": "Cevrim ici", "format": "boolean" }, "price": { "label": "Fiyat", "format": "object" }, "organizerName": { "label": "Organizator" }, "url": { "label": "URL", "format": "link" }, "sourceListUrl": { "label": "Bulundugu liste", "format": "link" } } } } }}FROM apify/actor-node:24
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" && node --version \ && echo "NPM version:" && npm --version \ && rm -r ~/.npm
COPY . ./
CMD npm start --silent{ "title": "Eventbrite Event Scraper", "type": "object", "schemaVersion": 1, "properties": { "city": { "title": "Sehir", "type": "string", "description": "Sehir adi (\"San Francisco\", \"Istanbul\") ya da Eventbrite slug'i (\"ca--san-francisco\"). Ad birden fazla yere uyarsa kosu durur ve adaylari listeler; o zaman tam slug verin. Cevrim ici etkinlikler icin \"online\" yazin. startUrls verilmisse yok sayilir.", "editor": "textfield", "prefill": "San Francisco" }, "category": { "title": "Kategori", "type": "string", "description": "Eventbrite kategori slug'i: music, business, arts, sports-and-fitness, food-and-drink, technology... Bos birakilirsa sehirdeki tum kategoriler.", "editor": "textfield" }, "subcategory": { "title": "Alt kategori", "type": "string", "description": "Eventbrite alt kategori slug'i (ornek: music icin rock, jazz, classical). Yalniz category ile birlikte anlamli.", "editor": "textfield" }, "startUrls": { "title": "Baslangic URL'leri (ileri kullanim)", "type": "array", "description": "Verilirse sehir/kategori kesfi ATLANIR. Etkinlik (/e/) ve gezinme (/b/) URL'leri guvenilir. Liste (/d/) yollari guvenilir degil: tek deneme yapilir, engellenirse yeniden denenmez; sehir/kategori girdisini ya da /b/ kullanin. Organizator (/o/) sayfalari statik HTML'de etkinlik tasimiyor.", "editor": "requestListSources" }, "maxItems": { "title": "En fazla kayit", "type": "integer", "description": "Kaydedilecek etkinlik sayisi ust siniri. Her kayit icin ucretlendirilirsiniz. Sehir/kategori icin yeterli sayfa yoksa bulunan kadari yazilir ve uyari loglanir.", "default": 50, "minimum": 1, "maximum": 10000 }, "futureOnly": { "title": "Yalniz gelecek etkinlikler", "type": "boolean", "description": "Gecmis etkinlikleri atar.", "default": true }, "dateFrom": { "title": "Baslangic tarihi (dahil)", "type": "string", "description": "YYYY-MM-DD. Bu filtre cekilen sonuclari suzer; arama kapsamini genisletmez veya daraltmaz. Istek sayisini azaltmaz; yalniz araliga girmeyen kayitlar yazilmaz.", "editor": "textfield" }, "dateTo": { "title": "Bitis tarihi (dahil)", "type": "string", "description": "YYYY-MM-DD. Bu filtre cekilen sonuclari suzer; arama kapsamini genisletmez veya daraltmaz. Istek sayisini azaltmaz; yalniz araliga girmeyen kayitlar yazilmaz.", "editor": "textfield" }, "onlineOnly": { "title": "Yalniz cevrim ici", "type": "boolean", "description": "city yerine \"online\" kullanir (city ile birlikte verilirse bu kazanir).", "default": false } }, "required": []}{ "actorOutputSchemaVersion": 1, "title": "Eventbrite Event Scraper output", "description": "Event rows are stored in the default dataset; per-run counters and verdicts in the default key-value store.", "properties": { "results": { "type": "string", "title": "Events", "description": "One row per event, 18 fields (see dataset schema). Missing values are null, never omitted.", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "runStats": { "type": "string", "title": "Run stats", "description": "Counters, request budget (plan vs actual), proxy and charging verdicts, discovery summary.", "template": "{{links.apiDefaultKeyValueStoreUrl}}/records/RUN_STATS" } }}{ "result": { "eventTitle": "Event record", "eventDescription": "Dataset'e yazilan her etkinlik kaydi", "eventPriceUsd": 0.0035 }}1/**2 * Engel sayaci.3 * 09 §1: ardisik 5 engelde durur.4 * O5 - EK: kayan pencerede engel orani esigi (son 50 istekte %50 engel -> dur).5 * Ardisik sayac araya giren basarilarla sifirlaniyordu; oran esigi bunu kapatir.6 */7
8export const BLOCK_STATUSES = [403, 405, 429];9
10export const BLOCK_MARKERS = [11 'just a moment',12 'human verification',13 'access denied',14 'request blocked',15 'are you a human',16 'verify you are human',17 'enable javascript and cookies',18];19
20export function hasBlockMarker(body) {21 if (!body) return null;22 const low = String(body).slice(0, 4000).toLowerCase();23 return BLOCK_MARKERS.find((m) => low.includes(m)) ?? null;24}25
26export class BlockTracker {27 constructor({28 maxConsecutive = 5,29 windowSize = 50,30 blockRateThreshold = 0.5,31 minSamples = 20,32 onAbort = null,33 log = null,34 } = {}) {35 this.maxConsecutive = maxConsecutive;36 this.windowSize = windowSize;37 this.blockRateThreshold = blockRateThreshold;38 this.minSamples = minSamples;39 this.onAbort = onAbort;40 this.log = log;41
42 this.consecutive = 0;43 this.byStatus = new Map();44 this.total = 0;45 this.successes = 0;46 this.aborted = false;47 this.abortReason = null;48 this.window = []; // true = engel49 }50
51 #push(isBlock) {52 this.window.push(isBlock);53 if (this.window.length > this.windowSize) this.window.shift();54 }55
56 blockRate() {57 if (this.window.length === 0) return 0;58 return this.window.filter(Boolean).length / this.window.length;59 }60
61 #evaluate() {62 if (this.aborted) return;63 if (this.consecutive >= this.maxConsecutive) {64 this.#abort(`ardisik ${this.consecutive} engel`);65 return;66 }67 if (this.window.length >= this.minSamples && this.blockRate() >= this.blockRateThreshold) {68 const pct = Math.round(this.blockRate() * 100);69 this.#abort(`son ${this.window.length} istekte %${pct} engel`);70 }71 }72
73 #abort(reason) {74 this.aborted = true;75 this.abortReason = reason;76 this.log?.error(`Engel esigi asildi (${reason}): durduruluyor.`);77 this.onAbort?.(this.stats());78 }79
80 recordSuccess() {81 this.consecutive = 0;82 this.successes += 1;83 this.#push(false);84 this.#evaluate();85 }86
87 /** @returns {boolean} true = bu yanit engel sayildi */88 recordResponse({ status, body = null, url = null } = {}) {89 const marker = hasBlockMarker(body);90 const isBlock = BLOCK_STATUSES.includes(status) || Boolean(marker);91 if (!isBlock) {92 this.recordSuccess();93 return false;94 }95 this.total += 1;96 this.consecutive += 1;97 this.#push(true);98 const key = marker ? `${status}:${marker}` : String(status);99 this.byStatus.set(key, (this.byStatus.get(key) ?? 0) + 1);100 this.log?.warning(101 `Engel ${key}${url ? ' @ ' + url : ''} (ardisik ${this.consecutive}/${this.maxConsecutive}, ` +102 `pencere %${Math.round(this.blockRate() * 100)}, toplam ${this.total})`,103 );104 this.#evaluate();105 return true;106 }107
108 shouldAbort() {109 return this.aborted;110 }111
112 stats() {113 return {114 totalBlocks: this.total,115 consecutive: this.consecutive,116 successes: this.successes,117 byStatus: Object.fromEntries(this.byStatus),118 windowSize: this.window.length,119 blockRatePct: Math.round(this.blockRate() * 100),120 aborted: this.aborted,121 abortReason: this.abortReason,122 };123 }124}1export { RobotsGate, parseRobots, selectGroup, isPathAllowed, pathToRegex } from './robots.js';2export { defineSchema, applyWhitelist, scanForbiddenDeep, DEFAULT_STRING_LIMIT } from './schema.js';3export { BlockTracker, hasBlockMarker, BLOCK_STATUSES, BLOCK_MARKERS } from './blockTracker.js';4export { LIMITS, jitterMs, jitterDelay, sleep, backoffMs, crawlerLimits } from './politeness.js';5export { extractJsonLd, hasUnparsableJsonLd, findByType, findByKnownType, typeOf, typesOf, MAX_GRAPH_DEPTH } from './jsonld.js';6export { UrlPolicy } from './urlPolicy.js';7export { maskPii, maskFields, looksLikePhone } from './pii.js';1/**2 * JSON-LD cikarimi.3 * O2 - regex yerine cheerio ile script dugumu. Olculmus kazanim: HTML spec'e uygun4 * kapanis biçimleri (ornek `</script >`) artik kaciriliyor degil, dogru okunuyor.5 * NOT: <script> ham metin elementidir; entity'ler COZULMEZ (tarayici da cozmez).6 * Kacirilmamis bir `</script>` HTML'i gercekten sonlandirir, ayristirilamaz -7 * bu durum artik SESSIZ degil: hasUnparsableJsonLd ile tanili raporlanir.8 * O3 - @graph duzlestirmesinde derinlik siniri (yigin tasmasi yok).9 */10import * as cheerio from 'cheerio';11
12export const MAX_GRAPH_DEPTH = 100;13
14/**15 * @param {string|Function} input - HTML dizesi ya da hazir cheerio `$`16 * @returns {object[]}17 */18export function extractJsonLd(input, { maxDepth = MAX_GRAPH_DEPTH } = {}) {19 const $ = typeof input === 'function' ? input : cheerio.load(String(input ?? ''));20 const out = [];21 const seen = new Set();22
23 $('script[type="application/ld+json"], script[type="application/ld+JSON"]').each((_, el) => {24 const text = $(el).text().trim();25 if (!text) return;26 let parsed;27 try {28 parsed = JSON.parse(text);29 } catch {30 return; // bozuk blok: atla31 }32 for (const node of flatten(parsed, maxDepth)) {33 if (!node || typeof node !== 'object' || seen.has(node)) continue;34 seen.add(node);35 out.push(node);36 }37 });38
39 return out;40}41
42/** Bir HTML'de ld+json betigi var mi ama hicbiri ayristirilamadi mi? Tani icin. */43export function hasUnparsableJsonLd(input) {44 const $ = typeof input === 'function' ? input : cheerio.load(String(input ?? ''));45 let found = 0;46 let bad = 0;47 $('script[type="application/ld+json"]').each((_, el) => {48 const t = $(el).text().trim();49 if (!t) return;50 found += 1;51 try { JSON.parse(t); } catch { bad += 1; }52 });53 return { found, bad };54}55
56/** O3: derinlik sinirli duzlestirme. Sinir asilirsa o dal birakilir. */57function flatten(node, maxDepth, depth = 0, acc = []) {58 if (depth > maxDepth) return acc;59 if (Array.isArray(node)) {60 for (const n of node) flatten(n, maxDepth, depth + 1, acc);61 return acc;62 }63 if (node && typeof node === 'object') {64 acc.push(node);65 if (Array.isArray(node['@graph'])) flatten(node['@graph'], maxDepth, depth + 1, acc);66 }67 return acc;68}69
70/** @type her zaman dizi olarak dondurulur (K1/D1 icin). */71export function typesOf(node) {72 const t = node?.['@type'];73 if (!t) return [];74 return Array.isArray(t) ? t.filter((x) => typeof x === 'string') : (typeof t === 'string' ? [t] : []);75}76
77export function typeOf(node) {78 return typesOf(node)[0] ?? null;79}80
81export function findByType(blocks, typePattern) {82 const re = typePattern instanceof RegExp ? typePattern : new RegExp(`^${typePattern}$`);83 return blocks.find((b) => typesOf(b).some((t) => re.test(t))) ?? null;84}85
86/** D1: bilinen tip beyaz listesiyle arama. */87export function findByKnownType(blocks, knownTypes) {88 const set = new Set(knownTypes);89 return blocks.find((b) => typesOf(b).some((t) => set.has(t))) ?? null;90}1/**2 * Serbest metin PII maskeleme (K2).3 * Beyaz liste yalniz ANAHTAR adina bakiyordu; bu modul DEGER icerigine bakar.4 *5 * ReDoS notu: tum niceleyiciler SINIRLI. Sinirsiz `+` ile yazilan ilk surum6 * 5 MB girdide katastrofik geri izleme yapiyordu (kendi regresyon testimizde yakalandi).7 * Ayrica MAX_SCAN ile girdi uzunlugu tavani var.8 */9
10export const MAX_SCAN = 20000; // bu uzunlugun otesi taranmaz (maskelenmeden birakilir)11
12const EMAIL_RE = /\b[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63}){1,4}\b/g;13const URL_RE = /\b(?:https?:\/\/|www\.)[^\s<>"')\]]{1,300}/gi;14const PHONE_CANDIDATE_RE = /\+?\d[\d\s().-]{7,18}\d/g;15
16const MASK = { email: '[e-posta]', phone: '[telefon]', url: '[url]' };17
18function digitCount(s) {19 let n = 0;20 for (let i = 0; i < s.length; i += 1) {21 const c = s.charCodeAt(i);22 if (c >= 48 && c <= 57) n += 1;23 }24 return n;25}26
27/** Telefon adayi gercekten telefon mu? 9-15 rakam kabul edilir. */28export function looksLikePhone(candidate) {29 const d = digitCount(candidate);30 return d >= 9 && d <= 15;31}32
33/**34 * @returns {{text: string, masked: number, kinds: Record<string, number>, scanTruncated: boolean}}35 */36export function maskPii(input, { maxScan = MAX_SCAN } = {}) {37 if (typeof input !== 'string' || input === '') {38 return { text: input, masked: 0, kinds: {}, scanTruncated: false };39 }40
41 // Tarama tavani: cok uzun metin once kirpilir (ReDoS ve maliyet korumasi)42 const scanTruncated = input.length > maxScan;43 let text = scanTruncated ? input.slice(0, maxScan) : input;44
45 const kinds = { url: 0, email: 0, phone: 0 };46 text = text.replace(URL_RE, () => { kinds.url += 1; return MASK.url; });47 text = text.replace(EMAIL_RE, () => { kinds.email += 1; return MASK.email; });48 text = text.replace(PHONE_CANDIDATE_RE, (m) => {49 if (!looksLikePhone(m)) return m;50 kinds.phone += 1;51 return MASK.phone;52 });53
54 const masked = kinds.url + kinds.email + kinds.phone;55 for (const k of Object.keys(kinds)) if (kinds[k] === 0) delete kinds[k];56 return { text, masked, kinds, scanTruncated };57}58
59/** Secili serbest metin alanlarini maskeler; toplam sayaci dondurur. */60export function maskFields(obj, fieldNames, opts = {}) {61 let masked = 0;62 let scanTruncated = false;63 const kinds = {};64 const out = { ...obj };65 for (const f of fieldNames) {66 const r = maskPii(out[f], opts);67 out[f] = r.text;68 masked += r.masked;69 if (r.scanTruncated) scanTruncated = true;70 for (const [k, v] of Object.entries(r.kinds)) kinds[k] = (kinds[k] ?? 0) + v;71 }72 return { obj: out, masked, kinds, scanTruncated };73}1/** Hiz sinirlama sabitleri ve yardimcilari (09 §1). */2
3export const LIMITS = {4 maxConcurrency: 2,5 maxRequestsPerMinute: 60,6 minDelayMs: 500,7 maxDelayMs: 1500,8 maxRetries: 3,9 backoffBaseMs: 1000,10 backoffCapMs: 30000,11};12
13/** [min, max] arasi rastgele bekleme. rnd enjekte edilebilir (test icin deterministik). */14export function jitterMs(min = LIMITS.minDelayMs, max = LIMITS.maxDelayMs, rnd = Math.random) {15 if (max < min) throw new Error('max < min');16 return Math.floor(min + rnd() * (max - min));17}18
19export function sleep(ms) {20 return new Promise((r) => setTimeout(r, ms));21}22
23export async function jitterDelay(min, max, rnd) {24 await sleep(jitterMs(min, max, rnd));25}26
27/** Ustel geri cekilme. attempt 0'dan baslar. Saf fonksiyon. */28export function backoffMs(attempt, base = LIMITS.backoffBaseMs, cap = LIMITS.backoffCapMs) {29 return Math.min(cap, base * 2 ** attempt);30}31
32/**33 * Yeniden denemeli, zaman asimli fetch. robots.txt gibi TEK istegin butun kosuyu34 * belirledigi yerler icin: gecici ag hatasi (fetch failed, zaman asimi, 5xx) ustel35 * geri cekilmeyle yeniden denenir; kalici hata (4xx) hemen doner.36 * Son deneme de basarisizsa hata FIRLATIR -- cagiran fail-closed karar verir.37 *38 * @param {string} url39 * @param {object} [opts]40 * @param {Function} [opts.fetchFn] enjekte edilebilir (test)41 * @param {Function} [opts.sleepFn] enjekte edilebilir (test)42 * @param {number} [opts.attempts] toplam deneme (varsayilan 3)43 * @param {number} [opts.timeoutMs] deneme basina zaman asimi44 * @param {object} [opts.init] fetch secenekleri (headers vb.)45 */46export async function fetchWithRetry(url, {47 fetchFn = globalThis.fetch,48 sleepFn = sleep,49 attempts = 3,50 timeoutMs = 10000,51 init = {},52 log = null,53} = {}) {54 let lastErr;55 for (let attempt = 0; attempt < attempts; attempt += 1) {56 try {57 const signal = typeof AbortSignal?.timeout === 'function' ? AbortSignal.timeout(timeoutMs) : undefined;58 const res = await fetchFn(url, { ...init, signal });59 if (res.status >= 500) throw new Error(`HTTP ${res.status}`); // gecici say60 return res; // 2xx, 3xx, 4xx: kalici cevap, cagiran yorumlar61 } catch (e) {62 lastErr = e;63 if (attempt < attempts - 1) {64 const wait = backoffMs(attempt);65 log?.warning?.(`${url}: deneme ${attempt + 1}/${attempts} basarisiz (${e?.message ?? e}), ${wait} ms sonra tekrar`);66 await sleepFn(wait);67 }68 }69 }70 throw new Error(`${attempts} denemede alinamadi: ${lastErr?.message ?? lastErr}`);71}72
73/** Crawlee CheerioCrawler icin ortak ayarlar. */74export function crawlerLimits(overrides = {}) {75 return {76 maxConcurrency: LIMITS.maxConcurrency,77 maxRequestsPerMinute: LIMITS.maxRequestsPerMinute,78 maxRequestRetries: LIMITS.maxRetries,79 ...overrides,80 };81}1/**2 * Ilk platform kosusunu KENDINI DOGRULAYAN hale getiren yardimcilar.3 *4 * Neden gerekli: SDK'nin proxy davranisi yerelde ve platformda farkli.5 * - Yerelde proxy sifresi yoksa uyari basar ve proxysiz kosar.6 * - Platformda ayni durum HATA firlatir.7 * - Ama input semasinda proxy alani "None" secilirse `{ useApifyProxy: false }` gelir,8 * createProxyConfiguration() `undefined` doner ve kosu SESSIZCE proxysiz gecer.9 * Bu yuzden "proxy devrede mi" sorusu log'a bakarak degil, CIKIS IP'SI OLCULEREK10 * yanitlanmali: ayni IP => proxy yok.11 *12 * Olcum ucu Apify'in kendi belgelerindeki "Test proxy" endpoint'i:13 * GET https://api.apify.com/v2/browser-info -> { clientIp, countryCode, ... }14 */15
16export const PROBE_URL = 'https://api.apify.com/v2/browser-info?skipHeaders=1';17
18/** browser-info govdesinden IP ve ulke kodu. Bozuk govdede null. */19export function parseBrowserInfo(bodyText) {20 if (typeof bodyText !== 'string' || bodyText === '') return null;21 let o;22 try { o = JSON.parse(bodyText); } catch { return null; }23 if (!o || typeof o !== 'object') return null;24 const clientIp = typeof o.clientIp === 'string' ? o.clientIp : null;25 if (!clientIp) return null;26 return { clientIp, countryCode: typeof o.countryCode === 'string' ? o.countryCode : null };27}28
29/**30 * Iki olcumu karsilastirip kesin hukum verir.31 * @param {{direct: ?object, proxied: ?object, isAtHome: boolean, proxyConfigured: boolean}} a32 * @returns {{ok: boolean, verdict: string, detail: string}}33 */34export function proxyVerdict({ direct, proxied, isAtHome, proxyConfigured }) {35 const d = direct?.clientIp ?? null;36 const p = proxied?.clientIp ?? null;37
38 if (!proxyConfigured) {39 return {40 ok: false,41 verdict: 'PROXY YOK',42 detail: isAtHome43 ? 'createProxyConfiguration() undefined dondu: girdide proxy "None" secilmis olabilir. Kosu dogrudan IP ile gecti.'44 : 'Yerel kosu: proxy nesnesi yok, bu yerelde beklenen davranis.',45 };46 }47 if (!p) return { ok: false, verdict: 'OLCULEMEDI', detail: 'Proxy uzerinden browser-info okunamadi.' };48 if (!d) return { ok: false, verdict: 'OLCULEMEDI', detail: `Dogrudan IP okunamadi; proxy IP=${p}. Karsilastirma yapilamadi.` };49 if (d === p) {50 return { ok: false, verdict: 'PROXY DEVREDE DEGIL', detail: `Dogrudan ve proxy IP ayni: ${d}. Istekler proxy'den GECMIYOR.` };51 }52 return {53 ok: true,54 verdict: 'PROXY DEVREDE',55 detail: `dogrudan=${d} (${direct.countryCode ?? '?'}) != proxy=${p} (${proxied.countryCode ?? '?'})`,56 };57}58
59/**60 * Ucretlendirme hukmu. ChargeResult'lari toplayip ne olduguna karar verir.61 * @param {{isAtHome: boolean, pushed: number, chargedCount: number, limitReached: boolean}} a62 */63export function chargeVerdict({ isAtHome, pushed, chargedCount, limitReached }) {64 if (pushed === 0) return { ok: false, verdict: 'OLCULEMEDI', detail: 'Hic kayit yazilmadi, ucretlendirme denenmedi.' };65 if (limitReached) {66 return { ok: true, verdict: 'BUTCE SINIRI', detail: `Kullanicinin harcama tavani doldu; ${chargedCount}/${pushed} kayit ucretlendirildi.` };67 }68 if (chargedCount === 0) {69 return {70 ok: false,71 verdict: 'UCRETLENDIRME TETIKLENMEDI',72 detail: isAtHome73 ? 'Kayitlar yazildi ama chargedCount=0. Actor PPE degil ya da "result" olayi Console`da tanimli degil. Log`da "unknown event" ya da "does not use the pay-per-event" uyarisi aranmali.'74 : 'Yerel kosu: ucretlendirme yalniz platformda etkin (ACTOR_TEST_PAY_PER_EVENT=true ile yerelde denenebilir).',75 };76 }77 if (chargedCount < pushed) {78 return { ok: false, verdict: 'KISMI', detail: `${chargedCount}/${pushed} kayit ucretlendirildi; fark arastirilmali.` };79 }80 return { ok: true, verdict: 'UCRETLENDIRME CALISIYOR', detail: `${chargedCount}/${pushed} kayit ucretlendirildi.` };81}82
83/**84 * Sayac muhasebesi. Etkinlik sayfasina giren her istek tam olarak bir cikis85 * yoluna yazilmali. Tutmuyorsa sayilmayan bir dal vardir = sessiz kayip.86 *87 * Ilk platform kosusunda tam bu oldu: sema hatasi hicbir sayaca yazilmadi,88 * parseFail=0 ve pushed=0 ile kosu SUCCEEDED bitti.89 */90export const EXIT_COUNTERS = [91 'pushed', 'skippedPast', 'dateFiltered', 'parseFail', 'schemaFail', 'pushFail', 'duplicate', 'overMaxItems',92];93
94export function reconcileCounters(counters = {}) {95 const seen = counters.eventPages ?? 0;96 const total = EXIT_COUNTERS.reduce((a, k) => a + (counters[k] ?? 0), 0);97 if (total === seen) return { ok: true, seen, total, detail: `${seen} etkinlik sayfasi, ${total} cikis` };98 return {99 ok: false,100 seen,101 total,102 detail: `${seen} etkinlik sayfasi islendi ama cikis yollari ${total}; fark ${seen - total}`,103 };104}1/**2 * robots.txt ayristirma ve izin kontrolu. Ag erisimi YOK - saf fonksiyonlar + enjekte edilen fetch.3 * RFC 9309: en uzun eslesen kural kazanir; esitlikte Allow kazanir.4 *5 * Saldiri raporu duzeltmesi:6 * O4 - regex kural basina BIR KEZ derlenir (parseRobots icinde), her URL kontrolunde degil.7 */8
9/** robots kural yolunu regex'e cevirir. `*` = herhangi, sondaki `$` = bitis capasi. */10export function pathToRegex(rulePath) {11 let p = rulePath;12 let anchored = false;13 if (p.endsWith('$')) { anchored = true; p = p.slice(0, -1); }14 let re = '';15 for (const ch of p) {16 if (ch === '*') re += '.*';17 else re += ch.replace(/[.+?^${}()|[\]\\]/g, '\\$&');18 }19 return new RegExp('^' + re + (anchored ? '$' : ''));20}21
22/** Bir robots.txt metnini gruplara ayirir; kural regex'leri burada derlenir (O4). */23export function parseRobots(text) {24 const groups = [];25 let current = null;26 let lastLineWasAgent = false;27
28 for (const rawLine of String(text ?? '').split(/\r?\n/)) {29 const line = rawLine.split('#')[0].trim();30 if (!line) continue;31 const idx = line.indexOf(':');32 if (idx === -1) continue;33 const field = line.slice(0, idx).trim().toLowerCase();34 const value = line.slice(idx + 1).trim();35
36 if (field === 'user-agent') {37 if (!lastLineWasAgent || !current) {38 current = { agents: [], rules: [], crawlDelay: null };39 groups.push(current);40 }41 current.agents.push(value.toLowerCase());42 lastLineWasAgent = true;43 continue;44 }45
46 lastLineWasAgent = false;47 if (!current) continue;48
49 if (field === 'allow' || field === 'disallow') {50 if (value === '') continue; // bos Disallow = serbest51 current.rules.push({52 allow: field === 'allow',53 path: value,54 regex: pathToRegex(value), // O4: bir kez derle55 len: value.replace(/\$$/, '').length,56 });57 } else if (field === 'crawl-delay') {58 const n = Number(value);59 if (Number.isFinite(n)) current.crawlDelay = n;60 }61 }62 return groups;63}64
65/** Verilen user-agent icin en ozgul grubu secer. Tam eslesme > yildiz. */66export function selectGroup(groups, userAgent) {67 const ua = String(userAgent || '*').toLowerCase();68 let best = null;69 let bestLen = -1;70 for (const g of groups) {71 for (const a of g.agents) {72 if (a === '*') {73 if (bestLen < 0) { best = g; bestLen = 0; }74 } else if (ua.includes(a) && a.length > bestLen) {75 best = g;76 bestLen = a.length;77 }78 }79 }80 return best;81}82
83/** Bir yol (path + query) icin karar. Kural yoksa serbest. */84export function isPathAllowed(group, pathWithQuery) {85 if (!group || group.rules.length === 0) return true;86 let best = null;87 for (const rule of group.rules) {88 // elle kurulmus kural nesneleri icin geriye donuk uyumluluk89 const regex = rule.regex ?? pathToRegex(rule.path);90 const len = rule.len ?? rule.path.replace(/\$$/, '').length;91 if (!regex.test(pathWithQuery)) continue;92 if (!best || len > best.len || (len === best.len && rule.allow && !best.allow)) {93 best = { allow: rule.allow, len };94 }95 }96 return best ? best.allow : true;97}98
99export class RobotsGate {100 constructor({ userAgent = '*', fetchFn = fetch, failClosed = true, log = null } = {}) {101 this.userAgent = userAgent;102 this.fetchFn = fetchFn;103 this.failClosed = failClosed;104 this.log = log;105 this.cache = new Map();106 }107
108 async load(origin) {109 if (this.cache.has(origin)) return this.cache.get(origin);110 let entry = { group: null, crawlDelay: null, ok: false };111 try {112 const res = await this.fetchFn(new URL('/robots.txt', origin).href);113 if (res.status >= 500) {114 entry = { group: null, crawlDelay: null, ok: false };115 } else if (res.status === 404 || res.status === 410) {116 entry = { group: null, crawlDelay: null, ok: true };117 } else if (res.ok) {118 const text = await res.text();119 const group = selectGroup(parseRobots(text), this.userAgent);120 entry = { group, crawlDelay: group?.crawlDelay ?? null, ok: true };121 }122 } catch (err) {123 this.log?.warning(`robots.txt cekilemedi (${origin}): ${err.message}`);124 }125 this.cache.set(origin, entry);126 return entry;127 }128
129 async isAllowed(urlString) {130 const url = new URL(urlString);131 const entry = await this.load(url.origin);132 if (!entry.ok) return !this.failClosed;133 return isPathAllowed(entry.group, url.pathname + url.search);134 }135
136 crawlDelayFor(origin) {137 return this.cache.get(origin)?.crawlDelay ?? null;138 }139}1/**2 * Beyaz liste sema filtresi. Semada olmayan HER alan duser ve sayilir.3 * Yasakli (kisisel veri) alanlar ayrica sayilir: log'da 0 gormek zorundayiz (09 §8).4 *5 * Saldiri raporu duzeltmeleri:6 * O1 - object/array alanlarin ICI derin taranir VE yasakli alan kayittan DUSURULUR.7 * D2 - string alanlara uzunluk siniri (limits).8 * D3 - bos string null olur.9 * D4 - `in` yerine Object.hasOwn: prototip anahtarlari semada sanilmaz.10 * Y2 - derin tarama yigin tasirmaz: maxDepth siniri, asilan dal kayittan duser (fail-closed).11 * Y3 - dongusel nesne sonsuz ozyinelemeye girmez (ayni derinlik siniri kapatir).12 * Y4 - temizleme girdiyi mutasyona ugratmaz; "__proto__" anahtari prototip degistirmez.13 */14
15export const DEFAULT_STRING_LIMIT = 10000;16export const MAX_SCAN_DEPTH = 20;17
18const TYPES = {19 // D3: bos string null20 string: (v) => {21 if (v == null) return null;22 const s = typeof v === 'string' ? v : String(v);23 return s === '' ? null : s;24 },25 number: (v) => {26 if (v == null || v === '') return null;27 const n = Number(v);28 return Number.isFinite(n) ? n : null;29 },30 boolean: (v) => (typeof v === 'boolean' ? v : v == null ? null : Boolean(v)),31 iso: (v) => {32 if (!v) return null;33 const d = new Date(v);34 return Number.isNaN(d.getTime()) ? null : d.toISOString();35 },36 object: (v) => (v && typeof v === 'object' && !Array.isArray(v) ? v : null),37 array: (v) => (Array.isArray(v) ? v : null),38};39
40/**41 * @param {object} cfg42 * @param {Record<string,string>} cfg.fields43 * @param {string[]} [cfg.forbidden]44 * @param {Record<string,number>} [cfg.limits] - alan basina string uzunluk siniri45 * @param {number} [cfg.defaultStringLimit]46 * @param {number} [cfg.maxDepth] - ic ice nesnelerde tarama derinligi siniri47 */48export function defineSchema({49 name,50 fields,51 forbidden = [],52 limits = {},53 defaultStringLimit = DEFAULT_STRING_LIMIT,54 maxDepth = MAX_SCAN_DEPTH,55}) {56 for (const [k, t] of Object.entries(fields)) {57 if (!Object.hasOwn(TYPES, t)) throw new Error(`Bilinmeyen tip "${t}" (alan: ${k})`);58 }59 for (const k of Object.keys(limits)) {60 if (!Object.hasOwn(fields, k)) throw new Error(`limits bilinmeyen alana isaret ediyor: ${k}`);61 }62 if (!(maxDepth >= 1)) throw new Error(`maxDepth en az 1 olmali: ${maxDepth}`);63 return { name, fields, forbidden: forbidden.map((f) => f.toLowerCase()), limits, defaultStringLimit, maxDepth };64}65
66/**67 * @returns {{record: object, dropped: string[], forbiddenHits: string[],68 * truncated: string[], stripped: string[], depthTrimmed: string[]}}69 * - forbiddenHits: bulunan yasakli alanlarin tam yolu (ust duzey ad + derin yol)70 * - stripped: kayittan FIILEN silinen derin yollar71 * - depthTrimmed: derinlik siniri asildigi icin silinen dallar72 */73export function applyWhitelist(raw, schema) {74 const record = {};75 const dropped = [];76 const forbiddenHits = [];77 const truncated = [];78 const stripped = [];79 const depthTrimmed = [];80 const limits = schema.limits ?? {};81 const defaultLimit = schema.defaultStringLimit ?? DEFAULT_STRING_LIMIT;82 const maxDepth = schema.maxDepth ?? MAX_SCAN_DEPTH;83
84 for (const key of Object.keys(raw ?? {})) {85 if (Object.hasOwn(schema.fields, key)) continue; // D486 dropped.push(key);87 if (schema.forbidden.includes(key.toLowerCase())) forbiddenHits.push(key);88 }89
90 for (const [key, type] of Object.entries(schema.fields)) {91 let value = TYPES[type](raw?.[key]);92
93 if (typeof value === 'string') {94 const max = limits[key] ?? defaultLimit; // D295 if (max > 0 && value.length > max) {96 value = value.slice(0, max);97 truncated.push(key);98 }99 if (value === '') value = null; // D3100 }101
102 // O1: object/array alanin ici temizlenir. Tek gecis: hem bulur hem siler.103 if (value && typeof value === 'object') {104 const res = stripForbiddenDeep(value, schema.forbidden, { path: key, maxDepth });105 value = res.value;106 forbiddenHits.push(...res.removed);107 stripped.push(...res.removed);108 depthTrimmed.push(...res.trimmed);109 }110
111 record[key] = value;112 }113
114 return { record, dropped, forbiddenHits, truncated, stripped, depthTrimmed };115}116
117/**118 * O1: yasakli alanlari ic ice yapilardan ARINDIRIR.119 * Girdi mutasyona ugramaz; temizlenmis bir KOPYA doner (Y4).120 * @returns {{value: any, removed: string[], trimmed: string[]}}121 */122export function stripForbiddenDeep(value, forbidden, { path = '', maxDepth = MAX_SCAN_DEPTH } = {}) {123 const removed = [];124 const trimmed = [];125 const out = strip(value, forbidden, path, 0, maxDepth, removed, trimmed);126 return { value: out, removed, trimmed };127}128
129function strip(node, forbidden, path, depth, maxDepth, removed, trimmed) {130 if (!node || typeof node !== 'object') return node;131
132 // Y2/Y3: sinir asilirsa dal birakilmaz, DUSURULUR. Taranmamis veri kayitta kalmaz.133 if (depth >= maxDepth) {134 trimmed.push(path || '(kok)');135 return null;136 }137
138 if (Array.isArray(node)) {139 return node.map((v, i) => strip(v, forbidden, `${path}[${i}]`, depth + 1, maxDepth, removed, trimmed));140 }141
142 const entries = [];143 for (const [k, v] of Object.entries(node)) {144 const p = path ? `${path}.${k}` : k;145 if (forbidden.includes(k.toLowerCase())) {146 removed.push(p);147 continue;148 }149 entries.push([k, strip(v, forbidden, p, depth + 1, maxDepth, removed, trimmed)]);150 }151 // Y4: duz atama "__proto__" anahtarinda prototipi degistirir; fromEntries own alan tanimlar.152 return Object.fromEntries(entries);153}154
155/** Yasakli alan adlarini ic ice nesnelerde de arar (dogrulama yardimcisi). */156export function scanForbiddenDeep(value, forbidden, path = '', out = [], depth = 0, maxDepth = MAX_SCAN_DEPTH) {157 if (!value || typeof value !== 'object' || depth >= maxDepth) return out;158 if (Array.isArray(value)) {159 value.forEach((v, i) => scanForbiddenDeep(v, forbidden, `${path}[${i}]`, out, depth + 1, maxDepth));160 return out;161 }162 for (const [k, v] of Object.entries(value)) {163 const p = path ? `${path}.${k}` : k;164 if (forbidden.includes(k.toLowerCase())) out.push(p);165 scanForbiddenDeep(v, forbidden, p, out, depth + 1, maxDepth);166 }167 return out;168}169
170// ---------------------------------------------------------------------------171// Dataset semasi turetme ve kayit dogrulama.172//173// P1 - Platform, datasete yazilan her kaydi .actor/dataset_schema.json'a gore174// dogruluyor; YEREL depolama dogrulamiyor. Iki ayri dogruluk kaynagi175// (kod semasi + elle yazilmis JSON sema) kaciniImaz olarak ayrisiyor.176// Cozum: JSON semayi kod semasindan turet, testle kilitle.177// P2 - applyWhitelist her alani HER ZAMAN uretir; eksik olan null olur.178// JSON Schema'da {"type":"string"} null'i REDDEDER. Bu yuzden turetilen179// her alan ["<tip>","null"] olmak zorunda.180// ---------------------------------------------------------------------------181
182/** Ic tip -> JSON Schema tipi. */183export const JSON_TYPES = {184 string: 'string',185 number: 'number',186 boolean: 'boolean',187 iso: 'string',188 object: 'object',189 array: 'array',190};191
192export function jsonTypeOf(value) {193 if (value === null) return 'null';194 if (Array.isArray(value)) return 'array';195 return typeof value;196}197
198/** .actor/dataset_schema.json icindeki fields.properties blogunu uretir. */199export function datasetProperties(schema) {200 const out = {};201 for (const [k, t] of Object.entries(schema.fields)) {202 out[k] = { type: [JSON_TYPES[t], 'null'] };203 }204 return out;205}206
207/**208 * Kaydi kod semasina gore dogrular. Platformdaki "Schema validation failed"209 * hatasini, ALAN ADI vererek, istek yapilmadan once yakalamak icin.210 * @returns {string[]} bos dizi = gecerli211 */212export function validateRecord(record, schema) {213 const out = [];214 if (!record || typeof record !== 'object' || Array.isArray(record)) {215 return [`kayit nesne degil: ${jsonTypeOf(record)}`];216 }217 for (const [k, t] of Object.entries(schema.fields)) {218 if (!Object.hasOwn(record, k)) { out.push(`${k}: alan eksik`); continue; }219 const v = record[k];220 if (v === null) continue; // null her alanda serbest221 const want = JSON_TYPES[t];222 const got = jsonTypeOf(v);223 if (got !== want) out.push(`${k}: beklenen ${want}, gelen ${got}`);224 }225 for (const k of Object.keys(record)) {226 if (!Object.hasOwn(schema.fields, k)) out.push(`${k}: semada tanimli degil`);227 }228 return out;229}1/**2 * Hedefe ozgu URL politikasi: izinli desenler + ASLA listesi (09 §5).3 *4 * Saldiri raporu duzeltmesi:5 * O6 - encode edilmis yol karakterleri (%2f, %2e, %5c) reddedilir; kontrol6 * hem ham hem cozulmus yol uzerinde yapilir. `..` segmenti reddedilir.7 */8
9const ENCODED_TRAVERSAL_RE = /%2f|%2e|%5c/i;10
11export class UrlPolicy {12 constructor({ host, allow = [], deny = [] }) {13 this.host = host.toLowerCase();14 this.allow = allow;15 this.deny = deny;16 }17
18 /** @returns {{ok: boolean, reason: string}} */19 check(urlString) {20 let url;21 try {22 url = new URL(urlString);23 } catch {24 return { ok: false, reason: 'gecersiz URL' };25 }26 if (url.protocol !== 'https:') return { ok: false, reason: 'https degil' };27 if (url.hostname.toLowerCase() !== this.host) {28 return { ok: false, reason: `host disi: ${url.hostname}` };29 }30
31 const rawPath = url.pathname;32
33 // O6: encode edilmis slash/nokta/ters slash -> reddet34 if (ENCODED_TRAVERSAL_RE.test(rawPath) || ENCODED_TRAVERSAL_RE.test(url.search)) {35 return { ok: false, reason: 'encode edilmis yol karakteri (%2f/%2e/%5c)' };36 }37
38 let decodedPath;39 try {40 decodedPath = decodeURIComponent(rawPath);41 } catch {42 return { ok: false, reason: 'bozuk yuzde kodlamasi' };43 }44
45 if (decodedPath.split('/').includes('..')) {46 return { ok: false, reason: 'yol traversal (..)' };47 }48 if (decodedPath.includes('//')) {49 return { ok: false, reason: 'cift slash' };50 }51
52 const candidates = [rawPath + url.search, decodedPath + url.search];53
54 // ASLA listesi: her iki bicimde de bakilir55 for (const re of this.deny) {56 for (const p of candidates) {57 if (re.test(p)) return { ok: false, reason: `ASLA listesi: ${re}` };58 }59 }60
61 if (this.allow.length === 0) return { ok: true, reason: 'allow listesi bos' };62
63 // Izinli olmak icin cozulmus yol eslesmeli (ham bicim tek basina yetmez)64 for (const re of this.allow) {65 if (re.test(decodedPath + url.search)) return { ok: true, reason: `izinli: ${re}` };66 }67 return { ok: false, reason: 'izinli desene uymuyor' };68 }69
70 isAllowed(urlString) {71 return this.check(urlString).ok;72 }73}1/**2 * .actor/ altindaki iki semanin TEK kaynagi. Dosyalar elle yazilmaz;3 * tools/gen-dataset-schema.mjs buradan uretir, testler dosyalari buraya kilitler.4 *5 * - dataset_schema.json : kayitta HANGI alanlar var (fields, EVENT_SCHEMA'dan turetilir) + Console gorunumu6 * - output_schema.json : cikti NEREDE (dataset, KV kaydi). Store'a yayin icin zorunlu:7 * "To publish your Actor to Apify Store, an output schema is required."8 * (docs.apify.com/platform/actors/development/actor-definition/output-schema)9 */10import { datasetProperties } from '../../core/schema.js';11import { EVENT_SCHEMA } from './schema.js';12
13export const DATASET_SCHEMA_FILE = './dataset_schema.json';14export const OUTPUT_SCHEMA_FILE = './output_schema.json';15export const RUN_STATS_KEY = 'RUN_STATS';16
17export function buildDatasetSchema() {18 return {19 actorSpecification: 1,20 fields: {21 type: 'object',22 properties: datasetProperties(EVENT_SCHEMA),23 },24 views: {25 overview: {26 title: 'Etkinlikler',27 transformation: {28 fields: ['title', 'startDate', 'city', 'isOnline', 'price', 'organizerName', 'url', 'sourceListUrl'],29 },30 display: {31 component: 'table',32 properties: {33 title: { label: 'Etkinlik' },34 startDate: { label: 'Baslangic', format: 'date' },35 city: { label: 'Sehir' },36 isOnline: { label: 'Cevrim ici', format: 'boolean' },37 price: { label: 'Fiyat', format: 'object' },38 organizerName: { label: 'Organizator' },39 url: { label: 'URL', format: 'link' },40 sourceListUrl: { label: 'Bulundugu liste', format: 'link' },41 },42 },43 },44 },45 };46}47
48/**49 * Actor output schema (spec v1). Zorunlu: actorOutputSchemaVersion, title, properties;50 * her cikti icin title + template. `type: "string"` resmi ornekteki gibi.51 */52export function buildOutputSchema() {53 const fieldCount = Object.keys(EVENT_SCHEMA.fields).length;54 return {55 actorOutputSchemaVersion: 1,56 title: 'Eventbrite Event Scraper output',57 description: 'Event rows are stored in the default dataset; per-run counters and verdicts in the default key-value store.',58 properties: {59 results: {60 type: 'string',61 title: 'Events',62 description: `One row per event, ${fieldCount} fields (see dataset schema). Missing values are null, never omitted.`,63 template: '{{links.apiDefaultDatasetUrl}}/items',64 },65 runStats: {66 type: 'string',67 title: 'Run stats',68 description: 'Counters, request budget (plan vs actual), proxy and charging verdicts, discovery summary.',69 template: `{{links.apiDefaultKeyValueStoreUrl}}/records/${RUN_STATS_KEY}`,70 },71 },72 };73}1/**2 * Kesif katmani: sitemap -> /b/ gezinme sayfalari -> /e/ etkinlik baglari.3 *4 * Neden /b/ (olcum 14 Eyl 2026, datacenter proxy, uc tur, ayri olcum actor'u):5 * /b/ 55/55 = %100, denemeler arasi birebir ayni, sayfa basina ort. 5,5 bag, tavan 86 * /d/ 15/58 = %26, ayni URL denemeler arasi karar degistiriyor -> kesifte KULLANILMAZ7 * /o/ 200 doner ama statik HTML'de 0 /e/ bagi (istemci tarafinda render) -> kullanilmaz8 * sitemap_xml/*.gz %100, parca basina 50.000 URL9 *10 * Sehir/kategori filtresi URL'nin icinde: /b/<sehir>/<kategori>/[<altkategori>/]11 * Sehir slug bicimleri: ca--san-francisco | ny--new-york--brooklyn | slovenia | online12 *13 * Bu dosya ag'a cikmaz; hepsi saf fonksiyon, testlenebilir.14 */15import { gunzipSync } from 'node:zlib';16
17export const SHARD_RE = /\/sitemap_xml\/category_browse_pages\d+\.xml\.gz$/;18export const LINKS_PER_LIST_PAGE = 5.5; // olculdu: 20 buyuk-sehir sayfasi, ortalama19export const MAX_LINKS_PER_LIST_PAGE = 8; // olculdu: hicbir sayfa 8'i asmadi20export const LIST_MARGIN = 1.3; // 0 baglantili sayfalar icin pay21export const FIXED_DISCOVERY_REQUESTS = 6; // 1 indeks + 5 parca (olculdu: 5 parca var)22
23const TR_MAP = { ı: 'i', İ: 'i', ş: 's', Ş: 's', ğ: 'g', Ğ: 'g', ç: 'c', Ç: 'c', ö: 'o', Ö: 'o', ü: 'u', Ü: 'u' };24
25/** "San Francisco" -> "san-francisco"; "ca--san-francisco" korunur; "İstanbul" -> "istanbul". */26export function slugify(input) {27 if (input == null) return '';28 let s = String(input).trim();29 s = s.replace(/[ıİşŞğĞçÇöÖüÜ]/g, (ch) => TR_MAP[ch]);30 s = s.normalize('NFD').replace(/[̀-ͯ]/g, ''); // aksanlari at31 s = s.toLowerCase();32 s = s.replace(/[^a-z0-9-]+/g, ' '); // noktalama ve alt cizgi bosluk olur; "&" sahte "--" uretmez33 s = s.trim();34 s = s.replace(/\s*-\s*/g, '-'); // tire cevresindeki bosluklari yutar: "san - francisco"35 s = s.replace(/\s+/g, '-');36 s = s.replace(/-{3,}/g, '--'); // "--" ayrac olarak korunur, daha uzunu kirpilir37 s = s.replace(/^-+|-+$/g, '');38 return s;39}40
41/** Sitemap XML'inden <loc> degerleri. XML ayristirici gerekmez; sitemap duz yapidadir. */42export function extractLocs(xml) {43 if (typeof xml !== 'string' || xml === '') return [];44 const out = [];45 const re = /<loc>\s*([^<\s]+)\s*<\/loc>/g;46 let m;47 while ((m = re.exec(xml)) !== null) out.push(m[1]);48 return out;49}50
51/** Indeksten yalniz category_browse_pages parcalari. */52export function shardUrlsFromIndex(xml) {53 return extractLocs(xml).filter((u) => SHARD_RE.test(u));54}55
56/** gz ise acar; degilse (sunucu ya da istemci zaten actiysa) metin olarak doner. */57export function gunzipMaybe(body) {58 if (body == null) return '';59 const buf = Buffer.isBuffer(body) ? body : Buffer.from(String(body), 'utf8');60 // gzip sihirli sayisi 1f 8b61 if (buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b) {62 try {63 return gunzipSync(buf).toString('utf8');64 } catch {65 return '';66 }67 }68 return buf.toString('utf8');69}70
71/**72 * /b/<sehir>/<kategori>/ ya da /b/<sehir>/<kategori>/<altkategori>/73 * @returns {{city:string, category:string, subcategory:(string|null)}|null}74 */75export function parseBrowseUrl(url) {76 let path;77 try {78 path = new URL(String(url), 'https://www.eventbrite.com').pathname;79 } catch {80 return null;81 }82 const m = path.match(/^\/b\/([^/]+)\/([^/]+)\/(?:([^/]+)\/)?$/);83 if (!m) return null;84 return { city: m[1], category: m[2], subcategory: m[3] ?? null };85}86
87/** Sehir slug'i, kullanici girdisiyle eslesebilir mi? (parcalari toplarken ucuz on-eleme) */88export function couldMatchCity(citySlug, userSlug) {89 if (!citySlug || !userSlug) return false;90 if (userSlug.includes('--')) return citySlug === userSlug || citySlug.startsWith(`${userSlug}--`);91 return citySlug.split('--').includes(userSlug);92}93
94/**95 * Kullanici girdisini sitemap'teki sehir slug'larina cozer.96 * - "ca--san-francisco" (ayrac iceriyor): tam eslesme + mahalleleri (ca--san-francisco--*)97 * - "san francisco": herhangi bir parcasi esit olan slug'lar; taban = o parcaya kadar olan kisim98 * ny--new-york, ny--new-york--brooklyn -> tek taban (ny--new-york) -> ok99 * il--springfield, mo--springfield -> iki taban -> belirsiz, adaylar listelenir100 * @param {string} input101 * @param {Iterable<string>} citySlugs102 * @returns {{ok:true, base:string, slugs:string[]} | {ok:false, reason:string, candidates:string[]}}103 */104export function resolveCity(input, citySlugs) {105 const S = slugify(input);106 if (!S) return { ok: false, reason: 'sehir bos', candidates: [] };107 const all = [...new Set(citySlugs)];108
109 if (S.includes('--')) {110 const slugs = all.filter((c) => c === S || c.startsWith(`${S}--`));111 if (slugs.length === 0) {112 return { ok: false, reason: `"${S}" sitemap'te yok`, candidates: nearest(S, all) };113 }114 return { ok: true, base: S, slugs: slugs.sort() };115 }116
117 const bases = new Map();118 for (const c of all) {119 const parts = c.split('--');120 const i = parts.indexOf(S);121 if (i < 0) continue;122 const base = parts.slice(0, i + 1).join('--');123 if (!bases.has(base)) bases.set(base, []);124 bases.get(base).push(c);125 }126 if (bases.size === 0) {127 return { ok: false, reason: `"${S}" sitemap'te yok`, candidates: nearest(S, all) };128 }129 if (bases.size > 1) {130 return {131 ok: false,132 reason: `"${S}" birden fazla yere uyuyor; tam slug verin`,133 candidates: [...bases.keys()].sort(),134 };135 }136 const [[base, slugs]] = bases.entries();137 return { ok: true, base, slugs: slugs.sort() };138}139
140function nearest(S, all, limit = 8) {141 const hits = all.filter((c) => c.split('--').some((p) => p.includes(S) || (p.length >= 4 && S.includes(p))));142 hits.sort((a, b) => a.length - b.length || a.localeCompare(b));143 return hits.slice(0, limit);144}145
146/**147 * Eslesen /b/ URL'leri; sitemap sirasi korunur, yinelenenler atilir.148 * Kategori verilmisse hem /b/sehir/kat/ hem /b/sehir/kat/alt/ dahil.149 */150export function selectBrowseUrls(urls, { citySlugs, category = null, subcategory = null }) {151 const cities = citySlugs instanceof Set ? citySlugs : new Set(citySlugs);152 const cat = category ? slugify(category) : null;153 const sub = subcategory ? slugify(subcategory) : null;154 const seen = new Set();155 const out = [];156 for (const u of urls) {157 const p = parseBrowseUrl(u);158 if (!p || !cities.has(p.city)) continue;159 if (cat && p.category !== cat) continue;160 if (sub && p.subcategory !== sub) continue;161 const key = `${p.city}/${p.category}/${p.subcategory ?? ''}`;162 if (seen.has(key)) continue;163 seen.add(key);164 out.push(u);165 }166 return out;167}168
169/**170 * Deterministik liste sirasi: ana sehir sayfalari once, mahalleler (base--*) sonra;171 * kendi icinde URL'ye gore (kod noktasi karsilastirmasi, yerel ayardan bagimsiz).172 * Parcalarin inis sirasi kosudan kosuya degisiyordu; cikti artik tekrar uretilebilir.173 */174export function orderBrowseUrls(urls, base) {175 const rank = (u) => (parseBrowseUrl(u)?.city === base ? 0 : 1);176 return [...urls].sort((a, b) => rank(a) - rank(b) || (a < b ? -1 : a > b ? 1 : 0));177}178
179/**180 * Secili sehir icin sitemap'te var olan kategori slug'lari (category verilirse o181 * kategorinin alt kategorileri). Sirali, benzersiz. Sifir eslesme hatasinda182 * kullaniciya "ne yazacagini" soylemek icin.183 */184export function availableCategories(urls, { citySlugs, category = null }) {185 const cities = citySlugs instanceof Set ? citySlugs : new Set(citySlugs);186 const cat = category ? slugify(category) : null;187 const out = new Set();188 for (const u of urls) {189 const p = parseBrowseUrl(u);190 if (!p || !cities.has(p.city)) continue;191 if (cat) {192 if (p.category === cat && p.subcategory) out.add(p.subcategory);193 } else {194 out.add(p.category);195 }196 }197 return [...out].sort();198}199
200/**201 * Kesif baslayabildi mi? Indeks ya da parcalar alinamadiysa "sehir/kategori icin sayfa yok"202 * uyarisi YANILTICIDIR; kosu dusmeli. (Yerel kosu: indeks 2x session error ile dustu,203 * kosu "musik icin /b/ sayfasi yok" uyarisiyla sessizce bitti.)204 */205export function discoveryVerdict({206 sitemapIndex = 0, indexFailed = 0, shardsTotal = 0, shardsDone = 0, shardsFailed = 0,207}) {208 if (sitemapIndex === 0) {209 return { ok: false, reason: `sitemap indeksi alinamadi (${indexFailed} basarisiz istek)` };210 }211 if (shardsTotal === 0) return { ok: false, reason: 'sitemap indeksinde category_browse_pages parcasi yok' };212 if (shardsDone - shardsFailed <= 0) return { ok: false, reason: `${shardsTotal} parcanin hicbiri alinamadi` };213 if (shardsFailed > 0) {214 return { ok: true, partial: true, reason: `${shardsFailed}/${shardsTotal} parca alinamadi; kesif eksik parcalarla yapildi` };215 }216 return { ok: true, partial: false, reason: 'tamam' };217}218
219/** Kac /b/ sayfasi gerekir? (kalan kayit / sayfa basina bag) x pay */220export function listPagesNeeded(remainingItems) {221 const n = Math.max(0, Number(remainingItems) || 0);222 return n === 0 ? 0 : Math.ceil((n / LINKS_PER_LIST_PAGE) * LIST_MARGIN);223}224
225/**226 * Kac etkinlik bagi stoklanmali? Filtreler bag kaybettirir:227 * tarih araligi 1,5x; yalniz futureOnly 1,2x (olcum: 20 /b/ bagindan 0 gecmis); filtresiz 1,1x.228 */229export function linkStockTarget(maxItems, { dateFilter = false, futureOnly = true } = {}) {230 const items = Math.max(1, Number(maxItems) || 1);231 const k = dateFilter ? 1.5 : (futureOnly ? 1.2 : 1.1);232 return Math.ceil(items * k);233}234
235/**236 * Bag stogu yetmeyecekse kac liste sayfasi daha kuyruga alinmali?237 * Kuyrukta bekleyen sayfalarin beklenen verimi DUSULUR (kor +2 yerine);238 * verim, gozlenen ortalama varsa odur, yoksa olculen 5,5.239 * Platform kosusu 3YWQ8aIIhiqfEKsoL: kor takviye 5 yerine 13 liste sayfasi cekti, 6'si stok doluyken.240 */241export function listsToTopUp({ stock, needed, pending = 0, observedYield = 0, available = Infinity }) {242 const perPage = Math.max(1, Number(observedYield) || LINKS_PER_LIST_PAGE);243 const projected = (Number(stock) || 0) + (Number(pending) || 0) * perPage;244 if (projected >= needed) return 0;245 const deficit = needed - projected;246 return Math.max(0, Math.min(available, Math.max(1, Math.ceil(deficit / perPage))));247}248
249/**250 * Kosu basinda loglanan plan. total = sabit + liste + etkinlik.251 * linkTarget verilirse liste sayfalari o stok hedefi icin hesaplanir (plan ve takviye ayni olcuyu kullanir).252 */253export function estimateRequests(maxItems, { fixed = FIXED_DISCOVERY_REQUESTS, linkTarget = null } = {}) {254 const items = Math.max(1, Number(maxItems) || 1);255 const lists = listPagesNeeded(linkTarget ?? items);256 return { fixed, lists, events: items, total: fixed + lists + items };257}258
259/** ISO tarih (YYYY-MM-DD ya da tam ISO) -> Date | null */260export function parseDateInput(s) {261 if (s == null || s === '') return null;262 const d = new Date(String(s));263 return Number.isNaN(d.getTime()) ? null : d;264}265
266/** Kayit tarih araligina giriyor mu? Sinirlar dahil. startDate yoksa gecmez. */267export function inDateRange(record, from, to) {268 if (!from && !to) return true;269 const sd = record?.startDate ? new Date(record.startDate) : null;270 if (!sd || Number.isNaN(sd.getTime())) return false;271 if (from && sd < from) return false;272 if (to && sd > to) return false;273 return true;274}275
276/** maxItems karsilanamadiginda somut uyari metni. */277export function shortfallMessage({278 requested, found, listPagesScanned, listPagesAvailable, city, category,279 dateFiltered = 0, pastFiltered = 0, hardCapHit = false, blocked = 0,280}) {281 const parts = [`${requested} istendi, ${found} bulundu, ${listPagesScanned}/${listPagesAvailable} /b/ sayfasi tarandi.`];282 const nedenler = [];283 if (listPagesAvailable === 0) {284 nedenler.push(`sitemap'te ${city}${category ? `/${category}` : ''} icin /b/ sayfasi yok`);285 } else if (listPagesScanned >= listPagesAvailable) {286 nedenler.push(`${city}${category ? `/${category}` : ''} icin sitemap'teki tum /b/ sayfalari tarandi; her sayfa en cok ${MAX_LINKS_PER_LIST_PAGE} etkinlik listeliyor`);287 }288 if (dateFiltered > 0) nedenler.push(`${dateFiltered} kayit tarih araligi disinda kaldi`);289 if (pastFiltered > 0) nedenler.push(`${pastFiltered} kayit gecmis etkinlik olarak atlandi`);290 if (blocked > 0) nedenler.push(`${blocked} istek engellendi`);291 if (hardCapHit) nedenler.push('istek tavani doldu');292 if (nedenler.length) parts.push(`Neden: ${nedenler.join('; ')}.`);293 return parts.join(' ');294}1import { extractJsonLd, findByType, findByKnownType, typesOf, typeOf, hasUnparsableJsonLd } from '../../core/jsonld.js';2import { applyWhitelist } from '../../core/schema.js';3import { maskFields } from '../../core/pii.js';4import { EVENT_SCHEMA, KNOWN_EVENT_TYPES } from './schema.js';5import { eventIdFromUrl } from './urlPolicy.js';6
7/** Serbest metin alanlari: deger icerigi PII icin taranir (K2). */8export const FREE_TEXT_FIELDS = ['description', 'title'];9
10export function offsetFromIso(value) {11 if (!value) return null;12 const m = String(value).match(/(Z|[+-]\d{2}:?\d{2})$/);13 if (!m) return null;14 if (m[1] === 'Z') return '+00:00';15 return m[1].length === 5 ? `${m[1].slice(0, 3)}:${m[1].slice(3)}` : m[1];16}17
18/**19 * K1: @type BUTUN olarak degerlendirilir.20 * Dizide "Person" varsa reddedilir; kabul icin "Organization" bulunmali.21 */22export function readOrganizer(node) {23 const org = node?.organizer;24 if (!org || typeof org !== 'object') return { name: null, url: null, rejected: false };25 const types = typesOf(org);26
27 if (types.includes('Person')) {28 return { name: null, url: org.url ?? null, rejected: true, rejectedType: types.join('|') };29 }30 if (!types.includes('Organization')) {31 return { name: null, url: org.url ?? null, rejected: true, rejectedType: types.join('|') || null };32 }33 return { name: org.name ?? null, url: org.url ?? null, rejected: false };34}35
36export function readLocation(node) {37 const loc = node?.location;38 const types = typesOf(loc);39 if (types.includes('VirtualLocation')) {40 return { isOnline: true, venueName: null, city: null, country: null };41 }42 if (types.includes('Place')) {43 const a = loc.address ?? {};44 return {45 isOnline: false,46 venueName: loc.name ?? null,47 city: a.addressLocality ?? null,48 country: a.addressCountry ?? null,49 };50 }51 const mode = String(node?.eventAttendanceMode ?? '');52 return {53 isOnline: mode.includes('OnlineEventAttendanceMode') ? true : null,54 venueName: null,55 city: null,56 country: null,57 };58}59
60export function readPrice(node) {61 const offers = Array.isArray(node?.offers) ? node.offers[0] : node?.offers;62 if (!offers) return null;63 const num = (v) => { const n = Number(v); return Number.isFinite(n) ? n : null; };64 const min = num(offers.lowPrice ?? offers.price);65 const max = num(offers.highPrice ?? offers.price);66 const currency = offers.priceCurrency ?? null;67 if (min === null && max === null && !currency) return null;68 return { min, max, currency };69}70
71export function readCategory(blocks) {72 const bc = findByType(blocks, /^BreadcrumbList$/);73 const items = Array.isArray(bc?.itemListElement) ? bc.itemListElement : [];74 if (!items.length) return null;75 const last = items[items.length - 1];76 return last?.item?.name ?? last?.name ?? null;77}78
79/**80 * Saf ayristirici: ag yok, yan etki yok.81 * @returns {{ok:boolean, reason?:string, record?:object, dropped?:string[],82 * forbiddenHits?:string[], truncated?:string[], stripped?:string[],83 * depthTrimmed?:string[], meta?:object}}84 */85export function parseEventPage(html, url, { now = () => new Date() } = {}) {86 const blocks = extractJsonLd(html);87 if (blocks.length === 0) {88 const diag = hasUnparsableJsonLd(html);89 if (diag.bad > 0) {90 return { ok: false, reason: `JSON-LD bozuk (${diag.bad}/${diag.found} blok ayristirilamadi)` };91 }92 return { ok: false, reason: 'JSON-LD bulunamadi' };93 }94
95 // D1: bilinen Event alt tipleri; "CancelledNonEvent" gibi uydurma tipler eslesmez96 const ev = findByKnownType(blocks, KNOWN_EVENT_TYPES);97 if (!ev) {98 const gorulen = [...new Set(blocks.flatMap(typesOf))].slice(0, 6).join(', ');99 return { ok: false, reason: `Bilinen Event tipi yok (gorulen: ${gorulen || 'yok'})` };100 }101
102 const org = readOrganizer(ev);103 const loc = readLocation(ev);104
105 const rawUnmasked = {106 id: eventIdFromUrl(ev.url ?? url),107 url: ev.url ?? url,108 title: ev.name ?? null,109 description: ev.description ?? null,110 startDate: ev.startDate ?? null,111 endDate: ev.endDate ?? null,112 timezone: offsetFromIso(ev.startDate),113 ...loc,114 price: readPrice(ev),115 category: readCategory(blocks),116 organizerName: org.name,117 organizerUrl: org.url,118 attendeeCount: null, // Eventbrite JSON-LD'de yok; OLCULMELI119 scrapedAt: now().toISOString(),120 };121
122 // K2: once sema sinirina kirp (maskeleme megabaytlik metni taramasin), sonra maskele123 const preTruncated = [];124 for (const f of FREE_TEXT_FIELDS) {125 const cap = EVENT_SCHEMA.limits[f] ?? EVENT_SCHEMA.defaultStringLimit;126 if (typeof rawUnmasked[f] === 'string' && rawUnmasked[f].length > cap) {127 rawUnmasked[f] = rawUnmasked[f].slice(0, cap);128 preTruncated.push(f);129 }130 }131 const { obj: raw, masked: piiMasked, kinds: piiKinds } = maskFields(rawUnmasked, FREE_TEXT_FIELDS);132
133 const {134 record, dropped, forbiddenHits, truncated: whitelistTruncated, stripped, depthTrimmed,135 } = applyWhitelist(raw, EVENT_SCHEMA);136 const truncated = [...new Set([...preTruncated, ...whitelistTruncated])];137
138 return {139 ok: true,140 record,141 dropped,142 forbiddenHits,143 truncated,144 stripped,145 depthTrimmed,146 meta: {147 eventType: typeOf(ev),148 eventTypes: typesOf(ev),149 organizerRejected: org.rejected,150 organizerRejectedType: org.rejectedType ?? null,151 jsonLdBlocks: blocks.length,152 piiMasked,153 piiKinds,154 },155 };156}157
158export function isPastEvent(record, now = new Date()) {159 const end = record?.endDate ?? record?.startDate;160 if (!end) return false;161 const d = new Date(end);162 return Number.isNaN(d.getTime()) ? false : d.getTime() < now.getTime();163}1import { defineSchema } from '../../core/schema.js';2
3/**4 * 09 §2 Eventbrite/Meetup ortak semasi.5 * ALINMAZ listesi semaya girmez; ayrica `forbidden` ile sayilir ki log'da 0 gorelim.6 * D2/K2: string alanlara uzunluk siniri; description 2000 karakter.7 */8export const EVENT_SCHEMA = defineSchema({9 name: 'eventbrite-event',10 fields: {11 id: 'string',12 url: 'string',13 title: 'string',14 description: 'string',15 startDate: 'iso',16 endDate: 'iso',17 timezone: 'string',18 venueName: 'string',19 city: 'string',20 country: 'string',21 isOnline: 'boolean',22 price: 'object',23 category: 'string',24 organizerName: 'string',25 organizerUrl: 'string',26 attendeeCount: 'number',27 sourceListUrl: 'string', // izlenebilirlik: kaydin bulundugu /b/ sayfasi; startUrls ile dogrudan /e/ verildiyse null28 scrapedAt: 'iso',29 },30 limits: {31 description: 2000, // K232 title: 300,33 venueName: 200,34 city: 100,35 country: 100,36 category: 200,37 organizerName: 200,38 url: 500,39 organizerUrl: 500,40 sourceListUrl: 500,41 },42 defaultStringLimit: 1000,43 forbidden: [44 'attendeeName', 'attendeeNames', 'attendees', 'memberName', 'members',45 'hostName', 'personName', 'firstName', 'lastName', 'fullName',46 'email', 'emails', 'phone', 'phoneNumber', 'avatar', 'avatarUrl',47 'profilePicture', 'profileImage', 'twitter', 'facebook', 'instagram', 'linkedin',48 ],49});50
51/** D1: schema.org Event alt tipleri beyaz listesi. */52export const KNOWN_EVENT_TYPES = [53 'Event', 'BusinessEvent', 'SocialEvent', 'MusicEvent', 'EducationEvent',54 'SportsEvent', 'TheaterEvent', 'ScreeningEvent', 'ComedyEvent', 'DanceEvent',55 'ExhibitionEvent', 'Festival', 'FoodEvent', 'LiteraryEvent', 'VisualArtsEvent',56 'ChildrensEvent', 'CourseInstance', 'DeliveryEvent', 'PublicationEvent',57 'SaleEvent', 'EventSeries', 'Hackathon',58];1import { UrlPolicy } from '../../core/urlPolicy.js';2
3/**4 * 09 §5 Eventbrite satiri.5 * KULLANILACAK: /e/*, /o/*, /b/{yer}/{kategori}/[{alt}/], /directory/sitemap/*, /sitemap_xml/*6 * IZINLI AMA KESIFTE KULLANILMAZ: /d/{yer}/{kategori}/ (olcum: %26 basari, kararsiz)7 * ASLA: /api/v3/* (robots kapali), /checkout-external*, giris gerektiren her sey.8 * ASLA listesi robots.txt'deki Disallow'lari da yansitir (client_token, orderid vb.).9 */10export const EVENTBRITE_POLICY = new UrlPolicy({11 host: 'www.eventbrite.com',12 deny: [13 /^\/api\//,14 /^\/checkout-external/,15 /^\/(login|signin|logout|signup|register)\b/,16 /^\/manage\//,17 /^\/myevent/,18 /^\/esi_cache\//,19 /^\/(atom|rss)\//,20 /^\/events\/(rss|atom)\//,21 /^\/upload\//,22 /^\/feedback-service\//,23 /^\/directory\/(?!sitemap\/)/, // /directory/ robots'ta kapali, yalniz sitemap acik24 /[?&](client_token|orderid|auth|access_token)=/i,25 /^\/tickets-external/,26 ],27 allow: [28 /^\/e\/[^/?#]+/,29 /^\/o\/[^/?#]+/,30 /^\/directory\/sitemap\//,31 /^\/d\/[^/]+\/[^/]+\/?/,32 /^\/b\/[^/]+\/[^/]+\/?/, // olcum: 55/55 = %100, kesif omurgasi33 /^\/sitemap_xml\//,34 ],35});36
37/** Etkinlik URL'sinden sayisal kimlik. */38export function eventIdFromUrl(url) {39 const m = String(url).match(/\/e\/[^/?#]*?-?(\d{6,})(?:[/?#]|$)/);40 return m ? m[1] : null;41}42
43/** Organizator URL'sinden kimlik. */44export function organizerIdFromUrl(url) {45 const m = String(url).match(/\/o\/[^/?#]*?-?(\d{6,})(?:[/?#]|$)/);46 return m ? m[1] : null;47}