sahibinden-scraper-puppeteer-js

# Specify the base Docker image.
FROM apify/actor-node-puppeteer-chrome:20 AS intermediate

# Check preinstalled packages
RUN npm ls crawlee apify puppeteer playwright

# Copy just package.json and package-lock.json
COPY --chown=myuser package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
# NOTE: No dev dependencies anymore, but install includes production ones.
RUN npm install --include=dev --audit=false

# Copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# NO 'npm run build' STEP NEEDED HERE ANYMORE

# Create final image
FROM apify/actor-node-puppeteer-chrome:20

# Check preinstalled packages
RUN npm ls crawlee apify puppeteer playwright

# Copy just package.json and package-lock.json
COPY --chown=myuser package*.json ./

# Install ONLY production dependencies
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# NO COPY from builder for /dist NEEDED HERE ANYMORE

# Copy the remaining files and directories with the source code.
COPY --chown=myuser . ./

# Run the image.
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

{
    "actorSpecification": 1,
    "name": "my-actor",
    "title": "Project Puppeteer Crawler Typescript",
    "description": "Crawlee and Puppeteer project in typescript.",
    "version": "0.0",
    "meta": {
        "templateId": "ts-crawlee-puppeteer-chrome"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

{
    "title": "PuppeteerCrawler Template",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "URLs to start with.",
            "editor": "requestListSources",
            "prefill": [
                {
                    "url": "https://apify.com"
                }
            ]
        }
    }
}

1import axios from 'axios';
2import { Actor, log } from 'apify';
3
4/**
5 * BaseRow integration for storing scraped car listings
6 */
7export class BaseRowIntegration {
8    /**
9     * Initialize BaseRow integration
10     * @param {Object} options Configuration options
11     * @param {string} options.apiToken BaseRow API token
12     * @param {string} options.tableId BaseRow table ID
13     * @param {string} options.databaseId BaseRow database ID
14     */
15    constructor(options) {
16        const { apiToken, tableId, databaseId } = options;
17        
18        if (!apiToken) throw new Error('BaseRow API token is required');
19        if (!tableId) throw new Error('BaseRow table ID is required');
20        if (!databaseId) throw new Error('BaseRow database ID is required');
21        
22        this.apiToken = apiToken;
23        this.tableId = tableId;
24        this.databaseId = databaseId;
25        this.baseUrl = 'https://api.baserow.io/api';
26        this.client = axios.create({
27            baseURL: this.baseUrl,
28            headers: {
29                'Authorization': `Token ${this.apiToken}`,
30                'Content-Type': 'application/json'
31            }
32        });
33        
34        log.info('BaseRow integration initialized', { tableId, databaseId });
35    }
36    
37    /**
38     * Store a car listing in BaseRow
39     * @param {Object} carData Car listing data
40     * @returns {Promise<Object>} Created row data
41     */
42    async storeCarListing(carData) {
43        try {
44            // Prepare data for BaseRow
45            const rowData = this._prepareRowData(carData);
46            
47            // Check if listing already exists to avoid duplicates
48            const existingRow = await this._findExistingListing(carData.id);
49            
50            if (existingRow) {
51                log.info(`Updating existing listing: ${carData.id}`);
52                return await this._updateRow(existingRow.id, rowData);
53            } else {
54                log.info(`Creating new listing: ${carData.id}`);
55                return await this._createRow(rowData);
56            }
57        } catch (error) {
58            log.error(`Error storing car listing in BaseRow: ${error.message}`);
59            throw error;
60        }
61    }
62    
63    /**
64     * Store multiple car listings in BaseRow
65     * @param {Array<Object>} carListings Array of car listing data
66     * @returns {Promise<Array<Object>>} Created/updated row data
67     */
68    async storeCarListings(carListings) {
69        log.info(`Storing ${carListings.length} car listings in BaseRow`);
70        
71        const results = [];
72        for (const carData of carListings) {
73            try {
74                const result = await this.storeCarListing(carData);
75                results.push(result);
76            } catch (error) {
77                log.error(`Error storing car listing ${carData.id}: ${error.message}`);
78                // Continue with next listing
79            }
80        }
81        
82        log.info(`Successfully stored ${results.length} out of ${carListings.length} car listings`);
83        return results;
84    }
85    
86    /**
87     * Find an existing listing by ID
88     * @param {string} listingId Sahibinden.com listing ID
89     * @returns {Promise<Object|null>} Existing row or null if not found
90     * @private
91     */
92    async _findExistingListing(listingId) {
93        try {
94            const response = await this.client.get(
95                `/database/rows/table/${this.tableId}/`,
96                {
97                    params: {
98                        search: listingId,
99                        user_field_names: true
100                    }
101                }
102            );
103            
104            const rows = response.data.results;
105            return rows.find(row => row.id === listingId || row.listing_id === listingId) || null;
106        } catch (error) {
107            log.error(`Error finding existing listing: ${error.message}`);
108            return null;
109        }
110    }
111    
112    /**
113     * Create a new row in BaseRow
114     * @param {Object} rowData Row data
115     * @returns {Promise<Object>} Created row data
116     * @private
117     */
118    async _createRow(rowData) {
119        const response = await this.client.post(
120            `/database/rows/table/${this.tableId}/`,
121            rowData,
122            {
123                params: {
124                    user_field_names: true
125                }
126            }
127        );
128        
129        return response.data;
130    }
131    
132    /**
133     * Update an existing row in BaseRow
134     * @param {number} rowId BaseRow row ID
135     * @param {Object} rowData Row data
136     * @returns {Promise<Object>} Updated row data
137     * @private
138     */
139    async _updateRow(rowId, rowData) {
140        const response = await this.client.patch(
141            `/database/rows/table/${this.tableId}/${rowId}/`,
142            rowData,
143            {
144                params: {
145                    user_field_names: true
146                }
147            }
148        );
149        
150        return response.data;
151    }
152    
153    /**
154     * Prepare car data for BaseRow
155     * @param {Object} carData Car listing data
156     * @returns {Object} Prepared row data
157     * @private
158     */
159    _prepareRowData(carData) {
160        // Map car data to BaseRow fields
161        // Field names should match the BaseRow table structure
162        return {
163            listing_id: carData.id,
164            url: carData.url,
165            title: carData.title,
166            price: carData.price,
167            price_currency: carData.price_currency,
168            location: carData.location,
169            description: carData.description,
170            
171            // Main info fields
172            make: carData.info?.Marka || '',
173            model: carData.info?.Model || '',
174            series: carData.info?.Seri || '',
175            year: carData.info?.Yıl || '',
176            fuel_type: carData.info?.Yakıt || '',
177            transmission: carData.info?.Vites || '',
178            mileage: carData.info?.KM || '',
179            body_type: carData.info?.['Kasa Tipi'] || '',
180            engine_power: carData.info?.['Motor Gücü'] || '',
181            engine_capacity: carData.info?.['Motor Hacmi'] || '',
182            drive_type: carData.info?.Çekiş || '',
183            doors: carData.info?.Kapı || '',
184            color: carData.info?.Renk || '',
185            warranty: carData.info?.Garanti || '',
186            damage_record: carData.info?.['Ağır Hasar Kayıtlı'] || '',
187            plate_nationality: carData.info?.['Plaka / Uyruk'] || '',
188            seller_type: carData.info?.Kimden || '',
189            trade_in: carData.info?.Takas || '',
190            condition: carData.info?.Durumu || '',
191            
192            // Images as JSON string
193            images: JSON.stringify(carData.images || []),
194            
195            // Attributes and technical specs as JSON strings
196            attributes: JSON.stringify(carData.attributes || {}),
197            technical_specs: JSON.stringify(carData.technicalSpecs || {}),
198            
199            // Metadata
200            scraped_at: carData.scrapedAt,
201            last_updated: new Date().toISOString()
202        };
203    }
204}
205
206/**
207 * Create BaseRow integration from Actor input
208 * @returns {Promise<BaseRowIntegration>} BaseRow integration instance
209 */
210export async function createBaseRowIntegration() {
211    const input = await Actor.getInput() || {};
212    const { baseRowApiToken, baseRowTableId, baseRowDatabaseId } = input;
213    
214    if (!baseRowApiToken || !baseRowTableId || !baseRowDatabaseId) {
215        log.warning('BaseRow integration not configured. Data will only be stored in Apify dataset.');
216        return null;
217    }
218    
219    return new BaseRowIntegration({
220        apiToken: baseRowApiToken,
221        tableId: baseRowTableId,
222        databaseId: baseRowDatabaseId
223    });
224}

1// src/main.js - Puppeteer JavaScript Version for Bulk Category Scraping
2import { Actor } from 'apify';
3import { PuppeteerCrawler, log } from 'crawlee';
4// Import puppeteer-extra and the stealth plugin
5import puppeteer from 'puppeteer-extra';
6import StealthPlugin from 'puppeteer-extra-plugin-stealth';
7
8// Apply the stealth plugin to puppeteer
9puppeteer.use(StealthPlugin());
10
11// Initialize the Apify Actor
12await Actor.init();
13
14// Get input - simplified for bulk scraping
15const input = await Actor.getInput() || {};
16const {
17    startUrls = ['https://www.sahibinden.com/otomobil/elektrik'], // Specific category
18    maxItems = null, // Optional: Limit total listings scraped (null for all)
19    maxConcurrency = 5, // Can increase concurrency slightly for category pages
20    maxRequestsPerCrawl = 500, // Increase to handle many pages
21    proxyConfiguration = {
22        useApifyProxy: true,
23        apifyProxyGroups: ['RESIDENTIAL'],
24    }
25} = input;
26
27// Create the final proxy configuration object
28const proxyConfig = await Actor.createProxyConfiguration(proxyConfiguration);
29
30log.info('Starting BULK scraper', { startUrls, maxItems, maxConcurrency, maxRequestsPerCrawl });
31if (proxyConfig) {
32    log.info('Using proxy configuration', { type: proxyConfig.usesApifyProxy ? 'Apify Proxy' : 'Custom Proxies', groups: proxyConfig.apifyProxyGroups });
33} else {
34    log.info('No proxy configuration specified.');
35}
36
37let scrapedItemsCount = 0; // Counter for maxItems
38
39// --- Helper function for random User Agent (keep as before) ---
40function randomUserAgent() {
41    const userAgents = [ /* ... Fill with many user agents ... */ ];
42    return userAgents[Math.floor(Math.random() * userAgents.length)];
43}
44// -------------------------------------------
45
46
47// Create the Puppeteer crawler
48const crawler = new PuppeteerCrawler({
49    proxyConfiguration: proxyConfig,
50    maxConcurrency,
51    maxRequestsPerCrawl,
52    launchContext: {
53        launcher: puppeteer,
54        launchOptions: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] },
55        useChrome: false
56    },
57    preNavigationHooks: [ /* ... Keep your existing preNavigationHooks ... */ ],
58
59    requestHandler: async ({ page, request, enqueueLinks, log }) => {
60        log.info(`Processing page: ${request.url}`);
61        try {
62            // ... Keep Cloudflare check logic ...
63
64            await page.waitForSelector('body', { timeout: 45000 });
65
66            // --- Always handle as category page now ---
67            await handleCategoryPage(page, request, enqueueLinks, log);
68
69        } catch (error) {
70            let errorMessage = 'Unknown error during request handling';
71            if (error instanceof Error) { errorMessage = error.message; } else { errorMessage = String(error); }
72            log.error(`Error processing ${request.url}: ${errorMessage}`, { stack: error instanceof Error ? error.stack : undefined });
73            throw error;
74        }
75    },
76    failedRequestHandler: async ({ request, log }) => {
77        // ... Keep existing failedRequestHandler ...
78    },
79});
80
81// --- MODIFIED Handler Function for Category Pages ---
82
83async function handleCategoryPage(page, request, enqueueLinks, log) {
84    log.info(`Handling category page: ${request.url}`);
85
86    // --- VERIFY THESE SELECTORS on https://www.sahibinden.com/otomobil/elektrik ---
87    const listingRowSelector = 'tbody.searchResultsRowClass > tr.searchResultsItem'; // Row for each listing
88    // Selectors relative to the listingRowSelector element:
89    const makeSelector = 'td:nth-child(2)'; // Assuming Make is the 2nd column
90    const seriesSelector = 'td:nth-child(3)'; // Assuming Series is the 3rd column
91    const modelSelector = 'td:nth-child(4)'; // Assuming Model is the 4th column
92    const titleLinkSelector = 'td.searchResultsTitleValue a.classifiedTitle'; // Title link
93    const yearSelector = 'td:nth-child(7)'; // Assuming Year is the 7th column (VERIFY INDEX!)
94    const kmSelector = 'td:nth-child(8)'; // Assuming KM is the 8th column (VERIFY INDEX!)
95    const priceSelector = 'td.searchResultsPriceValue span'; // Price text
96    const dateSelector = 'td.searchResultsDateValue'; // Date column
97    const locationSelector = 'td.searchResultsLocationValue'; // Location column
98    // Selector for the next page link:
99    const nextPageSelector = 'a.prevNextBut[title="Sonraki"]:not(.passive)';
100    // -------------------------------------------------------------------------
101
102    try {
103        await page.waitForSelector(listingRowSelector, { timeout: 45000 });
104
105        const listingElements = await page.$$(listingRowSelector);
106        log.info(`Found ${listingElements.length} listings on page.`);
107
108        const results = []; // Collect results from this page
109
110        for (const element of listingElements) {
111
112             // Check if maxItems limit is reached
113             if (maxItems !== null && scrapedItemsCount >= maxItems) {
114                log.info(`Maximum items limit (${maxItems}) reached. Stopping scrape.`);
115                // Abort the crawl cleanly
116                await crawler.autoscaledPool.abort();
117                return; // Stop processing this page
118            }
119
120            try {
121                // Extract data relative to the current element 'element'
122                const make = await element.$eval(makeSelector, el => el.textContent?.trim()).catch(() => null);
123                const series = await element.$eval(seriesSelector, el => el.textContent?.trim()).catch(() => null);
124                const model = await element.$eval(modelSelector, el => el.textContent?.trim()).catch(() => null);
125                const titleElement = await element.$(titleLinkSelector); // Get handle for title/link
126                const title = await titleElement?.evaluate(el => el.textContent?.trim()).catch(() => null);
127                const detailUrl = await titleElement?.evaluate(el => el.href).catch(() => null);
128                const year = await element.$eval(yearSelector, el => el.textContent?.trim()).catch(() => null);
129                const km = await element.$eval(kmSelector, el => el.textContent?.trim()).catch(() => null);
130                const price = await element.$eval(priceSelector, el => el.textContent?.trim()).catch(() => null);
131                const date = await element.$eval(dateSelector, el => el.innerText?.trim().replace('\n', ' ')).catch(() => null); // Get innerText to handle <br>
132                const location = await element.$eval(locationSelector, el => el.innerText?.trim().replace('<br>', ' / ')).catch(() => null); // Get innerText and replace <br>
133
134                // Basic check if we got essential data
135                if (title && detailUrl) {
136                    const listingData = {
137                        // Match field names exactly as requested
138                        'Marka': make,
139                        'Seri': series,
140                        'Model': model,
141                        'İlan Başlığı': title,
142                        'Yıl': year,
143                        'KM': km, // Keep as string for now, cleaning later is easier
144                        'Fiyat': price, // Keep as string
145                        'İlan Tarihi': date,
146                        'İl / İlçe': location,
147                        'Detay Linki': detailUrl // Good to store the link too
148                    };
149                    results.push(listingData);
150                    scrapedItemsCount++;
151                } else {
152                    log.debug('Skipping row due to missing title or detailUrl.');
153                }
154
155            } catch (extractError) {
156                let errorMsg = 'Unknown error extracting item data';
157                if (extractError instanceof Error) { errorMsg = extractError.message; } else { errorMsg = String(extractError); }
158                log.warning(`Could not process one item fully on ${request.url}`, { error: errorMsg });
159            }
160        } // End of loop
161
162        // Push all results from this page
163        if (results.length > 0) {
164            await Actor.pushData(results);
165            log.info(`Pushed ${results.length} listings from page ${request.url}. Total scraped: ${scrapedItemsCount}`);
166        } else {
167            log.info(`No listings extracted from page ${request.url}.`);
168        }
169
170        // --- Enqueue next page ---
171        // Check if maxItems limit is reached before enqueueing next page
172         if (maxItems !== null && scrapedItemsCount >= maxItems) {
173            log.info(`Maximum items limit (${maxItems}) reached. Not enqueueing next page.`);
174            await crawler.autoscaledPool.abort(); // Ensure crawl stops
175            return;
176        }
177
178        const nextPageUrl = await page.$eval(nextPageSelector, anchor => anchor.href).catch(() => null);
179        if (nextPageUrl) {
180            log.info(`Enqueueing next category page: ${nextPageUrl}`);
181            const absoluteNextPageUrl = new URL(nextPageUrl, request.loadedUrl || request.url).toString();
182            // Add request with the same label 'CATEGORY'
183            // NOTE: No label needed if only one handler, but good practice
184            await enqueueLinks({
185                urls: [absoluteNextPageUrl],
186                userData: { label: 'CATEGORY' }, // Use label for consistency
187            });
188        } else {
189            log.info(`No next page button found or clickable on ${request.url}`);
190        }
191
192    } catch (error) {
193        let errorMessage = 'Unknown error handling category page';
194        if (error instanceof Error) { errorMessage = error.message; } else { errorMessage = String(error); }
195        log.warning(`Could not handle category page ${request.url}: ${errorMessage}`);
196        // Consider if this should throw error to retry the page
197        // throw error;
198    }
199}
200
201// --- handleDetailPage is no longer needed for this approach ---
202// async function handleDetailPage(page, request, log) { ... }
203
204
205// --- Start the Crawler ---
206const startRequests = startUrls.map(item => {
207    // ... (Keep the existing logic to handle string or object input for startUrls) ...
208    let urlString;
209    if (typeof item === 'string') { urlString = item; }
210    else if (item && typeof item.url === 'string') { urlString = item.url; }
211    else { log.warning('Skipping invalid start URL item:', { item }); return null; }
212
213    if (!urlString || !urlString.startsWith('http')) {
214        log.warning('Skipping item with invalid URL string:', { urlString }); return null;
215    }
216    return { url: urlString, userData: { label: 'CATEGORY' } }; // Always label as CATEGORY now
217}).filter(req => req !== null);
218
219
220if (startRequests.length > 0) {
221    await crawler.addRequests(startRequests);
222    log.info(`Added ${startRequests.length} initial requests to the queue.`);
223} else {
224     log.warning('No valid start URLs found in the input. Exiting.');
225     await Actor.exit(1, 'No valid start URLs provided.');
226}
227
228log.info('Starting the crawler...');
229await crawler.run();
230log.info(`Crawler finished. Total items scraped: ${scrapedItemsCount}`);
231
232await Actor.exit();
233// --- End of Script ---

1import { Dataset, createPuppeteerRouter } from 'crawlee';
2
3export const router = createPuppeteerRouter();
4
5router.addDefaultHandler(async ({ enqueueLinks, log }) => {
6    log.info(`enqueueing new URLs`);
7    await enqueueLinks({
8        globs: ['https://apify.com/*'],
9        label: 'detail',
10    });
11});
12
13router.addHandler('detail', async ({ request, page, log }) => {
14    const title = await page.title();
15    log.info(`${title}`, { url: request.loadedUrl });
16
17    await Dataset.pushData({
18        url: request.loadedUrl,
19        title,
20    });
21});

1import { Actor } from 'apify';
2import { log } from 'crawlee';
3import { createBaseRowIntegration } from './baserow.js';
4
5/**
6 * Scheduler for automating the Sahibinden.com scraper
7 * This module handles the integration with Apify Scheduler
8 */
9export class ScraperScheduler {
10    /**
11     * Initialize the scheduler
12     * @param {Object} options Configuration options
13     * @param {string} options.scheduleInterval How often to run the scraper (e.g., 'daily', 'weekly')
14     * @param {string} options.startTime Time to start the scraper (e.g., '02:00')
15     * @param {Array<string>} options.startUrls URLs to start scraping from
16     */
17    constructor(options = {}) {
18        const {
19            scheduleInterval = 'daily',
20            startTime = '02:00',
21            startUrls = ['https://www.sahibinden.com/kategori/vasita']
22        } = options;
23        
24        this.scheduleInterval = scheduleInterval;
25        this.startTime = startTime;
26        this.startUrls = startUrls;
27        
28        log.info('Scheduler initialized', { scheduleInterval, startTime, startUrls });
29    }
30    
31    /**
32     * Create an Apify scheduler configuration
33     * @returns {Object} Scheduler configuration
34     */
35    createSchedulerConfig() {
36        // Convert schedule interval to cron expression
37        let cronExpression;
38        switch (this.scheduleInterval.toLowerCase()) {
39            case 'hourly':
40                cronExpression = '0 * * * *';
41                break;
42            case 'daily':
43                const [hours, minutes] = this.startTime.split(':').map(Number);
44                cronExpression = `${minutes || 0} ${hours || 0} * * *`;
45                break;
46            case 'weekly':
47                const [weeklyHours, weeklyMinutes] = this.startTime.split(':').map(Number);
48                cronExpression = `${weeklyMinutes || 0} ${weeklyHours || 0} * * 1`; // Monday
49                break;
50            default:
51                cronExpression = '0 2 * * *'; // Default: 2 AM daily
52        }
53        
54        return {
55            cronExpression,
56            isEnabled: true,
57            isExclusive: true,
58            description: 'Scheduled run of Sahibinden.com scraper',
59            actorTaskId: Actor.getEnv().actorTaskId || undefined,
60            actorId: Actor.getEnv().actorId || undefined,
61            input: {
62                startUrls: this.startUrls,
63                maxConcurrency: 1,
64                maxRequestsPerCrawl: 1000,
65                proxyConfiguration: {
66                    useApifyProxy: true,
67                    apifyProxyGroups: ['RESIDENTIAL'],
68                    countryCode: 'TR'
69                }
70            }
71        };
72    }
73    
74    /**
75     * Set up the scheduler on Apify platform
76     * @returns {Promise<Object>} Created scheduler
77     */
78    async setupScheduler() {
79        try {
80            const client = Actor.newClient();
81            const config = this.createSchedulerConfig();
82            
83            log.info('Setting up scheduler with configuration', { cronExpression: config.cronExpression });
84            
85            // Create a new scheduler
86            const scheduler = await client.schedules().create({
87                name: 'Sahibinden.com Scraper Schedule',
88                cronExpression: config.cronExpression,
89                isEnabled: config.isEnabled,
90                isExclusive: config.isExclusive,
91                description: config.description,
92                actions: [{
93                    actorId: config.actorId,
94                    actorTaskId: config.actorTaskId,
95                    input: config.input
96                }]
97            });
98            
99            log.info('Scheduler created successfully', { schedulerId: scheduler.id });
100            return scheduler;
101        } catch (error) {
102            log.error('Failed to set up scheduler', { error: error.message });
103            throw error;
104        }
105    }
106    
107    /**
108     * Monitor the performance of scheduled runs
109     * @param {number} limit Number of runs to retrieve
110     * @returns {Promise<Array>} Recent runs with statistics
111     */
112    async monitorPerformance(limit = 10) {
113        try {
114            const client = Actor.newClient();
115            const actorId = Actor.getEnv().actorId;
116            
117            if (!actorId) {
118                log.warning('Cannot monitor performance: Actor ID not available');
119                return [];
120            }
121            
122            // Get recent runs
123            const runs = await client.actor(actorId).runs().list({
124                limit,
125                desc: true
126            });
127            
128            // Extract performance metrics
129            const performanceStats = runs.items.map(run => ({
130                id: run.id,
131                status: run.status,
132                startedAt: run.startedAt,
133                finishedAt: run.finishedAt,
134                durationSecs: run.finishedAt ? 
135                    (new Date(run.finishedAt) - new Date(run.startedAt)) / 1000 : 
136                    null,
137                resourceUsage: {
138                    cpu: run.stats?.cpuUsage,
139                    memory: run.stats?.memoryUsage,
140                },
141                datasetStats: {
142                    itemCount: run.stats?.datasetsOutStats?.['default']?.itemCount || 0,
143                    totalBytes: run.stats?.datasetsOutStats?.['default']?.totalBytes || 0,
144                },
145                errorMessage: run.errorMessage
146            }));
147            
148            log.info('Retrieved performance statistics', { 
149                runCount: performanceStats.length,
150                latestRun: performanceStats[0]
151            });
152            
153            return performanceStats;
154        } catch (error) {
155            log.error('Failed to monitor performance', { error: error.message });
156            return [];
157        }
158    }
159}
160
161/**
162 * Create scheduler from Actor input
163 * @returns {Promise<ScraperScheduler>} Scheduler instance
164 */
165export async function createScheduler() {
166    const input = await Actor.getInput() || {};
167    const { 
168        scheduleInterval, 
169        startTime, 
170        startUrls 
171    } = input;
172    
173    return new ScraperScheduler({
174        scheduleInterval,
175        startTime,
176        startUrls
177    });
178}

{
  "title": "Sahibinden.com Car Listing Schema",
  "type": "object",
  "properties": {
    "id": {
      "type": "string",
      "description": "Unique identifier for the listing"
    },
    "url": {
      "type": "string",
      "description": "Full URL of the listing"
    },
    "title": {
      "type": "string",
      "description": "Title of the car listing"
    },
    "price": {
      "type": ["number", "null"],
      "description": "Numeric price value (cleaned)"
    },
    "price_currency": {
      "type": "string",
      "description": "Currency of the price (e.g., TL, EUR)"
    },
    "location": {
      "type": "string",
      "description": "Location information (City/District/Neighborhood)"
    },
    "description": {
      "type": "string",
      "description": "Full description text"
    },
    "info": {
      "type": "object",
      "description": "Main information about the vehicle",
      "properties": {
        "İlan No": { "type": "string" },
        "İlan Tarihi": { "type": "string" },
        "Marka": { "type": "string" },
        "Seri": { "type": "string" },
        "Model": { "type": "string" },
        "Yıl": { "type": "string" },
        "Yakıt": { "type": "string" },
        "Vites": { "type": "string" },
        "KM": { "type": "string" },
        "Kasa Tipi": { "type": "string" },
        "Motor Gücü": { "type": "string" },
        "Motor Hacmi": { "type": "string" },
        "Çekiş": { "type": "string" },
        "Kapı": { "type": "string" },
        "Renk": { "type": "string" },
        "Garanti": { "type": "string" },
        "Ağır Hasar Kayıtlı": { "type": "string" },
        "Plaka / Uyruk": { "type": "string" },
        "Kimden": { "type": "string" },
        "Takas": { "type": "string" },
        "Durumu": { "type": "string" }
      },
      "additionalProperties": true
    },
    "images": {
      "type": "array",
      "description": "Array of image URLs",
      "items": {
        "type": "string"
      }
    },
    "attributes": {
      "type": "object",
      "description": "Grouped attributes by section (e.g., Güvenlik, İç Donanım, Dış Donanım, Multimedya)",
      "additionalProperties": {
        "type": "array",
        "items": {
          "type": "string"
        }
      }
    },
    "technicalSpecs": {
      "type": "object",
      "description": "Technical specifications grouped by section",
      "additionalProperties": {
        "type": "object",
        "additionalProperties": true
      }
    },
    "scrapedAt": {
      "type": "string",
      "format": "date-time",
      "description": "Timestamp when the data was scraped"
    }
  },
  "required": ["id", "url", "title", "scrapedAt"]
}

1import { randomBytes } from 'crypto';
2
3/**
4 * Generates a random user agent string for desktop browsers
5 * @returns {string} A random user agent string
6 */
7export function randomUserAgent() {
8    const browsers = [
9        // Chrome on Windows
10        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
11        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
12        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
13        
14        // Chrome on macOS
15        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
16        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
17        
18        // Firefox on Windows
19        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
20        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0',
21        
22        // Firefox on macOS
23        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0',
24        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:119.0) Gecko/20100101 Firefox/119.0',
25        
26        // Safari on macOS
27        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
28        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15',
29        
30        // Edge on Windows
31        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
32        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
33    ];
34    
35    return browsers[Math.floor(Math.random() * browsers.length)];
36}
37
38/**
39 * Generates a random session ID
40 * @returns {string} A random session ID
41 */
42export function generateSessionId() {
43    return `session_${randomBytes(8).toString('hex')}`;
44}
45
46/**
47 * Adds delay between actions to simulate human behavior
48 * @param {number} min Minimum delay in milliseconds
49 * @param {number} max Maximum delay in milliseconds
50 * @returns {Promise<void>} A promise that resolves after the delay
51 */
52export function randomDelay(min = 1000, max = 5000) {
53    const delay = Math.floor(Math.random() * (max - min + 1)) + min;
54    return new Promise(resolve => setTimeout(resolve, delay));
55}
56
57/**
58 * Formats price string to numeric value
59 * @param {string} priceStr Price string (e.g., "150.000 TL")
60 * @returns {number|null} Numeric price value or null if invalid
61 */
62export function formatPrice(priceStr) {
63    if (!priceStr) return null;
64    
65    // Remove all non-numeric characters except decimal point
66    const numericStr = priceStr.replace(/[^0-9,.]/g, '')
67        .replace(/\./g, '') // Remove thousands separator (.)
68        .replace(/,/g, '.'); // Replace comma with decimal point
69    
70    const price = parseFloat(numericStr);
71    return isNaN(price) ? null : price;
72}
73
74/**
75 * Extracts boolean value from Yes/No string in Turkish
76 * @param {string} value String value (e.g., "Evet", "Hayır")
77 * @returns {boolean|null} Boolean value or null if invalid
78 */
79export function parseYesNo(value) {
80    if (!value) return null;
81    
82    const normalized = value.toLowerCase().trim();
83    if (normalized === 'evet' || normalized === 'var') return true;
84    if (normalized === 'hayır' || normalized === 'yok') return false;
85    
86    return null;
87}
88
89/**
90 * Normalizes text by removing extra whitespace
91 * @param {string} text Input text
92 * @returns {string} Normalized text
93 */
94export function normalizeText(text) {
95    if (!text) return '';
96    return text.replace(/\s+/g, ' ').trim();
97}