Apple Ads Library Scraper

1import { Actor } from 'apify';
2import { PlaywrightCrawler } from 'crawlee';
3
4await Actor.init();
5
6// Get input from the schema
7const input = await Actor.getInput();
8const country = input.COUNTRY_OR_REGION || 'Austria'; // Use the input key from schema
9
10const crawler = new PlaywrightCrawler({
11    requestHandler: async ({ page, request, log }) => {
12        log.info(`Processing ${request.url} with label: ${request.label}`);
13
14        if (request.label === 'START') {
15            // Navigate to the main page
16            await page.goto('https://adrepository.apple.com/');
17            log.info(`Selecting Country or Region: ${country}`);
18            await page.getByLabel('Country or Region').click();
19            await page.getByLabel(country).check();
20            await page.getByRole('button', { name: 'Apply' }).click();
21
22            log.info('Waiting for results to load...');
23            await page.waitForTimeout(3000); // Allow results to begin loading
24            await page.waitForSelector('.br-md-bottom', { state: 'visible', timeout: 60000 });
25
26            // Locate buttons and process them manually
27            const buttons = page.locator('.br-md-bottom .button');
28            const buttonCount = await buttons.count();
29            log.info(`Found ${buttonCount} buttons for ad details.`);
30
31            for (let i = 0; i < buttonCount; i++) {
32                log.info(`Processing button ${i + 1} of ${buttonCount}`);
33
34                // Click on the button to navigate to ad details
35                await buttons.nth(i).click();
36                await page.waitForTimeout(3000); // Wait for ad details to load
37
38                // Extract ad details
39                const adDetails = {
40                    app: await page.locator('text=App').locator('xpath=following-sibling::*').nth(0).textContent(),
41                    developer: await page.locator('text=Developer').locator('xpath=following-sibling::*').nth(0).textContent(),
42                    legalName: await page.locator('text=Legal Name').locator('xpath=following-sibling::*').nth(0).textContent(),
43                    placement: await page.locator('text=Placement').locator('xpath=following-sibling::*').nth(0).textContent(),
44                    format: await page.locator('text=Format').locator('xpath=following-sibling::*').nth(0).textContent(),
45                    country: await page.locator('text=Country or Region').locator('xpath=following-sibling::*').nth(0).textContent(),
46                    parameters: await page.locator('text=Parameters').locator('xpath=following-sibling::*').nth(0).textContent(),
47                    firstImpression: await page.locator('text=First Impression').locator('xpath=following-sibling::*').nth(0).textContent(),
48                    latestImpression: await page.locator('text=Latest Impression').locator('xpath=following-sibling::*').nth(0).textContent(),
49                };
50                log.info(`Extracted Ad Details: ${JSON.stringify(adDetails)}`);
51                await Actor.pushData(adDetails);
52
53                // Return to main page
54                log.info('Returning to main page...');
55                await page.goBack();
56                await page.waitForSelector('.br-md-bottom', { state: 'visible', timeout: 60000 });
57            }
58
59            // Check for next page
60            const nextButton = page.getByLabel('next');
61            if (await nextButton.isVisible() && await nextButton.isEnabled()) {
62                log.info('Navigating to the next page...');
63                await nextButton.click();
64                await page.waitForTimeout(5000); // Wait for the next page to load
65                await page.waitForSelector('.br-md-bottom', { state: 'visible', timeout: 60000 });
66
67                // Repeat the process for the next page
68                const buttonsNextPage = page.locator('.br-md-bottom .button');
69                const buttonCountNextPage = await buttonsNextPage.count();
70                log.info(`Found ${buttonCountNextPage} buttons on the next page.`);
71            } else {
72                log.info('No more pages available.');
73            }
74        }
75    },
76});
77
78// Add the start URL with a label
79await crawler.addRequests([{ url: 'https://adrepository.apple.com/', label: 'START' }]);
80
81await crawler.run();
82await Actor.exit();

1import { Router } from 'crawlee';
2
3// Define routes for different scraping actions
4export const routes = Router();
5
6routes.addDefaultHandler(async ({ page, request, log }) => {
7    log.info(`Processing URL: ${request.url}`);
8
9    // Implement additional logic if required
10    await page.goto(request.url);
11    // Example: Collect page title
12    const title = await page.title();
13    log.info(`Page title: ${title}`);
14});

1import { test, expect } from '@playwright/test';
2
3test('navigate and apply filters', async ({ page }) => {
4  await page.goto('https://adrepository.apple.com/');
5  await page.getByLabel('Country or Region').click();
6  await page.getByLabel('Austria').click();
7  await page.getByRole('button', { name: 'Apply' }).click();
8
9  await page.waitForSelector('div.results-container', { state: 'visible' });
10  const resultsCount = await page.locator('div.results-container > div').count();
11  console.log('Results count:', resultsCount);
12});