Google Ads Scraper Pro beta

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:20

# Check preinstalled packages
RUN npm ls crawlee apify puppeteer playwright

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent

{
    "actorSpecification": 1,
    "name": "my-actor-2",
    "title": "Google Ads Transparency Scraper",
    "description": "Crawlee and Cheerio project in javascript.",
    "version": "0.0",
    "meta": {
        "templateId": "js-crawlee-cheerio"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile",
    "readme": "./README.md",
    "storages": {
        "dataset": {
            "actorSpecification": 1,
            "title": "Results",
            "views": {
                "results": {
                "title": "results to scan",
                "transformation": {
                    "fields": ["advertiserId", "creativeId", "format", "previewUrls", "creativeRegions", "adTransparencyUrl", "regionStats"]
                },
                "display": {
                    "component": "table",
                    "properties": {
                        "advertiserId": {
                            "label": "advertiser Id",
                            "format": "text"
                        },
                        "creativeId": {
                            "label": "creative Id",
                            "format": "text"
                        },
                        "format": {
                            "label": "format",
                            "format": "text"
                        },
                        "previewUrls": {
                            "label": "previewUrls",
                            "format": "text"
                        },
                        "creativeRegions": {
                            "label": "creative Regions",
                            "format": "text"
                        },
                        "adTransparencyUrl": {
                            "label": "adTransparency Url",
                            "format": "link"
                        },
                        "regionStats": {
                            "label": "region Stats",
                            "format": "text"
                        }
                }
                }
            }
        }
    }
}
}

{
    "title": "Ad Search Configuration",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "Use_URL_Filter": {
            "title": "Use parameter URL as a filter",
            "type": "boolean",
            "description": "Debug messages will be included in the log.",
            "default": true,
            "groupCaption": "Options",
            "groupDescription": "Various options for this Actor"
        },
        "isPolitical": {
            "title": "isPolitical",
            "type": "boolean",
            "description": "Debug messages will be included in the log.",
            "default": true,
            "groupCaption": "Options",
            "groupDescription": "Various options for this Actor"
        },
        "keyword_domain_advertiserid": {
            "title": "Search Keywords",
            "type": "array",
            "description": "Enter one or more keywords to search for relevant ads.",
            "editor": "stringList",
            "default":
            [
                "AR05789404224659914753",
                "https://adstransparency.google.com/advertiser/AR05789404224659914753?region=US&topic=political&preset-date=Last+7+days&format=VIDEO&platform=YOUTUBE"
            ]
        },
        "limitResults": {
            "title": "limit Results",
            "type": "integer",
            "description": "Specify the maximum number of unique domains to retrieve.",
            "editor": "number",
            "default": 3

        },
        "maxDomains": {
            "title": "Maximum Domains",
            "type": "integer",
            "description": "Specify the maximum number of unique domains to retrieve.",
            "default": 0,
            "editor": "number"
        },
        "maxAdvertisers": {
            "title": "Maximum Advertisers",
            "type": "integer",
            "description": "Set the maximum number of advertisers to fetch results from.",
            "default": 10,
            "editor": "number"
        },
        "platform": {
            "title": "",
            "type": "string",
            "description": "",
            "editor": "select",
            "default": "0",
            "enum": ["0", "2", "1", "4", "3", "5"],
            "enumTitles": ["All platforms", "Google Maps", "Google Play", "Google Shopping", "Google Search", "YouTube"]
        },
        "format": {
            "title": "Ad Format",
            "type": "string",
            "description": "Select the preferred ad format to filter results.",
            "editor": "select",
            "default": "0",
            "enum": ["0", "1", "2", "3"],
            "enumTitles": ["All formats", "Text", "Image", "Video"]
        },
        "presetDate": {
            "title": "Date Range",
            "type": "string",
            "description": "Select a predefined date range for filtering ads.",
            "editor": "select",
            "default": "0",
            "enum": ["0", "1", "7", "30", "-1"],
            "enumTitles": ["Today", "Yesterday", "Last 7 days", "Last 30 days", "Any time"]
        },
        "startDate": {
            "title": "Start Date",
            "type": "string",
            "description": "Specify a custom start date for filtering ads (YYYYMMDD).",
            "editor": "datepicker",
            "default": "2024-01-01"

        },
        "endDate": {
            "title": "End Date",
            "type": "string",
            "description": "Specify a custom end date for filtering ads (YYYYMMDD).",
            "editor": "datepicker"
        },
        "region": {
            "title": "Target Region",
            "type": "string",
            "description": "Select a specific country/region to filter ads.",
            "editor": "select",
            "enum": [
                "Albania",
                "Algeria",
                "American Samoa",
                "Andorra",
                "Angola",
                "Antarctica",
                "Antigua and Barbuda",
                "Argentina",
                "Armenia",
                "Australia",
                "Austria",
                "Azerbaijan",
                "Bahrain",
                "Bangladesh",
                "Barbados",
                "Belarus",
                "Belgium",
                "Belize",
                "Benin",
                "Bhutan",
                "Bolivia",
                "Bosnia and Herzegovina",
                "Botswana",
                "Brazil",
                "Brunei",
                "Bulgaria",
                "Burkina Faso",
                "Burundi",
                "Cambodia",
                "Cameroon",
                "Canada",
                "Cabo Verde",
                "Caribbean Netherlands",
                "Central African Republic",
                "Chad",
                "Chile",
                "China",
                "Christmas Island",
                "Cocos (Keeling) Islands",
                "Colombia",
                "Comoros",
                "Cook Islands",
                "Costa Rica",
                "Cote d'Ivoire",
                "Croatia",
                "Curacao",
                "Cyprus",
                "Czechia",
                "Democratic Republic of the Congo",
                "Denmark",
                "Djibouti",
                "Dominica",
                "Dominican Republic",
                "Ecuador",
                "Egypt",
                "El Salvador",
                "Equatorial Guinea",
                "Eritrea",
                "Estonia",
                "Eswatini",
                "Ethiopia",
                "Micronesia",
                "Fiji",
                "Finland",
                "France",
                "French Polynesia",
                "French Southern and Antarctic Lands",
                "Gabon",
                "Georgia",
                "Germany",
                "Ghana",
                "Greece",
                "Grenada",
                "Guam",
                "Guatemala",
                "Guernsey",
                "Guinea",
                "Guinea-Bissau",
                "Guyana",
                "Haiti",
                "Heard Island and McDonald Islands",
                "Honduras",
                "Hungary",
                "Iceland",
                "India",
                "Indonesia",
                "Iraq",
                "Ireland",
                "Israel",
                "Italy",
                "Jamaica",
                "Japan",
                "Jersey",
                "Jordan",
                "Kazakhstan",
                "Kenya",
                "Kiribati",
                "Kuwait",
                "Kyrgyzstan",
                "Laos",
                "Latvia",
                "Lebanon",
                "Lesotho",
                "Liberia",
                "Libya",
                "Liechtenstein",
                "Lithuania",
                "Luxembourg",
                "Madagascar",
                "Malawi",
                "Malaysia",
                "Maldives",
                "Malta",
                "Mali",
                "Marshall Islands",
                "Mauritania",
                "Mauritius",
                "Mexico",
                "Moldova",
                "Monaco",
                "Mongolia",
                "Montenegro",
                "Morocco",
                "Mozambique",
                "Myanmar (Burma)",
                "Namibia",
                "Nauru",
                "Nepal",
                "Netherlands",
                "New Caledonia",
                "New Zealand",
                "Nicaragua",
                "Niger",
                "Nigeria",
                "Niue",
                "Norfolk Island",
                "North Macedonia",
                "Northern Mariana Islands",
                "Norway",
                "Oman",
                "Pakistan",
                "Palau",
                "Panama",
                "Papua New Guinea",
                "Paraguay",
                "Peru",
                "Philippines",
                "Pitcairn Islands",
                "Poland",
                "Portugal",
                "Qatar",
                "Republic of the Congo",
                "Romania",
                "Rwanda",
                "Saint Helena, Ascension and Tristan da Cunha",
                "Saint Kitts and Nevis",
                "Saint Lucia",
                "Saint Pierre and Miquelon",
                "Saint Vincent and the Grenadines",
                "Samoa",
                "San Marino",
                "Sao Tome and Principe",
                "Saudi Arabia",
                "Senegal",
                "Seychelles",
                "Sierra Leone",
                "Singapore",
                "Sint Maarten",
                "Slovakia",
                "Slovenia",
                "Solomon Islands",
                "Somalia",
                "South Africa",
                "South Georgia and the South Sandwich Islands",
                "South Korea",
                "Spain",
                "Sri Lanka",
                "Suriname",
                "Switzerland",
                "Sweden",
                "Tajikistan",
                "Tanzania",
                "Thailand",
                "The Bahamas",
                "The Gambia",
                "Timor-Leste",
                "Togo",
                "Tokelau",
                "Tonga",
                "Trinidad and Tobago",
                "Tunisia",
                "Turkiye",
                "Turkmenistan",
                "Tuvalu",
                "Uganda",
                "Ukraine",
                "United Arab Emirates",
                "United Kingdom",
                "United States",
                "Uruguay",
                "Uzbekistan",
                "Vanuatu",
                "Vatican City",
                "Venezuela",
                "Vietnam",
                "Yemen",
                "Zambia",
                "Zimbabwe",
                "Bermuda",
                "Bouvet Island",
                "British Indian Ocean Territory",
                "British Virgin Islands",
                "Canary Islands",
                "Cayman Islands",
                "Taiwan",
                "Mayotte",
                "Faroe Islands",
                "Falkland Islands (Islas Malvinas)",
                "French Guiana",
                "Palestine",
                "Gibraltar",
                "Greenland",
                "Guadeloupe",
                "Hong Kong",
                "Isle of Man",
                "Macao",
                "Martinique",
                "Montserrat",
                "Aruba",
                "Puerto Rico",
                "Reunion",
                "Anguilla",
                "Western Sahara",
                "Saint Martin",
                "Svalbard and Jan Mayen",
                "Turks and Caicos Islands",
                "United States Minor Outlying Islands",
                "U.S. Virgin Islands",
                "Wallis and Futuna"
            ],
            "enumTitles": [
                "Albania",
                "Algeria",
                "American Samoa",
                "Andorra",
                "Angola",
                "Antarctica",
                "Antigua and Barbuda",
                "Argentina",
                "Armenia",
                "Australia",
                "Austria",
                "Azerbaijan",
                "Bahrain",
                "Bangladesh",
                "Barbados",
                "Belarus",
                "Belgium",
                "Belize",
                "Benin",
                "Bhutan",
                "Bolivia",
                "Bosnia and Herzegovina",
                "Botswana",
                "Brazil",
                "Brunei",
                "Bulgaria",
                "Burkina Faso",
                "Burundi",
                "Cambodia",
                "Cameroon",
                "Canada",
                "Cabo Verde",
                "Caribbean Netherlands",
                "Central African Republic",
                "Chad",
                "Chile",
                "China",
                "Christmas Island",
                "Cocos (Keeling) Islands",
                "Colombia",
                "Comoros",
                "Cook Islands",
                "Costa Rica",
                "Cote d'Ivoire",
                "Croatia",
                "Curacao",
                "Cyprus",
                "Czechia",
                "Democratic Republic of the Congo",
                "Denmark",
                "Djibouti",
                "Dominica",
                "Dominican Republic",
                "Ecuador",
                "Egypt",
                "El Salvador",
                "Equatorial Guinea",
                "Eritrea",
                "Estonia",
                "Eswatini",
                "Ethiopia",
                "Micronesia",
                "Fiji",
                "Finland",
                "France",
                "French Polynesia",
                "French Southern and Antarctic Lands",
                "Gabon",
                "Georgia",
                "Germany",
                "Ghana",
                "Greece",
                "Grenada",
                "Guam",
                "Guatemala",
                "Guernsey",
                "Guinea",
                "Guinea-Bissau",
                "Guyana",
                "Haiti",
                "Heard Island and McDonald Islands",
                "Honduras",
                "Hungary",
                "Iceland",
                "India",
                "Indonesia",
                "Iraq",
                "Ireland",
                "Israel",
                "Italy",
                "Jamaica",
                "Japan",
                "Jersey",
                "Jordan",
                "Kazakhstan",
                "Kenya",
                "Kiribati",
                "Kuwait",
                "Kyrgyzstan",
                "Laos",
                "Latvia",
                "Lebanon",
                "Lesotho",
                "Liberia",
                "Libya",
                "Liechtenstein",
                "Lithuania",
                "Luxembourg",
                "Madagascar",
                "Malawi",
                "Malaysia",
                "Maldives",
                "Malta",
                "Mali",
                "Marshall Islands",
                "Mauritania",
                "Mauritius",
                "Mexico",
                "Moldova",
                "Monaco",
                "Mongolia",
                "Montenegro",
                "Morocco",
                "Mozambique",
                "Myanmar (Burma)",
                "Namibia",
                "Nauru",
                "Nepal",
                "Netherlands",
                "New Caledonia",
                "New Zealand",
                "Nicaragua",
                "Niger",
                "Nigeria",
                "Niue",
                "Norfolk Island",
                "North Macedonia",
                "Northern Mariana Islands",
                "Norway",
                "Oman",
                "Pakistan",
                "Palau",
                "Panama",
                "Papua New Guinea",
                "Paraguay",
                "Peru",
                "Philippines",
                "Pitcairn Islands",
                "Poland",
                "Portugal",
                "Qatar",
                "Republic of the Congo",
                "Romania",
                "Rwanda",
                "Saint Helena, Ascension and Tristan da Cunha",
                "Saint Kitts and Nevis",
                "Saint Lucia",
                "Saint Pierre and Miquelon",
                "Saint Vincent and the Grenadines",
                "Samoa",
                "San Marino",
                "Sao Tome and Principe",
                "Saudi Arabia",
                "Senegal",
                "Seychelles",
                "Sierra Leone",
                "Singapore",
                "Sint Maarten",
                "Slovakia",
                "Slovenia",
                "Solomon Islands",
                "Somalia",
                "South Africa",
                "South Georgia and the South Sandwich Islands",
                "South Korea",
                "Spain",
                "Sri Lanka",
                "Suriname",
                "Switzerland",
                "Sweden",
                "Tajikistan",
                "Tanzania",
                "Thailand",
                "The Bahamas",
                "The Gambia",
                "Timor-Leste",
                "Togo",
                "Tokelau",
                "Tonga",
                "Trinidad and Tobago",
                "Tunisia",
                "Turkiye",
                "Turkmenistan",
                "Tuvalu",
                "Uganda",
                "Ukraine",
                "United Arab Emirates",
                "United Kingdom",
                "United States",
                "Uruguay",
                "Uzbekistan",
                "Vanuatu",
                "Vatican City",
                "Venezuela",
                "Vietnam",
                "Yemen",
                "Zambia",
                "Zimbabwe",
                "Bermuda",
                "Bouvet Island",
                "British Indian Ocean Territory",
                "British Virgin Islands",
                "Canary Islands",
                "Cayman Islands",
                "Taiwan",
                "Mayotte",
                "Faroe Islands",
                "Falkland Islands (Islas Malvinas)",
                "French Guiana",
                "Palestine",
                "Gibraltar",
                "Greenland",
                "Guadeloupe",
                "Hong Kong",
                "Isle of Man",
                "Macao",
                "Martinique",
                "Montserrat",
                "Aruba",
                "Puerto Rico",
                "Reunion",
                "Anguilla",
                "Western Sahara",
                "Saint Martin",
                "Svalbard and Jan Mayen",
                "Turks and Caicos Islands",
                "United States Minor Outlying Islands",
                "U.S. Virgin Islands",
                "Wallis and Futuna"
            ]
        },
        "proxyConfig": {
            "sectionCaption": "Proxy Configuration",
            "sectionDescription": "Configure proxy settings to use during the scan. Utilizing proxies can help you avoid IP bans, bypass WAF (Web Application Firewall) restrictions, and access specific geographical regions. This ensures smooth crawling across various environments and enhances the overall efficiency and reliability of your scanning activities.",
            "title": "Proxy",
            "type": "object",
            "description": "Configure proxy settings to use during the scan. Utilizing proxies can help you avoid IP bans, bypass WAF (Web Application Firewall) restrictions, and access specific geographical regions. This ensures smooth crawling across various environments and enhances the overall efficiency and reliability of your scanning activities.",
            "default": {"useApifyProxy": false},
            "editor": "proxy"
        }
    }
  }

1import { Actor } from 'apify';
2
3await Actor.init();
4
5// Load and parse input parameters
6const input = await Actor.getInput() ?? {};
7const { proxyConfig } = input;
8
9// Create a helper function to handle proxy rotation and retries
10export async function fetchWithProxyRotation(options, postData, maxRetries = 3) {
11    const { request } = await import('https');
12    let retryCount = 0;
13    let lastError = null;
14    let proxyConfiguration = null;
15    let useProxy = !!proxyConfig; // Check if proxyConfig is provided
16
17    if (useProxy) {
18        try {
19            proxyConfiguration = await Actor.createProxyConfiguration(proxyConfig);
20            if (!proxyConfiguration) {
21                console.warn("⚠️ Proxy configuration failed. Running without proxy.");
22                useProxy = false;
23            }
24        } catch (error) {
25            console.warn("⚠️ Failed to initialize proxy configuration. Running without proxy.");
26            useProxy = false;
27        }
28    }
29
30    while (retryCount < maxRetries) {
31        try {
32            let proxyAgent = null;
33            let proxyUrl = null;
34
35            if (useProxy && proxyConfiguration) {
36                try {
37                    proxyUrl = await proxyConfiguration.newUrl();
38                    if (!proxyUrl) {
39                        console.warn("⚠️ Failed to retrieve proxy URL. Continuing without a proxy.");
40                        useProxy = false;
41                    } else {
42                        const { HttpsProxyAgent } = await import('https-proxy-agent');
43                        proxyAgent = new HttpsProxyAgent(proxyUrl);
44                        console.log(`🔄 Attempt ${retryCount + 1}/${maxRetries} using proxy: ${proxyUrl.replace(/:[^:]*@/, ':****@')}`);
45                    }
46                } catch (error) {
47                    console.warn("⚠️ Error getting new proxy URL. Running without proxy.");
48                    useProxy = false;
49                }
50            } else {
51                console.log(`🔄 Attempt ${retryCount + 1}/${maxRetries} without a proxy.`);
52            }
53
54            // Update options with proxy settings if available
55            const proxyOptions = {
56                ...options,
57                agent: proxyAgent || undefined
58            };
59
60            // Make the request with the current proxy
61            const response = await new Promise((resolve, reject) => {
62                const req = request(proxyOptions, (res) => {
63                    let data = "";
64
65                    res.on("data", (chunk) => { data += chunk; });
66
67                    res.on("end", () => {
68                        // Check for blocks or CAPTCHA
69                        if (res.statusCode >= 400) {
70                            reject(new Error(`HTTP ${res.statusCode}: ${data}`));
71                            return;
72                        }
73
74                        if (data.includes("<title>302 Moved</title>") || data.includes("google.com/sorry")) {
75                            reject(new Error("CAPTCHA detected or request blocked by Google."));
76                            return;
77                        }
78
79                        resolve({
80                            data,
81                            headers: res.headers, // ✅ Capturing response headers
82                        });
83                    });
84                });
85
86                req.on("error", reject);
87
88                if (postData) {
89                    req.write(postData);
90                }
91
92                req.end();
93            });
94
95            return response;
96
97        } catch (error) {
98            lastError = error;
99            console.error(`❌ Request failed (attempt ${retryCount + 1}/${maxRetries}): ${error.message}`);
100
101            // Only retry if it's a block or CAPTCHA
102            if (error.message.includes("CAPTCHA") ||
103                error.message.includes("sorry") ||
104                error.message.includes("302 Moved") ||
105                error.message.includes("429") ||
106                error.message.includes("403")) {
107
108                retryCount++;
109
110                // Add exponential backoff
111                const waitTime = 2000 * Math.pow(2, retryCount - 1);
112                console.log(`⏱️ Waiting ${waitTime / 1000} seconds before retry...`);
113                await new Promise(resolve => setTimeout(resolve, waitTime));
114
115            } else {
116                // For other errors, don't retry
117                throw error;
118            }
119        }
120    }
121
122    // If we've exhausted all retries
123    throw lastError || new Error('Maximum retries reached');
124}

1import { getDateRange } from './getDateRange.js';
2import { parseAdvertiserUrl } from './parseAdvertiserUrl.js';
3
4
5export const filterAdvertiser = async (kw, Use_URL_Filter, maxAdvertisers, maxDomains, RegionCode, format, platform, presetDate, startDate, endDate, limitResults,domainExists) => {
6    const parsedData = parseAdvertiserUrl(kw);
7    try {
8        const filterData = Use_URL_Filter
9            ? {
10                advertiserId_OR_kw: parsedData.advertiserId_domain,
11                maxDomains,
12                maxAdvertisers,
13                RegionCode: parsedData.RegionCode,
14                format: parseInt(parsedData.format),
15                dateRange: getDateRange(parsedData.presetDate, parsedData.startDate, parsedData.endDate),
16                platform: parseInt(parsedData.platform),
17                startDate: parsedData.startDate,
18                endDate: parsedData.endDate,
19                limitResults,
20                domainExists : parsedData.domainExists
21            }
22            : {
23                advertiserId_OR_kw: parsedData.advertiserId_domain,
24                maxDomains,
25                maxAdvertisers,
26                RegionCode,
27                format: parseInt(format),
28                platform: parseInt(platform),
29                dateRange: getDateRange(presetDate, startDate, endDate),
30                startDate,
31                endDate,
32                limitResults,
33                domainExists
34            };
35            console.log(Use_URL_Filter ? 'If Use_URL_Filter is true, override input filter:' : 'Use Actor.getInput() filtering:', filterData);
36            return {
37                advertiserId_OR_kw: parsedData.advertiserId_domain,
38                maxAdvertisers,
39                maxDomains,
40                RegionCode: Use_URL_Filter ? parsedData.RegionCode : RegionCode,
41                format: parseInt(Use_URL_Filter ? parsedData.format : format),
42                dateRange: getDateRange(
43                    Use_URL_Filter ? parsedData.presetDate : presetDate,
44                    Use_URL_Filter ? parsedData.startDate : startDate,
45                    Use_URL_Filter ? parsedData.endDate : endDate
46                ),
47                platform: parseInt(Use_URL_Filter ? parsedData.platform : platform),
48                limitResults,
49                domainExists: parsedData.domainExists
50            };
51    } catch (error) {
52        console.error(`❌ Error processing advertiser ID "${parsedData.advertiserId_domain}":`, error);
53    }
54};

1import dayjs from 'dayjs';
2
3function validateCustomDates(Start_Date, End_Date) {
4    const isValidDate = (date) => /^\d{4}-\d{2}-\d{2}$/.test(date);
5
6    let fromDate = isValidDate(Start_Date) ? parseInt(Start_Date.replace(/-/g, ""), 10) : null;
7    let toDate = isValidDate(End_Date) ? parseInt(End_Date.replace(/-/g, ""), 10) : null;
8
9    if (fromDate && toDate) {
10        if (fromDate > toDate) {
11            console.error("❌ Error: Start_Date cannot be after End_Date.");
12            return null; // Do not return error, let it fall back to default
13        }
14        return { fromDate, toDate, anyTime: false, dateRangeLabel: "Custom Range" };
15    }
16
17    if ((Start_Date && !isValidDate(Start_Date)) || (End_Date && !isValidDate(End_Date))) {
18        console.error("❌ Error: Invalid date format. Expected YYYY-MM-DD. Falling back to default values.");
19        return null; // Do not return error, let it fall back to default
20    }
21
22    return null;
23}
24
25export function getDateRange(days, Start_Date, End_Date) {
26    let customDateRange = validateCustomDates(Start_Date, End_Date);
27    if (customDateRange) return customDateRange;
28
29    let yesterday = parseInt(dayjs().subtract(1, "day").format("YYYYMMDD"), 10);
30    let defaultToDate = yesterday;
31    let defaultFromDate;
32    let anyTime = false;
33    let dateRangeLabel = "";
34
35    if (days === "0") {
36        defaultFromDate = parseInt(dayjs().subtract(2, "day").format("YYYYMMDD"), 10);
37        dateRangeLabel = "Today";
38    } else if (days === "1") {
39        defaultFromDate = parseInt(dayjs().subtract(2, "day").format("YYYYMMDD"), 10);
40        dateRangeLabel = "Yesterday";
41    } else if (days === "7") {
42        defaultFromDate = parseInt(dayjs().subtract(8, "day").format("YYYYMMDD"), 10);
43        dateRangeLabel = "Last 7 days";
44    } else if (days === "30") {
45        defaultFromDate = parseInt(dayjs().subtract(31, "day").format("YYYYMMDD"), 10);
46        dateRangeLabel = "Last 30 days";
47    } else if (days === "-1") {
48        dateRangeLabel = "Any time";
49        return { anyTime: true, dateRangeLabel };
50    } else {
51        return null;
52    }
53
54    return { fromDate: defaultFromDate, toDate: defaultToDate, anyTime, dateRangeLabel };
55}

1import https from "https";
2import { processAds } from "./processAds.js";  // Import function
3import { fetchWithProxyRotation } from "./fetchWithProxyRotation.js";  // Import function
4/**
5 * Function to get creative details by ID
6 * @param {string} creativeId - The creative ID to fetch details for
7 * @param {string} advertiserId - The advertiser ID associated with the creative
8 * @param {number} regionCode - The region code (default is worldwide: 2012)
9 * @returns {Promise<Object>} - The creative details
10 */
11export async function getCreativeDetails(creativeId, advertiserId, regionCode = 2012) {
12    const requestPayload = {
13        "1": advertiserId,
14        "2": creativeId,
15        "5": {
16            "1": 1, // Page number
17            "2": 30, // Page size
18            "3": 2012
19        }
20    };
21
22    const postData = new URLSearchParams({
23        "f.req": JSON.stringify(requestPayload)
24    }).toString();
25
26    const options = {
27        hostname: "adstransparency.google.com",
28        path: "/anji/_/rpc/LookupService/GetCreativeById?authuser=0",
29        method: "POST",
30        headers: {
31            "Content-Type": "application/x-www-form-urlencoded",
32            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
33            "Connection": "keep-alive",
34            "Content-Length": Buffer.byteLength(postData),
35        }
36    };
37
38    try {
39        // Use our new proxy rotation function
40        const response = await fetchWithProxyRotation(options, postData, 5); // Try up to 5 times
41
42        try {
43            const parsedResponse = JSON.parse(response.data);
44            console.log(`✅ Successfully fetched creative ID: ${creativeId}`);
45            return parsedResponse;
46        } catch (error) {
47            console.error("❌ JSON Parsing Error:", error);
48            throw error;
49        }
50
51    } catch (error) {
52        console.error(`❌ Failed to fetch creative ID ${creativeId} after multiple attempts:`, error.message);
53        throw error;
54    }
55}
56
57/**
58 * Function to process each ad as soon as it's fetched
59 * @param {Array<Object>} allResults - The array of results from searchForAds
60 * @param {number} regionCode - The region code (default is worldwide: 2012)
61 */
62
63async function sleep(ms) {
64    return new Promise(resolve => setTimeout(resolve, ms));
65}
66
67export async function processCreativeDetails(allResults, regionCode = 2012) {
68    for (const result of allResults) {
69        if (result["2"]) {
70            try {
71                const creativeDetail = await getCreativeDetails(result["2"], result["1"], regionCode);
72                console.log(`✅ Processed creative: ${result["2"]}`);
73
74                // Process each creative detail as soon as it's fetched
75                processAds([creativeDetail]);
76
77                // Optional delay to avoid detection (increase if needed)
78                //await sleep(300); // 3 seconds delay
79
80            } catch (error) {
81                console.error(`❌ Failed to process creative: ${result["2"]}`, error);
82            }
83        }
84    }
85}

1const startTime = performance.now();
2import { Actor } from 'apify';
3import { Dataset } from 'crawlee';
4import { getDateRange } from './getDateRange.js';
5import { parseAdvertiserUrl } from './parseAdvertiserUrl.js';
6import { searchForAdsList } from './search_for_an_ads.js';
7import { searchForPoliticalAdsList } from './search_for_an_political_ads.js';
8import { filterAdvertiser } from './filterAdvertiser.js';
9import fs from 'fs';
10
11await Actor.init();
12
13// Load and parse input parameters
14const input = await Actor.getInput() ?? {};
15const {
16    keyword_domain_advertiserid,
17    maxAdvertisers,
18    maxDomains,
19    region,
20    format,
21    platform,
22    presetDate,
23    startDate,
24    endDate,
25    Use_URL_Filter,
26    limitResults,
27    isPolitical
28} = input;
29
30// Load regions.json and resolve region code
31const regionMapping = JSON.parse(fs.readFileSync("src/regions.json", "utf-8"));
32const regionEntry = regionMapping.regions.region.find(r => r["4"] === region);
33const RegionCode = regionEntry ? regionEntry["1"] : undefined;
34
35// Determine date range
36const dateRange = getDateRange(presetDate, startDate, endDate);
37
38// Function to detect URLs
39const isGenericUrl = (kw) => /^https?:\/\//.test(kw);
40const advertiserIdPattern = (kw) => /^AR\d+$/.test(kw);
41
42const isAdvertiserUrl = /https:\/\/adstransparency\.google\.com\/advertiser\/(AR\d+)/;
43const isDomainUrl = /[?&]domain=([^&]+)/;
44const isTopic = /[?&]topic=([^&]+)/;
45
46let domainExists
47
48const processKeyword = async (kw) => {
49    try {
50        let filterData;
51
52        if (kw.match(isTopic) || isPolitical && !kw.match(isAdvertiserUrl) && !kw.match(isDomainUrl)) {
53            kw = kw.replace(/\s+/g, "+");  // Replace spaces with '+'
54            console.log(`✅✅ Political : ${kw}`);
55            const parsedData = parseAdvertiserUrl(kw);
56
57            const filterData =  {
58                adveritserId_OR_kw: parsedData.advertiserId_domain,
59                maxAdvertisers,
60                RegionCode: parsedData.RegionCode,
61                format: parseInt(parsedData.format),
62                platform: parseInt(parsedData.platform),
63                dateRange: getDateRange(parsedData.presetDate, parsedData.startDate, parsedData.endDate),
64                startDate: parsedData.startDate,
65                endDate: parsedData.endDate,
66                limitResults,
67            }
68            console.log("Filter Data:", JSON.stringify(filterData, null, 2));
69            let adsData = await searchForPoliticalAdsList(
70                parsedData.advertiserId_domain,
71                maxAdvertisers,
72                parsedData.RegionCode,
73                parseInt(parsedData.format),
74                getDateRange(parsedData.presetDate, parsedData.startDate, parsedData.endDate),
75                parseInt(parsedData.platform),
76                limitResults
77            );
78
79            await Dataset.pushData({ advertiserId_OR_kw: kw, adsData });
80
81        }
82        else if (kw.match(isAdvertiserUrl) || kw.match(isDomainUrl)) {
83            const advertiserMatch = kw.match(isAdvertiserUrl);
84            const domainMatch = kw.match(isDomainUrl);
85            const parsedData = parseAdvertiserUrl(kw);
86
87            if (advertiserMatch && domainMatch) {
88                console.error(`❌ Invalid URL: Cannot contain both advertiser ID and domain -> ${kw}`);
89                return; // Skip processing this invalid URL
90            }
91            if (advertiserMatch) {
92                // Advertiser ID detected
93                if (!parsedData) throw new Error("Invalid advertiser URL");
94                await filterAdvertiser(kw, Use_URL_Filter, maxAdvertisers, maxDomains, RegionCode, format, platform, presetDate, startDate, endDate, limitResults, domainExists);
95            } else if (domainMatch) {
96                // Domain detected
97                console.log('🔹 Processing Domain:', parsedData.advertiserId_domain,);
98                let Filter_Advertiser = await filterAdvertiser(kw, Use_URL_Filter, maxAdvertisers, maxDomains, RegionCode, format, platform, presetDate, startDate, endDate, limitResults, domainExists);
99                //console.log(`Filter_Advertiser ${JSON.stringify(Filter_Advertiser, null, 2)}`);
100                let adsData = await searchForAdsList(
101                    parsedData.advertiserId_domain,
102                    maxAdvertisers,
103                    maxDomains,
104                    Filter_Advertiser.RegionCode,
105                    Filter_Advertiser.format,
106                    Filter_Advertiser.dateRange,
107                    Filter_Advertiser.platform,
108                    limitResults,
109                    Filter_Advertiser.domainExists
110                );
111
112                //console.log(`adsData ${JSON.stringify(adsData, null, 2)}`);
113
114            }
115        }
116        else {
117            // Handle generic keywords (not URLs)
118            if (isGenericUrl(kw)) console.warn(`⚠️ Detected a generic URL: ${kw}.\nTreating as a regular keyword.`);
119            kw = kw.replace(/\s+/g, "+");  // Replace spaces with '+'
120            if (advertiserIdPattern(kw)) console.log(`✅ Advertiser ID: ${kw}`);
121            filterData = {
122                advertiserId_OR_kw: kw,
123                maxDomains,
124                maxAdvertisers,
125                RegionCode,
126                format: parseInt(format),
127                platform: parseInt(platform),
128                dateRange,
129                startDate,
130                endDate
131            };
132
133            let adsData = await searchForAdsList(
134                kw,
135                maxAdvertisers,
136                maxDomains,
137                RegionCode,
138                parseInt(format),
139                getDateRange(presetDate, startDate, endDate),
140                parseInt(platform),
141                limitResults
142            );
143            //await Dataset.pushData({ advertiserId_OR_kw0: kw, adsData });
144        }
145    } catch (error) {
146        console.error(`❌ Error processing "${kw}":`, error);
147    }
148};
149
150
151// Process all queries in parallel
152await Promise.all(keyword_domain_advertiserid.map(processKeyword));
153
154const endTime = performance.now();
155console.log(`⏳ Execution time: ${(endTime - startTime).toFixed(2)} ms`);
156await Actor.exit();