1import { Actor } from 'apify';
2import { Dataset, RequestOptions, createCheerioRouter, log } from 'crawlee';
3
4export enum Label {
5 advertiserSearch = 'advertiserSearch',
6 search = 'search',
7 detail = 'detail',
8}
9
10export const router = createCheerioRouter();
11const ADS_PAGE_SIZE = 12;
12const ADVERTISERS_PAGE_SIZE = 12;
13const DEFAULT_MAX_ITEMS = 10;
14const REQUEST_TIMEOUT_MILLIS = 20_000;
15
16const getState = async () =>
17 Actor.useState<{ enqueuedItems: number }>('BING_ADS_STATE', {
18 enqueuedItems: 0,
19 });
20
21class HttpStatusError extends Error {
22 constructor(public statusCode: number) {
23 super(`Request failed with status ${statusCode}`);
24 }
25}
26
27const parseJsonBody = (body: string) => {
28 try {
29 return JSON.parse(body);
30 } catch {
31 const preview = body.slice(0, 200).replace(/\s+/g, ' ').trim();
32 throw new Error(`Expected JSON response, got: ${preview}`);
33 }
34};
35
36const requestJson = async (sendRequest: any, url: string) => {
37 const response = await sendRequest({
38 url,
39 http2: false,
40 timeout: { request: REQUEST_TIMEOUT_MILLIS },
41 });
42
43 if (response.statusCode >= 400) {
44 throw new HttpStatusError(response.statusCode);
45 }
46
47 return parseJsonBody(response.body);
48};
49
50router.addDefaultHandler(async ({ crawler }) => {
51 const input = await Actor.getInput<any>();
52 const maxPages = Math.max(1, Number(input?.maxPages ?? 1) || 1);
53 const maxItems = Math.max(
54 1,
55 Number(input?.maxItems ?? DEFAULT_MAX_ITEMS) || DEFAULT_MAX_ITEMS
56 );
57 const query = typeof input?.query === 'string' ? input.query.trim() : '';
58 const advertiserId =
59 typeof input?.advertiserId === 'string' ? input.advertiserId.trim() : '';
60
61 if (!query && !advertiserId) {
62 throw new Error('Either "query" or "advertiserId" must be provided.');
63 }
64
65 log.info('Default handler', {
66 query: query || null,
67 advertiserId: advertiserId || null,
68 maxPages,
69 maxItems,
70 });
71
72 if (advertiserId) {
73 await crawler.addRequests([
74 {
75 url: `https://adlibrary.ads.microsoft.com/search/${advertiserId}/1`,
76 label: Label.search,
77 userData: {
78 advertiserId,
79 query,
80 maxPages,
81 maxItems,
82 page: 1,
83 },
84 skipNavigation: true,
85 },
86 ]);
87
88 return;
89 }
90
91 await crawler.addRequests(
92 Array.from({ length: maxPages }).map((_, i) => {
93 const url = new URL(
94 'https://adlibrary.api.bingads.microsoft.com/api/v1/Advertisers'
95 );
96 url.searchParams.set('searchText', query);
97 url.searchParams.set('top', ADVERTISERS_PAGE_SIZE.toString());
98 url.searchParams.set('skip', (i * ADVERTISERS_PAGE_SIZE).toString());
99
100 return {
101 url: `https://adlibrary.ads.microsoft.com/advertisers/${i}`,
102 label: Label.advertiserSearch,
103 userData: {
104 query,
105 page: i + 1,
106 url: url.toString(),
107 maxPages,
108 maxItems,
109 },
110 skipNavigation: true,
111 };
112 })
113 );
114});
115
116router.addHandler(
117 Label.advertiserSearch,
118 async ({ request, crawler, sendRequest }) => {
119 const { query, page, url, maxPages, maxItems } = request.userData;
120 const state = await getState();
121
122 if (state.enqueuedItems >= maxItems) {
123 log.info('Reached maxItems, skipping advertiser search', { page });
124 return;
125 }
126
127 log.info('Advertiser search handler', {
128 page,
129 query,
130 maxPages,
131 });
132
133 const res: any = await requestJson(sendRequest, url.toString());
134
135 const advertisers = Array.isArray(res?.value) ? res.value : [];
136
137 log.info('Advertiser search result', {
138 page,
139 items: advertisers.length,
140 });
141
142 await crawler.addRequests(
143 advertisers.flatMap((advertiser: any) => {
144 const advertiserId =
145 advertiser?.Id ?? advertiser?.AdvertiserId ?? advertiser?.id;
146
147 if (advertiserId == null) {
148 return [];
149 }
150
151 return {
152 url: `https://adlibrary.ads.microsoft.com/search/${advertiserId}/1`,
153 label: Label.search,
154 skipNavigation: true,
155 userData: {
156 advertiserId,
157 maxPages,
158 maxItems,
159 page: 1,
160 },
161 };
162 })
163 );
164 }
165);
166
167router.addHandler(Label.search, async ({ request, crawler, sendRequest }) => {
168 const { advertiserId, query, maxPages, maxItems, page } = request.userData;
169 const state = await getState();
170
171 if (state.enqueuedItems >= maxItems) {
172 log.info('Reached maxItems, skipping search page', { page });
173 return;
174 }
175
176 const apiUrl = new URL(
177 'https://adlibrary.api.bingads.microsoft.com/api/v1/Ads'
178 );
179 apiUrl.searchParams.set('advertiserId', String(advertiserId));
180 apiUrl.searchParams.set('top', ADS_PAGE_SIZE.toString());
181 apiUrl.searchParams.set('skip', ((page - 1) * ADS_PAGE_SIZE).toString());
182
183 if (query) {
184 apiUrl.searchParams.set('searchText', String(query));
185 }
186
187 const res: any = await requestJson(sendRequest, apiUrl.toString());
188 const ads = Array.isArray(res?.value) ? res.value : [];
189 const adsToScrape = ads.slice(0, Math.max(maxItems - state.enqueuedItems, 0));
190 state.enqueuedItems += adsToScrape.length;
191
192 log.info('Search handler', {
193 page,
194 items: ads.length,
195 });
196
197 const requests: RequestOptions[] = adsToScrape.map((v: any) => {
198 const detailUrl = `https://adlibrary.api.bingads.microsoft.com/api/v1/Ads/${v.AdId}?expand=AdDetails(expand=ImpressionsByCountry,Targets)`;
199
200 return {
201 url: `https://adlibrary.ads.microsoft.com/detail/${v.AdId}`,
202 label: Label.detail,
203 skipNavigation: true,
204 forefront: true,
205 userData: {
206 adId: v.AdId,
207 url: detailUrl.toString(),
208 },
209 };
210 });
211
212 if (
213 ads.length === ADS_PAGE_SIZE &&
214 page < maxPages &&
215 state.enqueuedItems < maxItems
216 ) {
217 requests.push({
218 url: `https://adlibrary.ads.microsoft.com/search/${advertiserId}/${page + 1}`,
219 label: Label.search,
220 skipNavigation: true,
221 userData: {
222 advertiserId,
223 query,
224 maxPages,
225 maxItems,
226 page: page + 1,
227 },
228 });
229 }
230
231 await crawler.addRequests(requests);
232});
233
234router.addHandler(Label.detail, async ({ request, sendRequest }) => {
235 const { adId, url } = request.userData;
236
237 log.info('Detail handler', {
238 adId,
239 });
240
241 let res: any;
242 try {
243 res = await requestJson(sendRequest, url.toString());
244 } catch (error) {
245
246 if (
247 error instanceof HttpStatusError &&
248 error.statusCode >= 500 &&
249 request.retryCount >= 1
250 ) {
251 log.warning('Skipping ad after repeated server errors', {
252 adId,
253 statusCode: error.statusCode,
254 });
255 return;
256 }
257 throw error;
258 }
259
260 const ad = {
261 adId: res.AdId,
262 adTitle: res.Title,
263 adDescription: res.Description,
264 adDestination: res.DestinationUrl,
265 advertiserId: res.AdvertiserId,
266 advertiserName: res.AdvertiserName,
267 advertiserPaidForBy: res.AdDetails.PaidForByName,
268 adStartDate: res.AdDetails.StartDate,
269 adEndDate: res.AdDetails.EndDate,
270 adImpressions: res.AdDetails.TotalImpressionsRange,
271 impressionsByCountry: (res.AdDetails.ImpressionsByCountry ?? [])
272 .map((v) => ({
273 country: v.Country,
274 impressionShare: Number(v.ImpressionShare.replace('%', '')),
275 }))
276 .sort((a, b) => b.impressionShare - a.impressionShare),
277 targets: (res.AdDetails.Targets ?? []).map((v) => ({
278 targetType: v.TargetType,
279 usedForExclusion: v.UsedForExclusion,
280 })),
281 };
282
283 await Dataset.pushData(ad);
284});