1
2
3
4
5
6
7
8
9import { Actor, log } from 'apify';
10
11const USER_AGENT = 'foundry-actor/0.1 (Apify Actor; polite client; aviationweather.gov data)';
12const MIN_INTERVAL_MS = 1000;
13const MAX_STATIONS = 100;
14
15
16
17const MAX_PIREP_STATIONS = 25;
18let lastRequestAt = 0;
19
20async function politeFetch(url, attempt = 1) {
21 const wait = lastRequestAt + MIN_INTERVAL_MS - Date.now();
22 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
23 lastRequestAt = Date.now();
24 const res = await fetch(url, { headers: { 'user-agent': USER_AGENT, accept: 'application/json, text/plain' } });
25 if (res.status === 429 || res.status >= 500) {
26 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
27 await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
28 return politeFetch(url, attempt + 1);
29 }
30 return res;
31}
32
33async function politeGetJson(url) {
34 const res = await politeFetch(url);
35
36
37 if (res.status === 204) return [];
38 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
39 const text = await res.text();
40 return text ? JSON.parse(text) : [];
41}
42
43async function politeGetText(url) {
44 const res = await politeFetch(url);
45 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
46 return res.text();
47}
48
49
50
51function remainingBudget() {
52 try {
53 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
54 return Number.isFinite(n) ? n : Infinity;
55 } catch {
56 return Infinity;
57 }
58}
59
60function toIso(unixSeconds) {
61 return Number.isFinite(unixSeconds) ? new Date(unixSeconds * 1000).toISOString() : null;
62}
63
64
65
66function baseRecord() {
67 return {
68 icaoId: null,
69 stationName: null,
70 reportType: null,
71 issuedAt: null,
72 validFrom: null,
73 validTo: null,
74 rawText: null,
75 tempC: null,
76 dewpointC: null,
77 windDirDeg: null,
78 windSpeedKt: null,
79 windGustKt: null,
80 visibilitySm: null,
81 altimeterHpa: null,
82 flightCategory: null,
83 clouds: null,
84 forecastPeriods: null,
85 latitude: null,
86 longitude: null,
87 elevationM: null,
88 aircraftType: null,
89 flightLevelFt: null,
90 flightPhase: null,
91 turbulence: null,
92 icing: null,
93 hazardType: null,
94 qualifier: null,
95 altitudeBaseFt: null,
96 altitudeTopFt: null,
97 movementDir: null,
98 movementSpeedKt: null,
99 firId: null,
100 advisoryId: null,
101 subType: null,
102 forecastHour: null,
103 area: null,
104 windsAloft: null,
105 };
106}
107
108function mapMetar(r) {
109 return {
110 ...baseRecord(),
111 icaoId: r.icaoId ?? null,
112 stationName: r.name ?? null,
113 reportType: 'metar',
114 issuedAt: r.reportTime ?? null,
115 rawText: r.rawOb ?? null,
116 tempC: r.temp ?? null,
117 dewpointC: r.dewp ?? null,
118 windDirDeg: r.wdir ?? null,
119 windSpeedKt: r.wspd ?? null,
120 windGustKt: r.wgst ?? null,
121 visibilitySm: r.visib ?? null,
122 altimeterHpa: r.altim ?? null,
123 flightCategory: r.fltCat ?? null,
124 clouds: r.clouds ?? null,
125 latitude: r.lat ?? null,
126 longitude: r.lon ?? null,
127 elevationM: r.elev ?? null,
128 };
129}
130
131function mapTaf(r) {
132 return {
133 ...baseRecord(),
134 icaoId: r.icaoId ?? null,
135 stationName: r.name ?? null,
136 reportType: 'taf',
137 issuedAt: r.issueTime ?? null,
138 validFrom: toIso(r.validTimeFrom),
139 validTo: toIso(r.validTimeTo),
140 rawText: r.rawTAF ?? null,
141 forecastPeriods: (r.fcsts ?? []).map((f) => ({
142 from: toIso(f.timeFrom),
143 to: toIso(f.timeTo),
144 windDirDeg: f.wdir ?? null,
145 windSpeedKt: f.wspd ?? null,
146 windGustKt: f.wgst ?? null,
147 visibilitySm: f.visib ?? null,
148 altimeterHpa: f.altim ?? null,
149 weather: f.wxString ?? null,
150 clouds: f.clouds ?? null,
151 })),
152 latitude: r.lat ?? null,
153 longitude: r.lon ?? null,
154 elevationM: r.elev ?? null,
155 };
156}
157
158
159
160function pirepLayers(r, prefix, keys) {
161 const layers = [];
162 for (const n of [1, 2]) {
163 const [baseK, topK, intK, typeK, freqK] = keys.map((k) => `${prefix}${k}${n}`);
164 const intensity = r[intK];
165 if (!intensity) continue;
166 const layer = { base: r[baseK] ?? null, top: r[topK] ?? null, intensity, type: r[typeK] || null };
167 if (freqK) layer.frequency = r[freqK] || null;
168 layers.push(layer);
169 }
170 return layers.length ? layers : null;
171}
172
173function mapPirep(r) {
174 return {
175 ...baseRecord(),
176 icaoId: r.icaoId ?? null,
177 reportType: 'pirep',
178 issuedAt: toIso(r.obsTime),
179 rawText: r.rawOb ?? null,
180 tempC: r.temp ?? null,
181 windDirDeg: r.wdir ?? null,
182 windSpeedKt: r.wspd ?? null,
183 visibilitySm: r.visib ?? null,
184 clouds: r.clouds ?? null,
185 latitude: r.lat ?? null,
186 longitude: r.lon ?? null,
187 aircraftType: r.acType ?? null,
188 flightLevelFt: r.fltLvl != null && Number.isFinite(Number(r.fltLvl)) ? Number(r.fltLvl) * 100 : null,
189 flightPhase: r.fltLvlType ?? null,
190 turbulence: pirepLayers(r, 'tb', ['Bas', 'Top', 'Int', 'Type', 'Freq']),
191 icing: pirepLayers(r, 'icg', ['Bas', 'Top', 'Int', 'Type']),
192 qualifier: r.wxString || null,
193 };
194}
195
196
197
198
199function mapSigmet(r) {
200 return {
201 ...baseRecord(),
202 icaoId: r.icaoId ?? null,
203 stationName: r.firName ?? null,
204 reportType: 'sigmet',
205 issuedAt: r.issueTime ?? null,
206 validFrom: toIso(r.validTimeFrom),
207 validTo: toIso(r.validTimeTo),
208 rawText: r.rawSigmet ?? null,
209 hazardType: r.hazard ?? null,
210 qualifier: r.qualifier ?? null,
211 altitudeBaseFt: r.base ?? null,
212 altitudeTopFt: r.top ?? null,
213 movementDir: r.dir ?? null,
214 movementSpeedKt: r.spd != null && Number.isFinite(Number(r.spd)) ? Number(r.spd) : null,
215 firId: r.firId ?? null,
216 advisoryId: r.seriesId ?? null,
217 subType: r.sigmetType ?? null,
218 area: r.geom ? { geom: r.geom, coords: r.coords ?? [] } : null,
219 };
220}
221
222function mapGairmet(r) {
223 return {
224 ...baseRecord(),
225 reportType: 'gairmet',
226 issuedAt: toIso(r.issueTime),
227 validFrom: r.validTime ?? null,
228 validTo: toIso(r.expireTime),
229 hazardType: r.hazard ?? null,
230 qualifier: r.severity ?? r.due_to ?? null,
231 altitudeBaseFt: r.base ?? null,
232 altitudeTopFt: r.top ?? null,
233 advisoryId: r.tag ?? null,
234 subType: r.product ?? null,
235 forecastHour: r.forecastHour ?? null,
236 area: r.geom ? { geom: r.geom, coords: r.coords ?? [] } : null,
237 };
238}
239
240
241
242
243
244
245
246function decodeWindGroup(code4) {
247 if (code4 === '9900') return { windDirDeg: null, windSpeedKt: null };
248 const dirRaw = Number(code4.slice(0, 2));
249 let speedKt = Number(code4.slice(2, 4));
250 let windDirDeg = dirRaw * 10;
251 if (dirRaw > 36) {
252 windDirDeg = (dirRaw - 50) * 10;
253 speedKt += 100;
254 }
255 return { windDirDeg, windSpeedKt: speedKt };
256}
257
258
259
260
261
262
263
264
265function decodeWindTempRow(line, altitudesFt) {
266 const icaoId = line.slice(0, 3).trim();
267 if (!icaoId) return null;
268 let pos = 4;
269 const windsAloft = [];
270 for (const flightLevelFt of altitudesFt) {
271 const width = flightLevelFt <= 3000 ? 4 : flightLevelFt <= 24000 ? 7 : 6;
272 const field = line.slice(pos, pos + width);
273 pos += width + 1;
274 if (!field.trim()) continue;
275 if (width === 4) {
276 const { windDirDeg, windSpeedKt } = decodeWindGroup(field);
277 windsAloft.push({ flightLevelFt, windDirDeg, windSpeedKt, tempC: null });
278 } else if (width === 7) {
279 const { windDirDeg, windSpeedKt } = decodeWindGroup(field.slice(0, 4));
280 const sign = field.slice(4, 5);
281 const tempDigits = field.slice(5, 7);
282 windsAloft.push({ flightLevelFt, windDirDeg, windSpeedKt, tempC: sign === '-' ? -Number(tempDigits) : Number(tempDigits) });
283 } else {
284 const { windDirDeg, windSpeedKt } = decodeWindGroup(field.slice(0, 4));
285 const tempDigits = field.slice(4, 6);
286 windsAloft.push({ flightLevelFt, windDirDeg, windSpeedKt, tempC: -Number(tempDigits) });
287 }
288 }
289 return { icaoId, rawText: line.trimEnd(), windsAloft: windsAloft.length ? windsAloft : null };
290}
291
292
293
294
295
296function ddhhmmToDate(ddhhmm, now) {
297 const day = Number(ddhhmm.slice(0, 2));
298 const hour = Number(ddhhmm.slice(2, 4));
299 const minute = Number(ddhhmm.slice(4, 6));
300 const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), day, hour, minute));
301 const FIFTEEN_DAYS_MS = 15 * 24 * 60 * 60 * 1000;
302 if (d.getTime() - now.getTime() > FIFTEEN_DAYS_MS) d.setUTCMonth(d.getUTCMonth() - 1);
303 else if (now.getTime() - d.getTime() > FIFTEEN_DAYS_MS) d.setUTCMonth(d.getUTCMonth() + 1);
304 return d;
305}
306
307
308
309
310
311
312function windowBound(validDate, hhmm, isEnd) {
313 const hour = Number(hhmm.slice(0, 2));
314 const minute = Number(hhmm.slice(2, 4));
315 const validMinuteOfDay = validDate.getUTCHours() * 60 + validDate.getUTCMinutes();
316 const boundMinuteOfDay = hour * 60 + minute;
317 const d = new Date(validDate);
318 d.setUTCHours(hour, minute, 0, 0);
319 if (isEnd && boundMinuteOfDay < validMinuteOfDay) d.setUTCDate(d.getUTCDate() + 1);
320 if (!isEnd && boundMinuteOfDay > validMinuteOfDay) d.setUTCDate(d.getUTCDate() - 1);
321 return d;
322}
323
324
325
326
327function parseWindTemp(text) {
328 const lines = text.split('\n');
329 const ftLineIndex = lines.findIndex((l) => l.startsWith('FT'));
330 const basedOnMatch = text.match(/DATA BASED ON (\d{6})Z/);
331 const validMatch = text.match(/VALID (\d{6})Z\s+FOR USE (\d{4})-(\d{4})Z/);
332 if (ftLineIndex === -1 || !validMatch) return { issuedAt: null, validFrom: null, validTo: null, stations: [] };
333
334 const altitudesFt = lines[ftLineIndex].trim().split(/\s+/).slice(1).map(Number);
335 const now = new Date();
336 const issuedAt = basedOnMatch ? ddhhmmToDate(basedOnMatch[1], now).toISOString() : null;
337 const validDate = ddhhmmToDate(validMatch[1], now);
338 const validFrom = windowBound(validDate, validMatch[2], false).toISOString();
339 const validTo = windowBound(validDate, validMatch[3], true).toISOString();
340
341 const stations = [];
342 for (let i = ftLineIndex + 1; i < lines.length; i++) {
343 const line = lines[i];
344 if (!line.trim()) break;
345 const row = decodeWindTempRow(line, altitudesFt);
346 if (row) stations.push(row);
347 }
348 return { issuedAt, validFrom, validTo, stations };
349}
350
351function mapWindsAloft(station, product) {
352 return {
353 ...baseRecord(),
354 icaoId: station.icaoId,
355 reportType: 'windsaloft',
356 issuedAt: product.issuedAt,
357 validFrom: product.validFrom,
358 validTo: product.validTo,
359 rawText: station.rawText,
360 subType: product.region,
361 qualifier: product.level,
362 forecastHour: product.fcst,
363 windsAloft: station.windsAloft,
364 };
365}
366
367await Actor.init();
368
369const input = (await Actor.getInput()) ?? {};
370const rawStations = String(input.stations ?? 'KJFK,KLAX,KORD').trim();
371const includeTaf = input.includeTaf ?? true;
372const hoursBack = Number(input.hoursBack ?? 0);
373const includePireps = input.includePireps ?? false;
374const pirepRadiusSm = Number(input.pirepRadiusSm ?? 200);
375const pirepHoursBack = Number(input.pirepHoursBack ?? 3);
376const includeHazards = input.includeHazards ?? false;
377const includeWindsAloft = input.includeWindsAloft ?? false;
378const windsAloftRegion = String(input.windsAloftRegion ?? 'us');
379const windsAloftLevel = String(input.windsAloftLevel ?? 'low');
380const windsAloftFcst = String(input.windsAloftFcst ?? '06');
381const WINDS_ALOFT_REGIONS = ['us', 'bos', 'mia', 'chi', 'dfw', 'slc', 'sfo', 'alaska', 'hawaii', 'other_pac'];
382if (!WINDS_ALOFT_REGIONS.includes(windsAloftRegion)) {
383 throw new Error(`Input "windsAloftRegion" must be one of: ${WINDS_ALOFT_REGIONS.join(', ')}.`);
384}
385if (!['low', 'high'].includes(windsAloftLevel)) throw new Error('Input "windsAloftLevel" must be "low" or "high".');
386if (!['06', '12', '24'].includes(windsAloftFcst)) throw new Error('Input "windsAloftFcst" must be "06", "12" or "24".');
387if (!Number.isInteger(hoursBack) || hoursBack < 0 || hoursBack > 168) {
388 throw new Error('Input "hoursBack" must be a whole number from 0 (latest report only) to 168 (7 days).');
389}
390if (!Number.isFinite(pirepRadiusSm) || pirepRadiusSm < 10 || pirepRadiusSm > 500) {
391 throw new Error('Input "pirepRadiusSm" must be a number from 10 to 500.');
392}
393if (!Number.isInteger(pirepHoursBack) || pirepHoursBack < 1 || pirepHoursBack > 24) {
394 throw new Error('Input "pirepHoursBack" must be a whole number from 1 to 24.');
395}
396if (!rawStations) throw new Error('Input "stations" must not be empty.');
397
398const stations = [...new Set(
399 rawStations.split(/[,\s]+/).map((s) => s.trim().toUpperCase()).filter(Boolean),
400)].slice(0, MAX_STATIONS);
401
402const invalid = stations.filter((s) => !/^[A-Z0-9]{3,4}$/.test(s));
403if (invalid.length) throw new Error(`Invalid ICAO/station code(s): ${invalid.join(', ')}`);
404if (stations.length === 0) throw new Error('Input "stations" must contain at least one ICAO code.');
405
406const ids = stations.join(',');
407let metarRecords = [];
408let tafRecords = [];
409let pirepRecords = [];
410let hazardRecords = [];
411
412
413const hoursParam = hoursBack > 0 ? `&hours=${hoursBack}` : '';
414const metarData = await politeGetJson(`https://aviationweather.gov/api/data/metar?ids=${encodeURIComponent(ids)}&format=json${hoursParam}`);
415metarRecords = metarData.map(mapMetar);
416
417if (includeTaf) {
418 const tafData = await politeGetJson(`https://aviationweather.gov/api/data/taf?ids=${encodeURIComponent(ids)}&format=json`);
419 tafRecords = tafData.map(mapTaf);
420}
421
422if (includePireps) {
423
424
425 const pirepStations = stations.slice(0, MAX_PIREP_STATIONS);
426 if (stations.length > MAX_PIREP_STATIONS) {
427 log.info(`PIREPs are limited to the first ${MAX_PIREP_STATIONS} stations; skipping the rest.`);
428 }
429 for (const station of pirepStations) {
430 const url = `https://aviationweather.gov/api/data/pirep?format=json&id=${encodeURIComponent(station)}&distance=${pirepRadiusSm}&age=${pirepHoursBack}`;
431 const data = await politeGetJson(url);
432 pirepRecords = pirepRecords.concat(data.map(mapPirep));
433 }
434}
435
436if (includeHazards) {
437
438
439
440 const sigmetData = await politeGetJson('https://aviationweather.gov/api/data/sigmet?format=json');
441 const gairmetData = await politeGetJson('https://aviationweather.gov/api/data/gairmet?format=json');
442 hazardRecords = sigmetData.map(mapSigmet).concat(gairmetData.map(mapGairmet));
443}
444
445let windsAloftRecords = [];
446if (includeWindsAloft) {
447
448
449 const url = `https://aviationweather.gov/api/data/windtemp?region=${windsAloftRegion}&level=${windsAloftLevel}&fcst=${windsAloftFcst}`;
450 const text = await politeGetText(url);
451 const product = parseWindTemp(text);
452 windsAloftRecords = product.stations.map((s) => mapWindsAloft(s, { ...product, region: windsAloftRegion, level: windsAloftLevel, fcst: windsAloftFcst }));
453}
454
455
456
457
458const records = [...tafRecords, ...metarRecords, ...pirepRecords, ...hazardRecords, ...windsAloftRecords];
459if (records.length === 0) log.info(`No weather data found for: ${ids}`);
460
461const budget = remainingBudget();
462const batch = records.slice(0, budget);
463let saved = 0;
464if (batch.length > 0) {
465 await Actor.pushData(batch);
466 saved = batch.length;
467}
468if (batch.length < records.length) log.info('Reached the maximum charge set for this run; stopping.');
469
470log.info(`Saved ${saved} items for stations: ${ids}`);
471await Actor.exit();
