1import { request } from './http.js';
2import { htmlToText, decodeEntities, toIso } from './normalize.js';
3
4
5
6
7
8
9
10const list = (v) => (Array.isArray(v) ? v : v ? String(v).split(',') : []).map((s) => String(s).trim()).filter(Boolean);
11const num = (v) => (v === null || v === undefined || v === '' || Number(v) === 0 ? null : Number(v));
12
13function job(fields) {
14 return {
15 id: null, source: null, title: null, companyName: null, companyLogo: null, url: null, applyUrl: null,
16 locations: [], isWorldwide: false, employmentType: null, category: null, tags: [], seniority: null,
17 salaryMin: null, salaryMax: null, salaryCurrency: null, salaryPeriod: null, salaryText: null,
18 postedAt: null, descriptionHtml: null, descriptionText: null,
19 ...fields,
20 };
21}
22
23const WORLDWIDE = /\b(worldwide|anywhere|global|remote)\b/i;
24
25
26export const himalayas = {
27 name: 'himalayas',
28 title: 'Himalayas',
29 async fetch({ keywords, maxItems, cutoff }) {
30 const out = [];
31 const map = (j) => job({
32 id: `himalayas:${j.guid}`,
33 source: 'himalayas',
34 title: j.title,
35 companyName: j.companyName,
36 companyLogo: j.companyLogo || null,
37 url: j.applicationLink || j.guid,
38 applyUrl: j.applicationLink || j.guid,
39 locations: j.locationRestrictions?.length ? j.locationRestrictions : ['Worldwide'],
40 isWorldwide: !j.locationRestrictions?.length,
41 employmentType: j.employmentType || null,
42 category: (j.parentCategories?.[0] || j.categories?.[0] || '').replace(/-/g, ' ') || null,
43 tags: (j.categories || []).map((c) => c.replace(/-/g, ' ')),
44 seniority: j.seniority?.join(', ') || null,
45 salaryMin: num(j.minSalary), salaryMax: num(j.maxSalary), salaryCurrency: j.currency || null,
46 salaryPeriod: j.salaryPeriod || null,
47 postedAt: toIso(j.pubDate),
48 descriptionHtml: j.description || null,
49 });
50 const queries = keywords.length ? keywords : [null];
51 const perQuery = Math.ceil(maxItems / queries.length);
52 for (const q of queries) {
53 let got = 0;
54 let cursor = null;
55 for (let page = 1; page <= 250 && got < perQuery; page++) {
56 const url = q
57 ? `https://himalayas.app/jobs/api/search?q=${encodeURIComponent(q)}&limit=20&page=${page}`
58 : `https://himalayas.app/jobs/api?limit=20${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`;
59 const d = await request(url);
60 const jobs = d.jobs || [];
61 if (!jobs.length) break;
62 let tooOld = false;
63 for (const j of jobs) {
64 const m = map(j);
65 if (cutoff && m.postedAt && new Date(m.postedAt).getTime() < cutoff) { tooOld = true; continue; }
66 out.push(m);
67 got++;
68 }
69
70 if (tooOld && !q) break;
71 cursor = d.nextCursor;
72 if (!q && !cursor) break;
73 }
74 }
75 return out;
76 },
77};
78
79
80export const remoteok = {
81 name: 'remoteok',
82 title: 'RemoteOK',
83 async fetch() {
84 const d = await request('https://remoteok.com/api');
85 return d.filter((j) => j.id && j.position).map((j) => job({
86 id: `remoteok:${j.id}`,
87 source: 'remoteok',
88 title: decodeEntities(j.position),
89 companyName: decodeEntities(j.company?.trim()) || null,
90 companyLogo: j.company_logo || j.logo || null,
91 url: j.url,
92 applyUrl: j.apply_url || j.url,
93 locations: j.location ? [j.location] : ['Worldwide'],
94 isWorldwide: !j.location || WORLDWIDE.test(j.location),
95 tags: j.tags || [],
96 category: j.tags?.[0] || null,
97 salaryMin: num(j.salary_min), salaryMax: num(j.salary_max), salaryCurrency: num(j.salary_min) ? 'USD' : null,
98 salaryPeriod: num(j.salary_min) ? 'yearly' : null,
99 postedAt: toIso(j.date || Number(j.epoch)),
100 descriptionHtml: j.description || null,
101 }));
102 },
103};
104
105
106export const remotive = {
107 name: 'remotive',
108 title: 'Remotive',
109 async fetch({ keywords }) {
110
111 const urls = keywords.length
112 ? keywords.map((k) => `https://remotive.com/api/remote-jobs?search=${encodeURIComponent(k)}`)
113 : ['https://remotive.com/api/remote-jobs'];
114 const out = [];
115 for (const u of urls) {
116 const d = await request(u);
117 for (const j of d.jobs || []) {
118 out.push(job({
119 id: `remotive:${j.id}`,
120 source: 'remotive',
121 title: j.title,
122 companyName: j.company_name,
123 companyLogo: j.company_logo_url || j.company_logo || null,
124 url: j.url,
125 applyUrl: j.url,
126 locations: list(j.candidate_required_location || 'Worldwide'),
127 isWorldwide: WORLDWIDE.test(j.candidate_required_location || 'worldwide'),
128 employmentType: j.job_type ? j.job_type.replace(/_/g, ' ') : null,
129 category: j.category || null,
130 tags: j.tags || [],
131 salaryText: j.salary || null,
132 postedAt: toIso(j.publication_date ? `${j.publication_date}Z` : null),
133 descriptionHtml: j.description || null,
134 }));
135 }
136 }
137 return out;
138 },
139};
140
141
142export const jobicy = {
143 name: 'jobicy',
144 title: 'Jobicy',
145 async fetch({ keywords }) {
146 const urls = keywords.length
147 ? keywords.map((k) => `https://jobicy.com/api/v2/remote-jobs?count=100&tag=${encodeURIComponent(k)}`)
148 : ['https://jobicy.com/api/v2/remote-jobs?count=100'];
149 const out = [];
150 for (const u of urls) {
151 const d = await request(u);
152 for (const j of d.jobs || []) {
153 out.push(job({
154 id: `jobicy:${j.id}`,
155 source: 'jobicy',
156 title: decodeEntities(j.jobTitle),
157 companyName: decodeEntities(j.companyName),
158 companyLogo: j.companyLogo || null,
159 url: j.url,
160 applyUrl: j.url,
161 locations: list(j.jobGeo || 'Anywhere'),
162 isWorldwide: WORLDWIDE.test(j.jobGeo || 'anywhere'),
163 employmentType: list(j.jobType).join(', ') || null,
164 category: list(j.jobIndustry)[0] ? decodeEntities(list(j.jobIndustry)[0]) : null,
165 tags: list(j.jobIndustry).map(decodeEntities),
166 seniority: j.jobLevel || null,
167 salaryMin: num(j.salaryMin), salaryMax: num(j.salaryMax), salaryCurrency: j.salaryCurrency || null,
168 salaryPeriod: j.salaryPeriod || null,
169 postedAt: toIso(j.pubDate),
170 descriptionHtml: j.jobDescription || null,
171 }));
172 }
173 }
174 return out;
175 },
176};
177
178
179const rssTag = (item, tag) => {
180 const m = item.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`));
181 if (!m) return null;
182 return decodeEntities(m[1].replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, '$1').trim()) || null;
183};
184export const weworkremotely = {
185 name: 'weworkremotely',
186 title: 'We Work Remotely',
187 async fetch() {
188 const xml = await request('https://weworkremotely.com/remote-jobs.rss', { as: 'text' });
189 const items = xml.match(/<item>[\s\S]*?<\/item>/g) || [];
190 return items.map((it) => {
191 const full = rssTag(it, 'title') || '';
192 const idx = full.indexOf(': ');
193 const company = idx > 0 ? full.slice(0, idx) : null;
194 const title = idx > 0 ? full.slice(idx + 2) : full;
195 const link = rssTag(it, 'link') || rssTag(it, 'guid');
196 const region = rssTag(it, 'region');
197 const html = rssTag(it, 'description');
198 return job({
199 id: `weworkremotely:${(link || full).replace(/^https?:\/\/[^/]+/, '')}`,
200 source: 'weworkremotely',
201 title,
202 companyName: company,
203 companyLogo: it.match(/<media:content url="([^"]+)"/)?.[1] || null,
204 url: link,
205 applyUrl: link,
206 locations: list(region || rssTag(it, 'country') || 'Anywhere'),
207 isWorldwide: WORLDWIDE.test(region || 'anywhere') || /anywhere in the world/i.test(region || ''),
208 employmentType: rssTag(it, 'type'),
209 category: rssTag(it, 'category'),
210 tags: list(rssTag(it, 'skills')),
211 postedAt: toIso(rssTag(it, 'pubDate')),
212 descriptionHtml: html ? html.replace(/^<img[^>]*>\s*/, '') : null,
213 });
214 });
215 },
216};
217
218
219export const workingnomads = {
220 name: 'workingnomads',
221 title: 'Working Nomads',
222 async fetch() {
223 const d = await request('https://www.workingnomads.com/api/exposed_jobs/');
224 return (d || []).map((j) => job({
225 id: `workingnomads:${j.url}`,
226 source: 'workingnomads',
227 title: j.title,
228 companyName: j.company_name || null,
229 url: j.url,
230 applyUrl: j.url,
231 locations: list(j.location || 'Anywhere'),
232 isWorldwide: WORLDWIDE.test(j.location || 'anywhere'),
233 category: j.category_name || null,
234 tags: list(j.tags),
235 postedAt: toIso(j.pub_date),
236 descriptionHtml: j.description || null,
237 }));
238 },
239};
240
241
242export const arbeitnow = {
243 name: 'arbeitnow',
244 title: 'Arbeitnow',
245 async fetch({ maxItems, cutoff }) {
246 const out = [];
247 for (let page = 1; page <= 10 && out.length < maxItems; page++) {
248 const d = await request(`https://www.arbeitnow.com/api/job-board-api?page=${page}`);
249 const jobs = d.data || [];
250 if (!jobs.length) break;
251 let tooOld = false;
252 for (const j of jobs) {
253 if (!j.remote) continue;
254 const postedAt = toIso(j.created_at);
255 if (cutoff && postedAt && new Date(postedAt).getTime() < cutoff) { tooOld = true; continue; }
256 out.push(job({
257 id: `arbeitnow:${j.slug}`,
258 source: 'arbeitnow',
259 title: j.title,
260 companyName: j.company_name || null,
261 url: j.url,
262 applyUrl: j.url,
263 locations: list(j.location || 'Europe'),
264 employmentType: (j.job_types || []).join(', ') || null,
265 category: j.tags?.[0] || null,
266 tags: j.tags || [],
267 postedAt,
268 descriptionHtml: j.description || null,
269 }));
270 }
271 if (tooOld || !d.links?.next) break;
272 }
273 return out;
274 },
275};
276
277export const SOURCES = { himalayas, remoteok, remotive, jobicy, weworkremotely, workingnomads, arbeitnow };
278
279export function finalize(j, includeDescription) {
280 const out = { ...j };
281 if (includeDescription) out.descriptionText = htmlToText(j.descriptionHtml);
282 else { out.descriptionHtml = null; out.descriptionText = null; }
283 out.scrapedAt = new Date().toISOString();
284 return out;
285}
