Expired Domains Scraper

1import { DomainParser } from '../parsers/domainParser.js';
2import { Domain } from '../types/domain.js';
3
4export class DomainBuilder {
5    domainParser: DomainParser;
6    domain: Domain = {} as Domain;
7
8    constructor(domainParser: DomainParser) {
9        this.domainParser = domainParser;
10    }
11
12    addName(): DomainBuilder {
13        this.domain.name = this.domainParser.name;
14        return this;
15    }
16
17    addLink(): DomainBuilder {
18        this.domain.link = this.domainParser.link;
19        return this;
20    }
21
22    addEnddate(): DomainBuilder {
23        this.domain.enddate = this.domainParser.enddate;
24        return this;
25    }
26
27    addTld(): DomainBuilder {
28        this.domain.tld = this.domainParser.tld;
29        return this;
30    }
31
32    addGodaddyPrice(): DomainBuilder {
33        this.domain.godaddyPrice = this.domainParser.godaddyPrice;
34        return this;
35    }
36
37    addGodaddyBid(): DomainBuilder {
38        this.domain.godaddyBid = this.domainParser.godaddyBid;
39        return this;
40    }
41
42    addAuctionType(): DomainBuilder {
43        this.domain.auctionType = this.domainParser.auctionType;
44        return this;
45    }
46
47    addAiPricePredPrice(): DomainBuilder {
48        this.domain.aiPricePredPrice = this.domainParser.aiPricePredPrice;
49        return this;
50    }
51
52    addRdapRegistration(): DomainBuilder {
53        this.domain.rdapRegistration = this.domainParser.rdapRegistration;
54        return this;
55    }
56
57    addWaybackFirstYear(): DomainBuilder {
58        this.domain.waybackFirstYear = this.domainParser.waybackFirstYear;
59        return this;
60    }
61
62    addWaybackCount(): DomainBuilder {
63        this.domain.waybackCount = this.domainParser.waybackCount;
64        return this;
65    }
66
67    addSearchVolumeVolume(): DomainBuilder {
68        this.domain.searchVolumeVolume = this.domainParser.searchVolumeVolume;
69        return this;
70    }
71
72    addSearchVolumeCpc(): DomainBuilder {
73        this.domain.searchVolumeCpc = this.domainParser.searchVolumeCpc;
74        return this;
75    }
76
77    addGoogleStatusIsBlocked(): DomainBuilder {
78        this.domain.googleStatusIsBlocked = this.domainParser.googleStatusIsBlocked;
79        return this;
80    }
81
82    addGoogleStatusIsFaillisted(): DomainBuilder {
83        this.domain.googleStatusIsFaillisted = this.domainParser.googleStatusIsFaillisted;
84        return this;
85    }
86
87    addGoogleStatusIsAdult(): DomainBuilder {
88        this.domain.googleStatusIsAdult = this.domainParser.googleStatusIsAdult;
89        return this;
90    }
91
92    addFbShares(): DomainBuilder {
93        this.domain.fbShares = this.domainParser.fbShares;
94        return this;
95    }
96
97    addFbReact(): DomainBuilder {
98        this.domain.fbReact = this.domainParser.fbReact;
99        return this;
100    }
101
102    addMajesticExtbacklinks(): DomainBuilder {
103        this.domain.majesticExtbacklinks = this.domainParser.majesticExtbacklinks;
104        return this;
105    }
106
107    addMajesticCitationflow(): DomainBuilder {
108        this.domain.majesticCitationflow = this.domainParser.majesticCitationflow;
109        return this;
110    }
111
112    addMajesticTrustflow(): DomainBuilder {
113        this.domain.majesticTrustflow = this.domainParser.majesticTrustflow;
114        return this;
115    }
116
117    addMozDomainAuthority(): DomainBuilder {
118        this.domain.mozDomainAuthority = this.domainParser.mozDomainAuthority;
119        return this;
120    }
121
122    addMozPageAuthority(): DomainBuilder {
123        this.domain.mozPageAuthority = this.domainParser.mozPageAuthority;
124        return this;
125    }
126
127    addOwnranksBrandrank(): DomainBuilder {
128        this.domain.ownranksBrandrank = this.domainParser.ownranksBrandrank;
129        return this;
130    }
131
132    addOwnranksTldsaturation(): DomainBuilder {
133        this.domain.ownranksTldsaturation = this.domainParser.ownranksTldsaturation;
134        return this;
135    }
136
137    addOwnranksRadiotest(): DomainBuilder {
138        this.domain.ownranksRadiotest = this.domainParser.ownranksRadiotest;
139        return this;
140    }
141
142    build(): Domain {
143        return this.domain;
144    }
145}

1import { log, RequestOptions } from 'crawlee';
2import { config } from '../config.js';
3import { Input } from '../types/input.js';
4
5export class RequestBuilder {
6    private filters: string;
7    private sort: string;
8    private length: string;
9    private page: number;
10    private id: number;
11    private input: Input;
12
13    private includedColumns: string[] = [
14        'link',
15        'enddate',
16        'tld',
17        'godaddy_price',
18        'godaddy_bid',
19        'auction_type',
20        'ai_price_pred_price',
21        'rdap_registration',
22        'wayback_first_year',
23        'wayback_count',
24        'search_volume_volume',
25        'search_volume_cpc',
26        'google_status_is_blocked',
27        'google_status_is_faillisted',
28        'google_status_is_adult',
29        'fb_shares',
30        'fb_react',
31        'moz_domain_authority',
32        'moz_page_authority',
33        'majestic_citationflow',
34        'majestic_trustflow',
35        'majestic_extbacklinks',
36        'ownranks_brandrank',
37        'ownranks_tldsaturation',
38        'ownranks_radiotest',
39    ];
40
41    constructor(input: Input, page?: number) {
42        this.filters = '';
43        this.sort = '';
44        this.length = '';
45        this.page = page ?? 0;
46        this.id = 0;
47        this.input = input;
48    }
49
50    addTldFilters(): RequestBuilder {
51        if (this.input.filterTldCom) this.filters += 'com,';
52        if (this.input.filterTldNet) this.filters += 'net,';
53        if (this.input.filterTldOrg) this.filters += 'org,';
54        if (this.input.filterTldIo) this.filters += 'io,';
55        if (this.input.filterTldUk) this.filters += 'uk,';
56        if (this.input.filterTldN) this.filters += 'n,';
57        if (this.input.filterTldOther) this.filters += 'other,';
58
59        return this;
60    }
61
62    addLengthFilters(): RequestBuilder {
63        if (this.input.filterLengthMin > this.input.filterLengthMax) throw new Error('Min length cannot be greater than max length');
64        this.length += `min[domain_length]=${this.input.filterLengthMin}&max[domain_length]=${this.input.filterLengthMax}`;
65
66        return this;
67    }
68
69    addCharasterFilters(): RequestBuilder {
70        if (this.input.filterCharsHyp) this.filters += 'no-dashes,';
71        if (this.input.filterCharsNum) this.filters += 'no-numbers,';
72        if (this.input.filterCharsLet) this.filters += 'no-letters,';
73
74        return this;
75    }
76
77    addGoogleFilters(): RequestBuilder {
78        if (this.input.filterGglNA) this.filters += 'not-adult,';
79        if (this.input.filterGglNB) this.filters += 'not-faillisted-or-blocked,';
80        if (this.input.filterGglA) this.filters += 'Adult,';
81        if (this.input.filterGglB) this.filters += 'faillisted-or-blocked,';
82
83        return this;
84    }
85
86    addAuctionFilters(): RequestBuilder {
87        if (this.input.filterAucTypAuc) this.filters += 'godaddy-expiring,';
88        if (this.input.filterAucTypClo) this.filters += 'godaddy-closeouts,';
89        if (this.input.filterAucTypDel) this.filters += 'pending-delete,';
90
91        return this;
92    }
93
94    addExpDateFilter(): RequestBuilder {
95        const dateObj = new Date(this.input.filterExpDate);
96        const todayObj = new Date();
97        todayObj.setHours(0, 0, 0, 0);
98
99        const diffTime = dateObj.getTime() - todayObj.getTime();
100
101        let diffDays = Math.floor(diffTime / (1000 * 60 * 60 * 24));
102
103        if (diffTime < 0) {
104            log.warning('Selected date is in the past, setting it to today.');
105            diffDays = 0;
106        }
107
108        switch (diffDays) {
109            case 0:
110                this.filters += 'today,';
111                break;
112            case 1:
113                this.filters += 'tomorrow,';
114                break;
115            case 2:
116                this.filters += 'today2,';
117                break;
118            case 3:
119                this.filters += 'today3,';
120                break;
121            case 4:
122                this.filters += 'today4,';
123                break;
124            default:
125                log.warning('Date is outside of allowed range of 4 days from today. Setting it to today.');
126                this.filters += 'today,';
127                break;
128        }
129
130        return this;
131    }
132
133    addSort(): RequestBuilder {
134        this.sort = `${this.input.sortDir === 'asc' ? '' : '-'}${this.input.sortBy}`;
135
136        return this;
137    }
138
139    build(): RequestOptions {
140        const columns = this.includedColumns.map((col) => `columns[]=${col}`).join('&');
141        const payload = `filters=${this.filters}&sort=${this.sort}&page=${this.page}&${columns}&id=${this.id}`;
142
143        return {
144            url: config.fetchUrl,
145            method: 'POST',
146            uniqueKey: `page-${this.page}`,
147            headers: {
148                'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
149                origin: config.originUrl,
150            },
151            payload,
152        };
153    }
154}

1import { RequestOptions } from 'crawlee';
2import { RequestBuilder } from '../builders/requestBuilder.js';
3import { config } from '../config.js';
4import { Input } from '../types/input.js';
5
6export async function generateRequests(input: Input): Promise<RequestOptions[]> {
7    const requests: RequestOptions[] = [];
8
9    const pages = Math.ceil(input.numDomains / config.domainsPerPage);
10
11    for (let i = 0; i < pages; i++) {
12        const requestBuilder = new RequestBuilder(input, i);
13
14        const request = requestBuilder
15            .addTldFilters()
16            .addLengthFilters()
17            .addCharasterFilters()
18            .addGoogleFilters()
19            .addAuctionFilters()
20            .addExpDateFilter()
21            .addSort()
22            .build();
23
24        requests.push(request);
25    }
26
27    return requests;
28}
29
30export function convertToNumber(str: string) {
31    if (str.startsWith('$')) {
32        str = str.slice(1);
33    }
34
35    let number = parseFloat(str);
36
37    if (str.toLowerCase().endsWith('k')) {
38        number *= 1000;
39    }
40
41    if (str.toLowerCase().endsWith('m')) {
42        number *= 1000000;
43    }
44
45    return number;
46}

1import { Cheerio, Element } from 'cheerio';
2import { config } from '../config.js';
3import { convertToNumber } from '../helpers/helper.js';
4
5export class DomainParser {
6    element: Cheerio<Element>;
7
8    constructor(element: Cheerio<Element>) {
9        this.element = element;
10    }
11
12    get name(): string {
13        return this.element.find('td:nth-child(1)').text();
14    }
15
16    get link(): string | undefined {
17        const ref = this.element.find('td:nth-child(2) > a').attr('href');
18        return ref ? `${config.originUrl}${ref}` : undefined;
19    }
20
21    get enddate(): Date {
22        return new Date(this.element.find('td:nth-child(3)').text());
23    }
24
25    get tld(): string {
26        return this.element.find('td:nth-child(4)').text();
27    }
28
29    get godaddyPrice(): number | undefined {
30        const price = this.element.find('td:nth-child(5)').text();
31        return price !== '' ? convertToNumber(price) : undefined;
32    }
33
34    get godaddyBid(): number | undefined {
35        const bid = this.element.find('td:nth-child(6)').text();
36        return bid !== '' ? convertToNumber(bid) : undefined;
37    }
38
39    get auctionType(): string | undefined {
40        const type = this.element.find('td:nth-child(7)').text();
41        return type !== '' ? type : undefined;
42    }
43
44    get aiPricePredPrice(): number | undefined {
45        const price = this.element.find('td:nth-child(8)').text();
46        return price !== '' ? convertToNumber(price) : undefined;
47    }
48
49    get rdapRegistration(): number | undefined {
50        const year = this.element.find('td:nth-child(9)').text();
51        return year !== '' ? convertToNumber(year) : undefined;
52    }
53
54    get waybackFirstYear(): number | undefined {
55        const year = this.element.find('td:nth-child(10)').text();
56        return year !== '' ? convertToNumber(year) : undefined;
57    }
58
59    get waybackCount(): number | undefined {
60        const count = this.element.find('td:nth-child(11)').text();
61        return count !== '' ? convertToNumber(count) : undefined;
62    }
63
64    get searchVolumeVolume(): number | undefined {
65        const volume = this.element.find('td:nth-child(12)').text();
66        return volume !== '' ? convertToNumber(volume) : undefined;
67    }
68
69    get searchVolumeCpc(): number | undefined {
70        const cpc = this.element.find('td:nth-child(13)').text();
71        return cpc !== '' ? convertToNumber(cpc) : undefined;
72    }
73
74    get googleStatusIsBlocked(): boolean {
75        const status = this.element.find('td:nth-child(14)').text();
76        return status === '1';
77    }
78
79    get googleStatusIsFaillisted(): boolean {
80        const status = this.element.find('td:nth-child(15)').text();
81        return status === '1';
82    }
83
84    get googleStatusIsAdult(): boolean {
85        const status = this.element.find('td:nth-child(16)').text();
86        return status === '1';
87    }
88
89    get fbShares(): number | undefined {
90        const shares = this.element.find('td:nth-child(17)').text();
91        return shares !== '' ? convertToNumber(shares) : undefined;
92    }
93
94    get fbReact(): number | undefined {
95        const react = this.element.find('td:nth-child(18)').text();
96        return react !== '' ? convertToNumber(react) : undefined;
97    }
98
99    get mozDomainAuthority(): number | undefined {
100        const authority = this.element.find('td:nth-child(19)').text();
101        return authority !== '' ? convertToNumber(authority) : undefined;
102    }
103
104    get mozPageAuthority(): number | undefined {
105        const authority = this.element.find('td:nth-child(20)').text();
106        return authority !== '' ? convertToNumber(authority) : undefined;
107    }
108
109    get majesticCitationflow(): number | undefined {
110        const flow = this.element.find('td:nth-child(21)').text();
111        return flow !== '' ? convertToNumber(flow) : undefined;
112    }
113
114    get majesticTrustflow(): number | undefined {
115        const flow = this.element.find('td:nth-child(22)').text();
116        return flow !== '' ? convertToNumber(flow) : undefined;
117    }
118
119    get majesticExtbacklinks(): number | undefined {
120        const links = this.element.find('td:nth-child(23)').text();
121        return links !== '' ? convertToNumber(links) : undefined;
122    }
123
124    get ownranksBrandrank(): number | undefined {
125        const rank = this.element.find('td:nth-child(24)').text();
126        return rank !== '' ? convertToNumber(rank) : undefined;
127    }
128
129    get ownranksTldsaturation(): number | undefined {
130        const saturation = this.element.find('td:nth-child(25)').text();
131        return saturation !== '' ? convertToNumber(saturation) : undefined;
132    }
133
134    get ownranksRadiotest(): number | undefined {
135        const test = this.element.find('td:nth-child(26)').text();
136        return test !== '' ? convertToNumber(test) : undefined;
137    }
138}