1
2
3
4const Apify = require('apify');
5Apify.main(async () => {
6
7
8 const input = await Apify.getInput();
9 console.log('Input:');
10 console.dir(input);
11
12
13
14
15
16 const metamorphInput = {
17 "startUrls": [
18 {
19 "key": "START",
20 "value": "http://example.com"
21 }
22 ],
23 "crawlPurls": [
24 {
25 "key": "",
26 "value": "https://apps.carleton.edu/campus/registrar/schedule/enroll/?term=21FA&subject=[\\D+]"
27 }
28 ],
29
30 "clickableElementsSelector": null,
31 "pageFunction": function pageFunction(context) {
32 var $ = context.jQuery;
33 context.customData
34
35 if (context.request.label === "START") {
36 var SPREADSHEET_ID = "1nMnWyWLexxv4MZg7YOTS2YJT4c44RFGF4nThhfLelyY";
37 var NUMBER_OF_SHEETS = 1;
38 const majors = ["AFST",
39"AMMU",
40"AMST",
41"ARBC",
42"ARCN",
43"ARTH",
44"ASST",
45"ASTR",
46"BIOL",
47"CHEM",
48"CHIN",
49"CAMS",
50"CLAS",
51"CGSC",
52"CS",
53"CCST",
54"DANC",
55"ECON",
56"EDUC",
57"ENGL",
58"ENTS",
59"EUST",
60"FREN",
61"GEOL",
62"GERM",
63"GRK",
64"HIST",
65"IDSC",
66"JAPN",
67"LATN",
68"LTAM",
69"LING",
70"LCST",
71"MATH",
72"MARS",
73"MEST",
74"MELA",
75"MUSC",
76"NEUR",
77"PHIL",
78"PE",
79"PHYS",
80"POSC",
81"PSYC",
82"RELG",
83"RUSS",
84"SOAN",
85"SPAN",
86"ARTS",
87"THEA",
88"WGST"]
89 var loadData = function(id) {
90 var urlAPI = "https://spreadsheets.google.com/feeds/list/" + SPREADSHEET_ID + "/" + id + "/public/values?alt=json";
91 $.get(urlAPI, function(data) {
92 var entries = data.feed.entry;
93 $.each(entries, function(index, value) {
94 var url = value.title.$t;
95 context.enqueuePage(url);
96 });
97 if (id === NUMBER_OF_SHEETS) {
98 context.finish();
99 } else {
100 loadData(id + 1);
101 }
102 });
103 };
104 loadData(1);
105 context.skipOutput();
106 context.willFinishLater();
107 } else {
108 var results = [];
109 $(".course").each(function() {
110
111
112 if (( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).length ==3) {
113
114 x_m = null;
115
116 }
117 else {x_m = ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text())
118
119
120 }
121
122 results.push({
123 value: results.length,
124
125 id: $(this).find(".coursenum").text(),
126 key: $(this).find(".coursenum").text(),
127
128
129
130 text: $(this).find(".title").text().replace($(this).find(".coursenum").text(),"").replace($(this).find(".credits").text(),"").trim(),
131
132 credits: $(this).find(".credits").text(),
133
134 location: $(this).find(".data").find(".schedule").find(".locations").text(),
135 instructor: $(this).find(".faculty").text().trim(),
136
137
138
139
140 mo: ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).length ==3 ? null : ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
141
142 tu: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
143 we: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
144 th: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
145 fr: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
146 mo_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
147 tu_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
148 we_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
149 th_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
150 fr_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
151
152 });
153
154 });
155 return results;
156 }
157 },
158 "interceptRequest": function interceptRequest(context, newRequest) {
159
160
161 return newRequest;
162 },
163 "considerUrlFragment": false,
164 "loadImages": true,
165 "loadCss": true,
166 "injectJQuery": true,
167 "injectUnderscoreJs": false,
168 "ignoreRobotsTxt": false,
169 "skipLoadingFrames": false,
170 "verboseLog": true,
171 "disableWebSecurity": false,
172 "rotateUserAgents": false,
173 "maxCrawledPages": null,
174 "maxOutputPages": null,
175 "maxCrawlDepth": null,
176 "resourceTimeout": null,
177 "pageLoadTimeout": null,
178 "pageFunctionTimeout": null,
179 "maxInfiniteScrollHeight": null,
180 "randomWaitBetweenRequests": null,
181 "maxCrawledPagesPerSlave": null,
182 "customHttpHeaders": null,
183 "customData": null,
184 "cookies": null,
185 "cookiesPersistence": "PER_PROCESS",
186 "finishWebhookUrl": null,
187 "finishWebhookData": null,
188 "maxParallelRequests": 1,
189 "proxyConfiguration": {
190 "useApifyProxy": false
191 }
192 };
193
194
195 await Apify.metamorph('apify/legacy-phantomjs-crawler', metamorphInput);
196});