Carlendar2.0 Datagen

# Dockerfile contains instructions how to build a Docker image that
# will contain all the code and configuration needed to run your actor.
# For a full Dockerfile reference,
# see https://docs.docker.com/engine/reference/builder/

# First, specify the base Docker image. Apify provides the following
# base images for your convenience:
#  apify/actor-node-basic (Node.js 10 on Alpine Linux, small and fast)
#  apify/actor-node-chrome (Node.js 10 + Chrome on Debian)
#  apify/actor-node-chrome-xvfb (Node.js 10 + Chrome + Xvfb on Debian)
# For more information, see https://docs.apify.com/actor/build#base-images
# Note that you can use any other image from Docker Hub.
FROM apify/actor-node-basic

# Second, copy just package.json since it should be the only file
# that affects NPM install in the next step
COPY package.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && npm list || true \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
# CMD npm start

{
    "title": "My input schema",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "myField": {
            "title": "My input field",
            "type": "string",
            "nullable": false,
            "description": "This is a customizable description providing help to the users of your actor.",
            "editor": "textarea"
        }
    }
}

1// This is the main Node.js source code file of your actor.
2// It is referenced from the "scripts" section of the package.json file.
3
4const Apify = require('apify');
5Apify.main(async () => {
6    // Get input of the actor. Input fields can be modified in INPUT_SCHEMA.json file.
7    // For more information, see https://docs.apify.com/actor/input-schema
8    const input = await Apify.getInput();
9    console.log('Input:');
10    console.dir(input);
11
12
13
14    // Here you can prepare your input for actor apify/legacy-phantomjs-crawler this input is based on a actor
15    // task you used as the starting point.
16    const metamorphInput = {
17        "startUrls": [
18            {
19                "key": "START",
20                "value": "http://example.com"
21            }
22        ],
23        "crawlPurls": [
24            {
25                "key": "",
26                "value": "https://apps.carleton.edu/campus/registrar/schedule/enroll/?term=21FA&subject=[\\D+]"
27            }
28        ],
29
30        "clickableElementsSelector": null,
31        "pageFunction": function pageFunction(context) {
32            var $ = context.jQuery;
33            context.customData
34
35            if (context.request.label === "START") {
36                var SPREADSHEET_ID = "1nMnWyWLexxv4MZg7YOTS2YJT4c44RFGF4nThhfLelyY";
37                var NUMBER_OF_SHEETS = 1;
38                const majors = ["AFST",
39"AMMU",
40"AMST",
41"ARBC",
42"ARCN",
43"ARTH",
44"ASST",
45"ASTR",
46"BIOL",
47"CHEM",
48"CHIN",
49"CAMS",
50"CLAS",
51"CGSC",
52"CS",
53"CCST",
54"DANC",
55"ECON",
56"EDUC",
57"ENGL",
58"ENTS",
59"EUST",
60"FREN",
61"GEOL",
62"GERM",
63"GRK",
64"HIST",
65"IDSC",
66"JAPN",
67"LATN",
68"LTAM",
69"LING",
70"LCST",
71"MATH",
72"MARS",
73"MEST",
74"MELA",
75"MUSC",
76"NEUR",
77"PHIL",
78"PE",
79"PHYS",
80"POSC",
81"PSYC",
82"RELG",
83"RUSS",
84"SOAN",
85"SPAN",
86"ARTS",
87"THEA",
88"WGST"]
89                var loadData = function(id) {
90                    var urlAPI = "https://spreadsheets.google.com/feeds/list/" + SPREADSHEET_ID + "/" + id + "/public/values?alt=json";
91                    $.get(urlAPI, function(data) {
92                        var entries = data.feed.entry;
93                        $.each(entries, function(index, value) {
94                            var url = value.title.$t;
95                            context.enqueuePage(url);
96                        });
97                        if (id === NUMBER_OF_SHEETS) {
98                            context.finish();
99                        } else {
100                            loadData(id + 1);
101                        }
102                    });
103                };
104                loadData(1);
105                context.skipOutput();
106                context.willFinishLater();
107            } else {
108                var results = [];
109                $(".course").each(function() {
110
111                     
112                        if (( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).length ==3) {
113                        
114                                                x_m = null;
115        
116                        }
117                        else {x_m = ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text())
118                            
119                            
120                        }
121                    
122                    results.push({
123                    value: results.length,
124                    // CourseNum: $(this).find(".coursenum").text(),
125                                id: $(this).find(".coursenum").text(),
126                                                                key: $(this).find(".coursenum").text(),
127
128                                            // name: $(this).find(".coursenum").text(),
129        
130                    text: $(this).find(".title").text().replace($(this).find(".coursenum").text(),"").replace($(this).find(".credits").text(),"").trim(),
131        
132                    credits: $(this).find(".credits").text(),
133                    // Details: $(this).find(".title").text().replace($(this).find(".coursenum").text(),"").replace($(this).find(".credits").text(),"").trim(),
134                    location: $(this).find(".data").find(".schedule").find(".locations").text(),
135                    instructor: $(this).find(".faculty").text().trim(),
136                    // description: $(this).find(".description").find("p:nth-child(2)").text(),
137                    // Days: $(this).find(".data").find(".schedule").find(".schedule").find("thead").find(".used").text(),
138                    // start: $(this).find(".data").find(".schedule").find(".schedule").find("td:nth-child(1)").text(), 
139                    // end: $(this).find(".data").find(".schedule").find(".schedule").find("tbody").find(".used").text(), "ADD END"
140             mo: ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).length ==3 ? null : ( $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(1)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
141        
142        tu: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(2)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"), 
143        we: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(3)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
144        th: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(4)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
145        fr: ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(1)").find("td:nth-child(5)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
146        mo_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(1)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
147        tu_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(2)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
148        we_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(3)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
149        th_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(4)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
150        fr_lab: ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".end").text()).length == 3 ? null : ($(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".start").text() + " - " + $(this).find(".data").find(".schedule").find("tr:nth-child(2)").find("td:nth-child(5)").find(".end").text()).replace(/am/g, " am").replace(/pm/g, " pm"),
151        
152                        });
153                     
154                    });
155                return results;
156            }
157        },
158        "interceptRequest": function interceptRequest(context, newRequest) {
159            // called whenever the crawler finds a link to a new page,
160            // use it to override default behavior
161            return newRequest;
162        },
163        "considerUrlFragment": false,
164        "loadImages": true,
165        "loadCss": true,
166        "injectJQuery": true,
167        "injectUnderscoreJs": false,
168        "ignoreRobotsTxt": false,
169        "skipLoadingFrames": false,
170        "verboseLog": true,
171        "disableWebSecurity": false,
172        "rotateUserAgents": false,
173        "maxCrawledPages": null,
174        "maxOutputPages": null,
175        "maxCrawlDepth": null,
176        "resourceTimeout": null,
177        "pageLoadTimeout": null,
178        "pageFunctionTimeout": null,
179        "maxInfiniteScrollHeight": null,
180        "randomWaitBetweenRequests": null,
181        "maxCrawledPagesPerSlave": null,
182        "customHttpHeaders": null,
183        "customData": null,
184        "cookies": null,
185        "cookiesPersistence": "PER_PROCESS",
186        "finishWebhookUrl": null,
187        "finishWebhookData": null,
188        "maxParallelRequests": 1,
189        "proxyConfiguration": {
190            "useApifyProxy": false
191        }
192    };
193
194    // Now let's metamorph into actor apify/legacy-phantomjs-crawler using the created input.
195    await Apify.metamorph('apify/legacy-phantomjs-crawler', metamorphInput);
196});

{
    "name": "my-actor",
    "version": "0.0.1",
    "dependencies": {
        "apify": "^0.17.0"
    },
    "scripts": {
        "start": "node main.js"
    },
    "author": "Me!"
}