Cars & Bids Scraper

1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10node_modules
11
12# git folder
13.git

1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf

1{
2    "extends": "@apify",
3    "root": true
4}

1# This file tells Git which files shouldn't be added to source control
2
3.DS_Store
4.idea
5dist
6node_modules
7apify_storage
8storage
9storage

1{
2	"name": "carsNbids",
3	"version": "0.0.1",
4	"type": "module",
5	"description": "This is an example of an Apify actor.",
6	"dependencies": {
7		"apify": "^3.0.0",
8		"crawlee": "^3.0.0",
9		"playwright": "*"
10	},
11	"devDependencies": {
12		"@apify/eslint-config": "^0.3.1",
13		"eslint": "^8.20.0"
14	},
15	"scripts": {
16		"start": "node src/main.js",
17		"lint": "eslint ./src --ext .js,.jsx",
18		"lint:fix": "eslint ./src --ext .js,.jsx --fix",
19		"test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
20	},
21	"author": "It's not you it's me",
22	"license": "ISC"
23}

1# Specify the base Docker image. You can read more about
2# the available images at https://crawlee.dev/docs/guides/docker-images
3# You can also use any other image from Docker Hub.
4FROM apify/actor-node-playwright-chrome:16
5
6# Copy just package.json and package-lock.json
7# to speed up the build using Docker layer cache.
8COPY --chown=myuser package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14    && npm install --omit=dev --omit=optional \
15    && echo "Installed NPM packages:" \
16    && (npm list --omit=dev --all || true) \
17    && echo "Node.js version:" \
18    && node --version \
19    && echo "NPM version:" \
20    && npm --version \
21    && rm -r ~/.npm
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after NPM install, quick build will be really fast
25# for most source file changes.
26COPY --chown=myuser . ./
27
28
29# Run the image. If you know you won't need headful browsers,
30# you can remove the XVFB start script for a micro perf gain.
31CMD ./start_xvfb_and_run_cmd.sh && npm start --silent

1{
2    "title": "cars&bids inputs",
3    "description": "Inputs",
4    "type": "object",
5    "schemaVersion": 1,
6    "properties": {
7        "startYear": {
8            "title": "start_year",
9            "type": "integer",
10            "description": "Select the start year of the search",
11            "default": 1981,
12            "maximum": 2023,
13            "minimum": 1981
14        },
15        "endYear": {
16            "title": "end_year",
17            "type": "integer",
18            "description": "Select the end year of the search",
19            "default": 2023,
20            "maximum": 2023,
21            "minimum": 1981
22        },
23        "transmission": {
24            "title": "transmission",
25            "type": "string",
26            "description": "Select transmission type",
27            "editor": "select",
28            "default": "All",
29            "enum": [
30                "All",
31                "1",
32                "2"
33            ],
34            "enumTitles": [
35                "All",
36                "Automatic",
37                "Manual"
38            ]
39        },
40        "bodyStyles": {
41            "title": "Body style",
42            "type": "string",
43            "description": "Select the body style",
44            "editor": "select",
45            "default": "All",
46            "enum": [
47                "All",
48                "1",
49                "2",
50                "3",
51                "4",
52                "5",
53                "6",
54                "7",
55                "8"
56            ],
57            "enumTitles": [
58                "All",
59                "Coupe",
60                "Convertible",
61                "Hatchback",
62                "Sedan",
63                "SUV/Crossover",
64                "Truck",
65                "Van/Minivan",
66                "Wagon"
67            ]
68        },
69        "sort": {
70            "title": "Sort",
71            "type": "string",
72            "description": "Select sort type",
73            "editor": "select",
74            "default": "Ending soon",
75            "enum": [
76                "All",
77                "Ending soon",
78                "listed",
79                "no_reserve",
80                "lowest_mileage"
81            ],
82            "enumTitles": [
83                "All",
84                "Ending soon",
85                "Newly listed",
86                "No reserve",
87                "Lowest mileage"
88            ]
89        },
90        "maxItems": {
91            "title": "maxItems",
92            "type": "integer",
93            "description": "the number of items you want to scrap",
94            "nullable": true
95        },
96        "pastAuctions": {
97            "title": "Past Auctions",
98            "type": "boolean",
99            "description": "get past auctions results.",
100            "default": false
101        }
102    },
103    "required": []
104}

1{
2	"actorSpecification": 1,
3	"name": "carsNbids",
4	"version": "0.0",
5	"buildTag": "latest",
6	"environmentVariables": {},
7	"storages": {
8        "dataset": "./dataset_schema.json"
9    }
10}

1{
2    "actorSpecification": 1,
3    "views": {
4        "overview": {
5            "title": "Overview",
6            "transformation": {
7                "fields": [
8                    "title",
9                    "url",
10                    "ending",
11                    "bidValue",
12                    "timeLeft",
13                    "info",
14                    "images"
15                ]
16            },
17            "display": {
18                "component": "table",
19                "properties": {
20                    "title": {
21                        "label": "Text",
22                        "format": "text"
23                    },
24                    "url": {
25                        "label": "Link",
26                        "format": "link"
27                    },
28                    "ending": {
29                        "label": "Text",
30                        "format": "text"
31                    },
32                    "bidValue": {
33                        "label": "Number",
34                        "format": "number"
35                    },
36                    "timeLeft": {
37                        "label": "Text",
38                        "format": "text"
39                    },
40                    "info.Make": {
41                        "label": "Text",
42                        "format": "text"
43                    },
44                    "info.Model": {
45                        "label": "Text",
46                        "format": "text"
47                    },
48                    "info.Mileage": {
49                        "label": "Number",
50                        "format": "number"
51                    },
52                    "info.VIN": {
53                        "label": "Text",
54                        "format": "text"
55                    },
56                    "info.Title Status": {
57                        "label": "Text",
58                        "format": "text"
59                    },
60                    "info.Location": {
61                        "label": "Text",
62                        "format": "text"
63                    },
64                    "info.Seller": {
65                        "label": "Text",
66                        "format": "text"
67                    },
68                    "info.Engine": {
69                        "label": "Text",
70                        "format": "text"
71                    },
72                    "info.Drivetrain": {
73                        "label": "Text",
74                        "format": "text"
75                    },
76                    "info.Transmission": {
77                        "label": "Text",
78                        "format": "text"
79                    },
80                    "info.Body Style": {
81                        "label": "Text",
82                        "format": "text"
83                    },
84                    "info.Exterior Color": {
85                        "label": "Text",
86                        "format": "text"
87                    },
88                    "info.Interior Color": {
89                        "label": "Text",
90                        "format": "text"
91                    },
92                    "info.Seller Type": {
93                        "label": "Text",
94                        "format": "text"
95                    },
96                    "images": {
97                        "label": "Array",
98                        "format": "array"
99                    }
100                }
101            }
102        }
103    }
104}