Upwork Job Scraper

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf

{
    "extends": "@apify",
    "root": true
}

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

{
    "searchQuery": "javascript",
    "page": 1,
    "useApifyProxy": false,
    "maxConcurrency": 1,
    "maxRetries": 3,
    "useEnhancedCrawler": true,
    "minDelayBetweenRequests": 5,
    "maxDelayBetweenRequests": 10,
    "sessionRotationCount": 2,
    "useCloudScraper": true,
    "useFallbackHtmlParser": true
}

{
    "name": "crawlee-puppeteer-javascript",
    "version": "0.0.1",
    "type": "module",
    "description": "This is an example of a Crawlee project.",
    "dependencies": {
        "2captcha": "^3.0.7",
        "apify": "^3.1.4",
        "axios": "^1.6.2",
        "crawlee": "^3.5.4",
        "fingerprint-generator": "^2.1.66",
        "fingerprint-injector": "^2.1.66",
        "https-proxy-agent": "^7.0.6",
        "node-fetch": "^3.3.2",
        "proxy-chain": "^2.3.0",
        "puppeteer": "^22.8.2",
        "puppeteer-extra": "^3.3.6",
        "puppeteer-extra-plugin-adblocker": "^2.13.6",
        "puppeteer-extra-plugin-recaptcha": "^3.6.8",
        "puppeteer-extra-plugin-stealth": "^2.11.2",
        "puppeteer-extra-plugin-user-data-dir": "^2.4.1",
        "puppeteer-extra-plugin-user-preferences": "^2.4.1",
        "random-useragent": "^0.5.0",
        "uuid": "^9.0.1"
    },
    "devDependencies": {
        "@apify/eslint-config": "^0.4.0",
        "eslint": "^8.50.0"
    },
    "scripts": {
        "start": "node src/main.js",
        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
        "pull": "apify pull",
        "push": "apify push",
        "commit": "git add . && git commit -m \"Update\" && git push"
    },
    "author": "It's not you it's me",
    "license": "ISC"
}

# Dockerfile

FROM apify/actor-node-puppeteer-chrome:20

RUN npm ls crawlee apify puppeteer playwright

COPY --chown=myuser package*.json ./

RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && npm install puppeteer-extra puppeteer-extra-plugin-stealth puppeteer-extra-plugin-recaptcha random-useragent \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm


COPY --chown=myuser . ./

CMD ./start_xvfb_and_run_cmd.sh && npm start --silent

1

{
    "actorSpecification": 1,
    "name": "upwork-job-scraper",
    "title": "Project Puppeteer Crawler JavaScript",
    "description": "Crawlee and Puppeteer project in JavaScript.",
    "version": "0.10",
    "meta": {
        "templateId": "js-crawlee-puppeteer-chrome"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

{
  "title": "Upwork Job Search",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "searchQuery": {
      "title": "Search Query",
      "type": "string",
      "description": "Keywords to search for on Upwork (e.g., job title, skills)",
      "editor": "textfield",
      "default": ""
    },
    "page": {
      "title": "Page Number",
      "type": "integer",
      "description": "The page number to scrape on Upwork",
      "editor": "number",
      "default": 1
    },
    "useEnhancedCrawler": {
      "title": "Use Enhanced Anti-Detection System",
      "type": "boolean",
      "description": "Use the advanced Upwork challenge bypass system (recommended for 403 errors)",
      "editor": "checkbox",
      "default": true
    },
    "tlsFingerprintEvasion": {
      "title": "Enable TLS Fingerprint Evasion",
      "type": "boolean",
      "description": "Use advanced TLS fingerprinting evasion to bypass Upwork's security (requires Enhanced Crawler)",
      "editor": "checkbox",
      "default": true
    },
    "customHeaderOrder": {
      "title": "Randomize HTTP Header Order",
      "type": "boolean",
      "description": "Randomize HTTP header order to evade fingerprinting detection (requires Enhanced Crawler)",
      "editor": "checkbox",
      "default": true
    },
    "advancedFingerprinting": {
      "title": "Use Advanced Fingerprint Evasion",
      "type": "boolean",
      "description": "Apply comprehensive browser fingerprint masking techniques (Canvas, WebGL, Audio, etc.)",
      "editor": "checkbox",
      "default": true
    },
    "fingerprintConsistency": {
      "title": "Maintain Consistent Fingerprints",
      "type": "boolean",
      "description": "Keep browser fingerprints consistent within sessions (recommended)",
      "editor": "checkbox",
      "default": true
    },
    "fingerprintBrowserProfile": {
      "title": "Browser Fingerprint Profile",
      "type": "string",
      "description": "Browser profile to emulate for fingerprint generation",
      "editor": "select",
      "default": "auto-rotate",
      "enum": ["auto-rotate", "chrome", "safari", "firefox"],
      "enumTitles": ["Auto Rotate", "Chrome", "Safari", "Firefox"]
    },
    "useApifyProxy": {
      "title": "Use Apify Proxy",
      "type": "boolean",
      "description": "Whether to use Apify Proxy",
      "editor": "checkbox",
      "default": true,
      "prefill": true
    },
    "maxConcurrency": {
      "title": "Max Concurrency",
      "type": "integer",
      "description": "Maximum number of concurrent requests (lower is safer)",
      "editor": "number",
      "default": 1,
      "minimum": 1,
      "maximum": 10
    },
    "maxRetries": {
      "title": "Max Retries",
      "type": "integer",
      "description": "Maximum number of retries per request",
      "editor": "number",
      "default": 3,
      "minimum": 1,
      "maximum": 10
    },
    "upworkUsername": {
      "title": "Upwork Username",
      "type": "string",
      "description": "Optional: Your Upwork username for authenticated access (recommended to bypass 403 errors)",
      "editor": "textfield"
    },
    "upworkPassword": {
      "title": "Upwork Password",
      "type": "string",
      "description": "Optional: Your Upwork password for authenticated access",
      "editor": "textfield",
      "isSecret": true
    },
    "captchaApiKey": {
      "title": "2Captcha API Key",
      "type": "string",
      "description": "Optional: Your 2Captcha API key to solve CAPTCHAs automatically (https://2captcha.com/)",
      "editor": "textfield",
      "isSecret": true
    },
    "sessionRotationCount": {
      "title": "Session Rotation Count",
      "type": "integer",
      "description": "Number of browser sessions to rotate through (helps avoid detection)",
      "editor": "number",
      "default": 3,
      "minimum": 1,
      "maximum": 10
    },
    "minDelayBetweenRequests": {
      "title": "Min Delay Between Requests (seconds)",
      "type": "integer",
      "description": "Minimum delay in seconds between requests (higher values reduce blocking)",
      "editor": "number",
      "default": 15,
      "minimum": 5,
      "maximum": 120
    },
    "maxDelayBetweenRequests": {
      "title": "Max Delay Between Requests (seconds)",
      "type": "integer",
      "description": "Maximum delay in seconds between requests (higher values reduce blocking)",
      "editor": "number",
      "default": 45,
      "minimum": 10,
      "maximum": 180
    },
    "sessionCooldownMinutes": {
      "title": "Session Cooldown Minutes",
      "type": "integer",
      "description": "Time in minutes to rest a session after multiple uses",
      "editor": "number",
      "default": 30,
      "minimum": 5,
      "maximum": 120
    },
    "proxyRotationEnabled": {
      "title": "Enable Proxy Rotation",
      "type": "boolean",
      "description": "Whether to rotate proxies during the session (helps prevent IP-based blocking)",
      "editor": "checkbox",
      "default": true
    },
    "proxyRotationRequests": {
      "title": "Requests Per Proxy",
      "type": "integer",
      "description": "Number of requests before rotating to a new proxy",
      "editor": "number",
      "default": 3,
      "minimum": 1,
      "maximum": 10
    },
    "simulateBrowserHistory": {
      "title": "Simulate Browser History",
      "type": "boolean",
      "description": "Whether to simulate browser history and cache for a more authentic profile",
      "editor": "checkbox",
      "default": true
    },
    "randomizeTimezone": {
      "title": "Randomize Browser Timezone",
      "type": "boolean",
      "description": "Whether to randomize timezone and locale settings per session",
      "editor": "checkbox",
      "default": true
    },
    "useNewHeadless": {
      "title": "Use New Headless Mode",
      "type": "boolean",
      "description": "Use Chrome's new headless mode which is less detectable than the old mode",
      "editor": "checkbox",
      "default": true
    },
    "useRealProfiles": {
      "title": "Use Real Browser Profiles",
      "type": "boolean",
      "description": "Use real Chrome profiles with history and extensions (better anti-detection)",
      "editor": "checkbox",
      "default": true
    },
    "useCloudScraper": {
      "title": "Use CloudScraper",
      "type": "boolean",
      "description": "Use CloudScraper to bypass Cloudflare protection (fallback method)",
      "editor": "checkbox",
      "default": true
    },
    "simulateExtensions": {
      "title": "Simulate Browser Extensions",
      "type": "boolean",
      "description": "Simulate common browser extensions to appear more like a real user",
      "editor": "checkbox",
      "default": true
    },
    "useFallbackHtmlParser": {
      "title": "Use Direct HTML Parser Fallback",
      "type": "boolean",
      "description": "When browser approach fails, try to extract data directly from HTML (last resort)",
      "editor": "checkbox",
      "default": true
    },
    "disableWebSecurity": {
      "title": "Disable Web Security",
      "type": "boolean",
      "description": "Disable browser web security features (CORS, etc.) to bypass some protections",
      "editor": "checkbox",
      "default": false
    },
    "bypassCSP": {
      "title": "Bypass Content Security Policy",
      "type": "boolean",
      "description": "Bypass website Content Security Policy restrictions",
      "editor": "checkbox",
      "default": false
    },
    "proxyConfiguration": {
      "title": "Proxy configuration",
      "type": "object",
      "description": "Choose to use no proxy, Apify Proxy, or provide custom proxy URLs.",
      "prefill": {
        "useApifyProxy": true,
        "apifyProxyGroups": []
      },
      "default": {},
      "editor": "proxy"
    }
  }
}

1# Upwork Scraper Network Traffic Analysis
2
3This directory contains tools to analyze and improve network traffic patterns for the Upwork job scraper, helping to bypass 403 errors and anti-bot detection.
4
5## Tools Overview
6
71. **capture_traffic.sh** - Script for capturing and analyzing network traffic using TCPDump
82. **apply_findings.js** - Script to modify the scraper code based on network analysis findings
9
10## Prerequisites
11
12- TCPDump (`brew install tcpdump` on macOS)
13- Node.js 14+ for the apply_findings.js script
14- Fingerprinting packages (`npm install fingerprint-generator fingerprint-injector`)
15
16## Getting Started
17
18### Step 1: Capture Traffic
19
20The capture_traffic.sh script allows you to capture and compare network traffic from:
21- A successful manual browser session
22- A failing automated scraper session
23
24```bash
25# Make the script executable
26chmod +x capture_traffic.sh
27
28# Capture a successful manual browser session (use your normal browser)
29./capture_traffic.sh --manual
30
31# Capture a failing automated session (run the scraper in another terminal)
32./capture_traffic.sh --auto
33
34# Compare the two captures to identify differences
35./capture_traffic.sh --compare
36```
37
38### Step 2: Apply Fixes
39
40After analyzing the network traffic differences, you can apply fixes to the scraper:
41
42```bash
43# Install required dependencies
44npm install fingerprint-generator fingerprint-injector
45
46# Run the script to apply fixes based on traffic analysis
47node apply_findings.js
48```
49
50## What to Look For
51
52When comparing successful vs. failing traffic:
53
541. **TLS Fingerprinting** - Look for differences in TLS handshakes, cipher suites, and extensions
552. **HTTP Headers** - Check order, values, and presence/absence of specific headers
563. **Browser Fingerprinting** - JavaScript challenges, canvas checks, WebGL rendering
574. **Request Timing** - Natural patterns vs. automated timing
58
59## Technical Details
60
61### Browser Fingerprinting
62
63The fingerprint-generator package creates realistic browser fingerprints that match real browser behavior:
64
65```javascript
66const fingerprint = fingerprintGenerator.getFingerprint({
67    browserName: 'chrome',
68    browserVersion: 123,
69    operatingSystem: 'macos',
70    operatingSystemVersion: '10.15.7',
71    deviceCategory: 'desktop',
72    locale: 'en-US'
73});
74```
75
76Then fingerprint-injector applies this fingerprint to Puppeteer:
77
78```javascript
79await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint);
80```
81
82### HTTP Headers
83
84The order and values of HTTP headers can be used to identify bots. The analysis will help identify the correct header ordering used by real browsers:
85
86```
87Host: www.upwork.com
88Connection: keep-alive
89sec-ch-ua: "Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"
90sec-ch-ua-mobile: ?0
91sec-ch-ua-platform: "macOS"
92Upgrade-Insecure-Requests: 1
93User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36
94Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
95Sec-Fetch-Site: none
96Sec-Fetch-Mode: navigate
97Sec-Fetch-User: ?1
98Sec-Fetch-Dest: document
99Accept-Encoding: gzip, deflate, br
100Accept-Language: en-US,en;q=0.9
101```
102
103### JavaScript Fingerprinting Protection
104
105The analysis will also help identify and mitigate JavaScript-based fingerprinting techniques used by Upwork:
106
107- Canvas fingerprinting - Adding subtle noise to canvas rendering
108- WebGL fingerprinting - Spoofing vendor and renderer information
109- Audio fingerprinting - Adding minor variations to audio data
110- Navigator property checks - Fixing browser detection properties
111- DOM property access patterns - Matching real browser behavior
112
113## Troubleshooting
114
115If you're still getting 403 errors after applying fixes:
116
1171. Try different network interfaces or mobile tethering
1182. Use higher quality proxies (mobile/4G proxies work better)
1193. Increase the capture time to get more data
1204. Try different browsers for the manual capture (Safari, Firefox)
1215. Use tcpdump's deeper analysis with `-v` or `-vv` flags
122
123## Advanced Usage
124
125For more detailed packet analysis:
126
127```bash
128# Analyze SSL/TLS handshake in detail
129sudo tcpdump -i en0 -nn -s0 -vvv "host upwork.com and tcp port 443" -w detailed_capture.pcap
130
131# Extract all HTTP headers
132tcpdump -A -s 0 'tcp port 443 and (((ip[2:2] - ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0)' -r capture.pcap
133```
134
135## Additional Techniques to Try
136
137If the fingerprinting techniques don't fully resolve the 403 errors, consider:
138
1391. **Mobile User Agents** - Try using mobile user agents which often face less scrutiny
1402. **4G/5G Mobile Proxies** - These have cleaner IP reputation than data center proxies
1413. **Modify request timing** - Add more natural, human-like timing between navigation steps
1424. **Implement Chrome Extensions API** - More fully simulate popular browser extensions
1435. **HTTP/3 Support** - Implement HTTP/3 (QUIC) protocol support to bypass some fingerprinting

1/**
2 * This script applies network fingerprinting fixes to the Upwork scraper
3 * based on TCPDump analysis findings
4 */
5
6import fs from 'fs';
7import path from 'path';
8import { fileURLToPath } from 'url';
9
10const __dirname = path.dirname(fileURLToPath(import.meta.url));
11const MAIN_JS_PATH = path.join(__dirname, '..', 'src', 'main.js');
12
13// Check if main.js exists
14if (!fs.existsSync(MAIN_JS_PATH)) {
15  console.error(`Error: ${MAIN_JS_PATH} not found`);
16  process.exit(1);
17}
18
19// Read the existing main.js file
20let mainJsContent = fs.readFileSync(MAIN_JS_PATH, 'utf8');
21
22// Function to apply fingerprinting fixes based on TCPDump analysis
23function applyFingerprintFixes(mainJsContent) {
24  console.log('Applying fingerprinting fixes...');
25  
26  // Check if fingerprint-generator and fingerprint-injector modules are already imported
27  if (!mainJsContent.includes('fingerprint-generator') || !mainJsContent.includes('fingerprint-injector')) {
28    console.log('Adding fingerprinting modules...');
29    
30    // Add imports
31    mainJsContent = mainJsContent.replace(
32      'import { fileURLToPath } from \'url\';',
33      'import { fileURLToPath } from \'url\';\nimport { FingerprintGenerator } from \'fingerprint-generator\';\nimport { FingerprintInjector } from \'fingerprint-injector\';'
34    );
35    
36    // Initialize fingerprint generator after browserProfiles
37    const fingerprintGeneratorCode = `
38// Initialize fingerprint generator for more accurate browser emulation
39const fingerprintGenerator = new FingerprintGenerator({
40    browsers: [
41        { name: 'chrome', minVersion: 100, maxVersion: 123 },
42        { name: 'firefox', minVersion: 100, maxVersion: 123 },
43        { name: 'safari', minVersion: 15, maxVersion: 17 }
44    ],
45    devices: ['desktop'],
46    operatingSystems: ['macos', 'windows'],
47    locales: ['en-US', 'en-GB', 'de-DE'],
48    // Use consistent fingerprints per session for less detection
49    cache: true
50});
51
52// Fingerprint injector for applying fingerprints to browser
53const fingerprintInjector = new FingerprintInjector();`;
54    
55    // Insert fingerprint generator code after browserProfiles
56    mainJsContent = mainJsContent.replace(
57      '// Generate new sessions if needed',
58      fingerprintGeneratorCode + '\n\n// Generate new sessions if needed'
59    );
60    
61    console.log('Added fingerprint generator and injector');
62  }
63  
64  // Add fingerprint generation to session initialization
65  if (!mainJsContent.includes('generateFingerprint()')) {
66    console.log('Adding fingerprint generation to sessions...');
67    
68    // Add generateFingerprint function
69    const generateFingerprintCode = `
70// Generate consistent fingerprint for a session
71const generateFingerprint = (sessionId, profile) => {
72    // Generate fingerprint based on browser profile or random selection
73    const browserName = profile?.userAgent?.toLowerCase().includes('firefox') ? 'firefox' : 
74                        profile?.userAgent?.toLowerCase().includes('safari') ? 'safari' : 'chrome';
75    
76    const osName = profile?.platform === 'MacIntel' ? 'macos' : 'windows';
77    
78    const locale = profile?.locale || 'en-US';
79    
80    return fingerprintGenerator.getFingerprint({
81        browserName,
82        browserVersion: parseInt(profile?.userAgent?.match(/Chrome\\/(\\d+)/) 
83            || profile?.userAgent?.match(/Firefox\\/(\\d+)/)
84            || profile?.userAgent?.match(/Version\\/(\\d+)/) 
85            || [0, Math.floor(Math.random() * 20) + 100])[1],
86        operatingSystem: osName,
87        operatingSystemVersion: osName === 'macos' ? '10.15.7' : '10.0',
88        deviceCategory: 'desktop',
89        locale
90    });
91};`;
92    
93    // Insert generateFingerprint function
94    mainJsContent = mainJsContent.replace(
95      '// Generate fake browsing history for a more authentic browser profile',
96      generateFingerprintCode + '\n\n// Generate fake browsing history for a more authentic browser profile'
97    );
98    
99    // Add fingerprint to session generation
100    mainJsContent = mainJsContent.replace(
101      'return {',
102      'const fingerprint = generateFingerprint(`session_${Date.now()}_${i}`, profile);\n        return {'
103    );
104    
105    mainJsContent = mainJsContent.replace(
106      'id: `session_${Date.now()}_${i}`,',
107      'id: `session_${Date.now()}_${i}`,\n            fingerprint,'
108    );
109    
110    console.log('Added fingerprint generation to sessions');
111  }
112  
113  // Add fingerprint injection to preNavigationHooks
114  if (!mainJsContent.includes('injectFingerprint')) {
115    console.log('Adding fingerprint injection to navigation...');
116    
117    // Find the preNavigationHooks section
118    const navHooksRegex = /preNavigationHooks: \[\s*\/\/ Hook pour ajouter des délais aléatoires entre requêtes et configurer le navigateur\s*async \(\{ page, request, session \}\) => \{([\s\S]*?)\},\s*\],/;
119    const navHooksMatch = mainJsContent.match(navHooksRegex);
120    
121    if (navHooksMatch) {
122      // Add fingerprint injection code
123      const fingerPrintInjectionCode = `
124            // Apply fingerprint from session
125            if (sessionIndex !== -1 && sessions[sessionIndex].fingerprint) {
126                console.log(\`Injecting fingerprint for session \${sessionId}...\`);
127                try {
128                    await fingerprintInjector.attachFingerprintToPuppeteer(page, sessions[sessionIndex].fingerprint);
129                    console.log('Fingerprint successfully injected');
130                } catch (error) {
131                    console.error('Error injecting fingerprint:', error.message);
132                }
133            }`;
134      
135      // Insert after headers are set
136      mainJsContent = mainJsContent.replace(
137        'await page.setExtraHTTPHeaders(headers);',
138        'await page.setExtraHTTPHeaders(headers);\n' + fingerPrintInjectionCode
139      );
140      
141      console.log('Added fingerprint injection to navigation hooks');
142    } else {
143      console.log('Warning: Could not find preNavigationHooks section');
144    }
145  }
146  
147  return mainJsContent;
148}
149
150// Function to enhance HTTP header ordering based on TCPDump findings
151function enhanceHeaderOrdering(mainJsContent) {
152  console.log('Enhancing HTTP header ordering...');
153  
154  // Check if already implemented
155  if (mainJsContent.includes('naturalistic header ordering')) {
156    console.log('HTTP header enhancements already exist, skipping...');
157    return mainJsContent;
158  }
159  
160  // Find the headerOrder array in the code
161  const headerOrderRegex = /const headerOrder = \[([\s\S]*?)\];/;
162  const headerOrderMatch = mainJsContent.match(headerOrderRegex);
163  
164  if (headerOrderMatch) {
165    // Replace with improved header ordering based on TCPDump analysis
166    const improvedHeaderOrder = `const headerOrder = [
167                // More naturalistic header ordering based on TCPDump analysis
168                // Primary headers always come first in real browsers
169                'Host',
170                'Connection',
171                'Cache-Control',
172                'sec-ch-ua',
173                'sec-ch-ua-mobile',
174                'sec-ch-ua-platform',
175                'Upgrade-Insecure-Requests',
176                'User-Agent',
177                'Accept',
178                'Accept-Encoding',
179                'Accept-Language',
180                // Secondary headers might appear in this order
181                'Sec-Fetch-Site',
182                'Sec-Fetch-Mode',
183                'Sec-Fetch-User',
184                'Sec-Fetch-Dest',
185                'Referer',
186                'Cookie',
187                'DNT'
188            ].sort(() => {
189                // Maintain primary header ordering (first 10) but randomize others
190                if (Math.random() > 0.8) {
191                    return Math.random() - 0.5;
192                }
193                return 0;
194            });`;
195    
196    mainJsContent = mainJsContent.replace(headerOrderRegex, improvedHeaderOrder);
197    
198    console.log('Enhanced header ordering pattern');
199  } else {
200    console.log('Warning: Could not find header ordering code to enhance');
201  }
202  
203  return mainJsContent;
204}
205
206// Function to enhance browser fingerprinting protection
207function enhanceBrowserFingerprinting(mainJsContent) {
208  console.log('Enhancing browser fingerprinting protection...');
209  
210  // Check if enhanced fingerprinting already exists
211  if (mainJsContent.includes('enhanced fingerprinting protection')) {
212    console.log('Enhanced fingerprinting protection already exists, skipping...');
213    return mainJsContent;
214  }
215  
216  // Find the page.evaluateOnNewDocument section for fingerprinting
217  const fingerprintRegex = /await page\.evaluateOnNewDocument\(\(\) => \{([\s\S]*?)\}\);/;
218  const fingerprintMatch = mainJsContent.match(fingerprintRegex);
219  
220  if (fingerprintMatch) {
221    // Enhanced fingerprinting protection based on TCPDump analysis
222    const enhancedFingerprinting = `await page.evaluateOnNewDocument(() => {
223                // Enhanced fingerprinting protection based on TCPDump analysis
224                
225                // Hide automation fingerprints
226                delete Object.getPrototypeOf(navigator).webdriver;
227                
228                // Override platform info with more accurate values
229                Object.defineProperty(navigator, 'platform', {
230                    get: () => 'MacIntel',
231                });
232                
233                // Override hardware concurrency
234                Object.defineProperty(navigator, 'hardwareConcurrency', {
235                    get: () => 8,
236                });
237                
238                // Override device memory
239                Object.defineProperty(navigator, 'deviceMemory', {
240                    get: () => 8,
241                });
242                
243                // Improve Chrome properties emulation
244                if (typeof window !== 'undefined') {
245                    window.chrome = {
246                        app: {
247                            isInstalled: false,
248                            InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
249                            RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
250                        },
251                        runtime: {
252                            OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
253                            OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
254                            PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
255                            PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
256                            PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
257                            RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' }
258                        }
259                    };
260                }
261                
262                // Fix permissions behavior
263                const originalQuery = window.navigator.permissions?.query;
264                if (originalQuery) {
265                    window.navigator.permissions.query = (parameters) => {
266                        if (parameters.name === 'notifications') {
267                            return Promise.resolve({ state: Notification.permission });
268                        }
269                        if (parameters.name === 'clipboard-read' || parameters.name === 'clipboard-write') {
270                            return Promise.resolve({ state: 'prompt' });
271                        }
272                        return originalQuery(parameters);
273                    };
274                }
275                
276                // Fix WebGL fingerprinting - more accurate values from real browsers
277                if (window.WebGLRenderingContext) {
278                    const getParameter = WebGLRenderingContext.prototype.getParameter;
279                    WebGLRenderingContext.prototype.getParameter = function(parameter) {
280                        // UNMASKED_VENDOR_WEBGL
281                        if (parameter === 37445) {
282                            return 'Intel Inc.';
283                        }
284                        // UNMASKED_RENDERER_WEBGL
285                        if (parameter === 37446) {
286                            return 'Intel Iris Pro OpenGL Engine';
287                        }
288                        return getParameter.call(this, parameter);
289                    };
290                }
291                
292                // Fix canvas fingerprinting - add subtle noise
293                const originalGetContext = HTMLCanvasElement.prototype.getContext;
294                HTMLCanvasElement.prototype.getContext = function(type, attributes) {
295                    const context = originalGetContext.call(this, type, attributes);
296                    if (context && type === '2d') {
297                        const originalFillText = context.fillText;
298                        context.fillText = function() {
299                            // Add subtle variations to text rendering
300                            const args = arguments;
301                            if (Math.random() < 0.2 && args[0]) {
302                                args[0] = args[0].split('').map(c => Math.random() < 0.05 ? c + String.fromCharCode(8203) : c).join('');
303                            }
304                            return originalFillText.apply(this, args);
305                        };
306                        
307                        // Affect toDataURL to add subtle noise
308                        const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
309                        HTMLCanvasElement.prototype.toDataURL = function() {
310                            // Only add noise for likely fingerprinting scenarios
311                            if (this.width === 16 && this.height === 16 || 
312                                this.width <= 2 && this.height <= 2 ||
313                                this.width <= 200 && this.height <= 50) {
314                                
315                                const context = this.getContext('2d');
316                                if (context) {
317                                    // Add minimal noise that's invisible to humans
318                                    const imageData = context.getImageData(0, 0, this.width, this.height);
319                                    const data = imageData.data;
320                                    
321                                    // Only modify 1-2 pixels with minor variations
322                                    const pixelsToModify = Math.max(1, Math.floor(this.width * this.height * 0.01));
323                                    for (let i = 0; i < pixelsToModify; i++) {
324                                        const pixelIndex = Math.floor(Math.random() * data.length / 4) * 4;
325                                        // Only make very minor adjustments to color values
326                                        data[pixelIndex] = Math.min(255, Math.max(0, data[pixelIndex] + (Math.random() < 0.5 ? -1 : 1)));
327                                        data[pixelIndex + 1] = Math.min(255, Math.max(0, data[pixelIndex + 1] + (Math.random() < 0.5 ? -1 : 1)));
328                                        data[pixelIndex + 2] = Math.min(255, Math.max(0, data[pixelIndex + 2] + (Math.random() < 0.5 ? -1 : 1)));
329                                    }
330                                    
331                                    context.putImageData(imageData, 0, 0);
332                                }
333                            }
334                            return originalToDataURL.apply(this, arguments);
335                        };
336                    }
337                    return context;
338                };
339                
340                // Fix audio fingerprinting
341                const audioContext = window.AudioContext || window.webkitAudioContext;
342                if (audioContext) {
343                    const originalGetChannelData = AudioBuffer.prototype.getChannelData;
344                    AudioBuffer.prototype.getChannelData = function() {
345                        const channelData = originalGetChannelData.apply(this, arguments);
346                        // Only add noise for very short buffers (likely fingerprinting)
347                        if (this.length < 100) {
348                            const noise = 0.0001; // Very subtle noise
349                            // Only modify a few random samples
350                            const samplesToModify = Math.max(1, Math.floor(channelData.length * 0.001));
351                            for (let i = 0; i < samplesToModify; i++) {
352                                const index = Math.floor(Math.random() * channelData.length);
353                                channelData[index] += (Math.random() * 2 - 1) * noise;
354                            }
355                        }
356                        return channelData;
357                    };
358                }
359            });`;
360    
361    mainJsContent = mainJsContent.replace(fingerprintRegex, enhancedFingerprinting);
362    
363    console.log('Enhanced browser fingerprinting protection');
364  } else {
365    console.log('Warning: Could not find fingerprinting code to enhance');
366  }
367  
368  return mainJsContent;
369}
370
371// Apply all fixes
372let updatedContent = mainJsContent;
373updatedContent = applyFingerprintFixes(updatedContent);
374updatedContent = enhanceHeaderOrdering(updatedContent);
375updatedContent = enhanceBrowserFingerprinting(updatedContent);
376
377// Write the updated content back to main.js
378if (updatedContent !== mainJsContent) {
379  // Backup the original file
380  const backupPath = `${MAIN_JS_PATH}.bak.${Date.now()}`;
381  fs.writeFileSync(backupPath, mainJsContent);
382  console.log(`Original file backed up to: ${backupPath}`);
383  
384  // Write the updated content
385  fs.writeFileSync(MAIN_JS_PATH, updatedContent);
386  console.log(`Successfully updated ${MAIN_JS_PATH} with fingerprinting fixes`);
387} else {
388  console.log('No changes were made to the file');
389}
390
391console.log('\nNext steps:');
392console.log('1. Run the capture_traffic.sh script to collect network data');
393console.log('2. Use the data to further refine the fingerprinting in this script');
394console.log('3. Re-run this script to apply improved fingerprinting settings');

1/**
2 * Enhanced Upwork Crawler
3 * Uses advanced challenge bypass techniques
4 */
5
6import { Actor } from 'apify';
7import { Dataset } from 'crawlee';
8import fs from 'fs/promises';
9import path from 'path';
10import { fileURLToPath } from 'url';
11import { parseJobsFromHTML } from './main.js';
12import {
13  createEnhancedSession,
14  enhancedPageNavigation,
15  cleanupSession
16} from './upwork-challenge-integrator.js';
17import {
18  applySessionFingerprinting,
19  createFingerprintRotationStrategy
20} from './fingerprint-integration.js';
21
22const __dirname = path.dirname(fileURLToPath(import.meta.url));
23
24// Store active sessions
25const activeSessions = {};
26const sessionStats = {};
27
28/**
29 * Initialize a session pool for rotation
30 * @param {Object} options Configuration options
31 * @returns {Array} Array of session IDs
32 */
33async function initializeSessionPool(options = {}) {
34  const {
35    sessionCount = 3,
36    browserProfiles = [],
37    input = {},
38    proxyConfiguration = null,
39  } = options;
40  
41  console.log(`Creating ${sessionCount} new sessions for rotation`);
42  
43  const sessionIds = [];
44  
45  for (let i = 1; i <= sessionCount; i++) {
46    const sessionId = `session_${Date.now()}_${i}`;
47    sessionIds.push(sessionId);
48    
49    // Initialize session stats
50    sessionStats[sessionId] = {
51      requestCount: 0,
52      successCount: 0,
53      failureCount: 0,
54      lastUsed: null,
55      createdAt: Date.now(),
56    };
57  }
58  
59  return sessionIds;
60}
61
62/**
63 * Get the next available session for rotation
64 * @param {Array} sessionIds Array of session IDs
65 * @param {Object} options Configuration options
66 * @returns {string} Next session ID to use
67 */
68function getNextSession(sessionIds, options = {}) {
69  const {
70    cooldownMinutes = 30,
71    maxRequests = 3,
72  } = options;
73  
74  // Find least recently used session that's not in cooldown
75  const now = Date.now();
76  const cooldownMs = cooldownMinutes * 60 * 1000;
77  
78  // Sort sessions by priority:
79  // 1. Sessions that have never been used
80  // 2. Sessions that are out of cooldown and have made fewer requests
81  // 3. Sessions that have been used least recently
82  const sortedSessions = [...sessionIds].sort((a, b) => {
83    const statsA = sessionStats[a] || { requestCount: 0, lastUsed: null };
84    const statsB = sessionStats[b] || { requestCount: 0, lastUsed: null };
85    
86    // Never used sessions have highest priority
87    if (statsA.lastUsed === null && statsB.lastUsed !== null) return -1;
88    if (statsA.lastUsed !== null && statsB.lastUsed === null) return 1;
89    
90    // Check if either session is in cooldown
91    const aInCooldown = statsA.lastUsed && (now - statsA.lastUsed < cooldownMs);
92    const bInCooldown = statsB.lastUsed && (now - statsB.lastUsed < cooldownMs);
93    
94    if (!aInCooldown && bInCooldown) return -1;
95    if (aInCooldown && !bInCooldown) return 1;
96    
97    // If both are in cooldown or both are not, check request count
98    if (statsA.requestCount < maxRequests && statsB.requestCount >= maxRequests) return -1;
99    if (statsA.requestCount >= maxRequests && statsB.requestCount < maxRequests) return 1;
100    
101    // If both have similar request counts, use least recently used
102    return (statsA.lastUsed || 0) - (statsB.lastUsed || 0);
103  });
104  
105  // Return the best session
106  return sortedSessions[0];
107}
108
109/**
110 * Update session stats after use
111 * @param {string} sessionId Session ID
112 * @param {Object} result Result of the request
113 */
114function updateSessionStats(sessionId, result = {}) {
115  if (!sessionStats[sessionId]) {
116    sessionStats[sessionId] = {
117      requestCount: 0,
118      successCount: 0,
119      failureCount: 0,
120      lastUsed: null,
121      createdAt: Date.now(),
122    };
123  }
124  
125  sessionStats[sessionId].requestCount += 1;
126  sessionStats[sessionId].lastUsed = Date.now();
127  
128  if (result.success) {
129    sessionStats[sessionId].successCount += 1;
130  } else {
131    sessionStats[sessionId].failureCount += 1;
132  }
133}
134
135// Capture a screenshot and store it on Apify Storage
136const takeScreenshot = async (page, filename) => {
137    try {
138        const screenshotBuffer = await page.screenshot({
139            type: "jpeg",
140            quality: 50,
141            fullPage: false,
142        });
143        const screenshotKey = `${filename}-${Date.now()}.jpeg`;
144        await Actor.setValue(screenshotKey, screenshotBuffer, {
145            contentType: "image/jpeg",
146        });
147        const screenshotUrl = `https://api.apify.com/v2/key-value-stores/${Actor.getEnv().defaultKeyValueStoreId}/records/${screenshotKey}`;
148        console.log(`📸 Screenshot saved: ${screenshotUrl}`);
149        return screenshotUrl;
150    } catch (error) {
151        console.error(`❌ Failed to take screenshot: ${error.message}`);
152        return null;
153    }
154};
155
156/**
157 * Main crawler function
158 * @param {Object} options Crawler options
159 */
160async function crawlUpworkJobs(options = {}) {
161  const {
162    searchQuery = 'javascript',
163    maxPages = 5,
164    input = {},
165    proxyConfiguration = null,
166    browserProfiles = [],
167  } = options;
168  
169  try {
170    // Initialize session pool
171    const sessionIds = await initializeSessionPool({
172      sessionCount: input.sessionRotationCount || 3,
173      browserProfiles,
174      input,
175      proxyConfiguration,
176    });
177    
178    // Process each page
179    for (let page = 1; page <= maxPages; page++) {
180      // Get next available session
181      const sessionId = getNextSession(sessionIds, {
182        cooldownMinutes: input.sessionCooldownMinutes || 30,
183        maxRequests: input.proxyRotationRequests || 3,
184      });
185      
186      console.log(`Using session ${sessionId} for this crawl`);
187      
188      // Get or create session
189      let session = activeSessions[sessionId];
190      if (!session) {
191        // Get proxy URL if configured
192        let proxyUrl = null;
193        if (proxyConfiguration) {
194          proxyUrl = await proxyConfiguration.newUrl(sessionId);
195        }
196        
197        // Create enhanced session
198        session = await createEnhancedSession({
199          sessionId,
200          proxyUrl,
201          headless: !input.headful,
202          input,
203          browserProfiles,
204        });
205        
206        // Apply advanced fingerprinting specific to this session
207        if (session.page && !session.error) {
208          try {
209            const fingerprintConfig = await applySessionFingerprinting(session.page, sessionId);
210            console.log(`Applied consistent fingerprinting for session ${sessionId} using profile: ${fingerprintConfig.browserProfile}`);
211            session.fingerprintConfig = fingerprintConfig;
212          } catch (err) {
213            console.log(`Error applying fingerprinting: ${err.message}`);
214          }
215        }
216        
217        activeSessions[sessionId] = session;
218      }
219      
220      // Check if session has issues
221      if (session.error || session.connectionFailed || session.setupFailed) {
222        console.log(`Session ${sessionId} has issues: ${session.error || 'Failed setup'}`);
223        
224        // Clean up and remove problematic session
225        await cleanupSession(session);
226        delete activeSessions[sessionId];
227        
228        // Create a replacement session
229        const newSessionId = `session_${Date.now()}_replacement`;
230        sessionIds.push(newSessionId);
231        
232        console.log(`Created replacement session ${newSessionId}`);
233        continue;
234      }
235      
236      // Build search URL
237      let url;
238      if (searchQuery && searchQuery.trim() !== '') {
239        url = `https://www.upwork.com/nx/search/jobs/?q=${encodeURIComponent(searchQuery)}${page > 1 ? `&page=${page}` : ''}`;
240      } else {
241        url = `https://www.upwork.com/nx/search/jobs/${page > 1 ? `?page=${page}` : ''}`;
242      }
243      
244      // Navigate to page and extract jobs
245      const result = await enhancedPageNavigation({
246        session,
247        url,
248        input,
249        parseJobsFunction: parseJobsFromHTML,
250      });
251      
252      // Update session stats
253      updateSessionStats(sessionId, result);
254      
255      // Handle successful result
256      if (result.success && result.jobs && result.jobs.length > 0) {
257        console.log(`Successfully extracted ${result.jobs.length} jobs from page ${page} using ${result.method}`);
258        
259        // Save jobs to dataset
260        await Dataset.pushData(result.jobs);
261        
262        // Add delay between successful requests
263        const delaySeconds = Math.floor(Math.random() * 
264          (input.maxDelayBetweenRequests || 45) - (input.minDelayBetweenRequests || 15) + 1) + 
265          (input.minDelayBetweenRequests || 15);
266          
267        console.log(`Waiting ${delaySeconds} seconds before the next request...`);
268        await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
269      } else {
270        // Handle failure
271        console.log(`Failed to extract jobs from page ${page}: ${result.error || 'Unknown error'}`);
272        
273        if (result.needsSessionRotation) {
274          // Clean up and rotate to new session
275          await cleanupSession(session);
276          delete activeSessions[sessionId];
277          
278          // Retry with fresh session after delay
279          const delaySeconds = Math.floor(Math.random() * 30) + 30;
280          console.log(`Session rotation needed. Waiting ${delaySeconds} seconds before retrying...`);
281          await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
282          
283          // Decrement page to retry
284          page--;
285        }
286        
287        if (result.needsBrowserReset) {
288          // Reset browser but keep session
289          await cleanupSession(session);
290          delete activeSessions[sessionId];
291        }
292        
293        // Increase delay after errors
294        await new Promise(resolve => setTimeout(resolve, 10000));
295      }
296      
297      // Check if we need to rotate proxy
298      if (proxyConfiguration && input.proxyRotationEnabled && 
299          sessionStats[sessionId].requestCount >= (input.proxyRotationRequests || 3)) {
300        console.log(`Rotating proxy for session ${sessionId}`);
301        
302        if (session.browser) {
303          // Close existing browser
304          await cleanupSession(session);
305        }
306        
307        const proxyUrl = await proxyConfiguration.newUrl(sessionId);
308        console.log(`Rotating proxy for session ${sessionId} to: ${proxyUrl}`);
309        
310        // Reset session with new proxy
311        delete activeSessions[sessionId];
312      }
313
314      if (result.isChallengePage && session.page) {
315        const screenshotUrl = await takeScreenshot(session.page, 'cloudflare_challenge_enhanced');
316        const html = await session.page.content();
317        await Actor.setValue(`cloudflare_challenge_enhanced_${Date.now()}.html`, html);
318        if (screenshotUrl) {
319            console.log(`Cloudflare challenge screenshot: ${screenshotUrl}`);
320        }
321      }
322    }
323    
324    // Clean up all sessions
325    console.log('Crawl complete, cleaning up sessions');
326    for (const sessionId in activeSessions) {
327      await cleanupSession(activeSessions[sessionId]);
328    }
329    
330  } catch (error) {
331    console.error('Error in crawler:', error);
332    
333    // Clean up on error
334    for (const sessionId in activeSessions) {
335      await cleanupSession(activeSessions[sessionId]);
336    }
337  }
338}
339
340export {
341  crawlUpworkJobs,
342  initializeSessionPool,
343  getNextSession,
344};