Ai Job Finder

1apify-client>=1.0.0,<2.0.0
2openai>=1.0.0
3anthropic>=0.5.0
4google-generativeai==0.3.0
5pydantic>=2.0.0

1#!/usr/bin/env python3
2import asyncio
3import json
4import sys
5import os
6from typing import Dict, Any, Optional
7
8# Import from our modules
9from src.llm_providers.factory import create_llm_provider
10from src.cv_processor import process_cv
11from src.prompt_processor import process_prompt
12from src.parameter_handler import apply_parameter_defaults
13
14async def test_cv_processing():
15    """Test CV processing with a local file"""
16    # Check if file path was provided
17    if len(sys.argv) < 2:
18        print("Usage: python test.py path/to/cv.pdf [prompt]")
19        sys.exit(1)
20    
21    # Get CV file path and optional prompt
22    cv_path = sys.argv[1]
23    prompt = sys.argv[2] if len(sys.argv) > 2 else None
24    
25    # Check if API key is set
26    openai_key = os.environ.get("OPENAI_API_KEY")
27    if not openai_key:
28        print("ERROR: OPENAI_API_KEY environment variable not set.")
29        print("Please set it with: export OPENAI_API_KEY=your-api-key")
30        sys.exit(1)
31    
32    # Read CV file
33    try:
34        with open(cv_path, "rb") as f:
35            cv_data = f.read()
36            
37        # Convert to base64 for testing
38        import base64
39        import mimetypes
40        mime_type, _ = mimetypes.guess_type(cv_path)
41        if not mime_type:
42            mime_type = "application/octet-stream"
43            
44        cv_data_base64 = f"data:{mime_type};base64,{base64.b64encode(cv_data).decode('utf-8')}"
45    except Exception as e:
46        print(f"Error reading CV file: {str(e)}")
47        sys.exit(1)
48    
49    # Create LLM provider
50    provider = create_llm_provider("openai", openai_key)
51    
52    # Process CV
53    print("Processing CV...")
54    cv_parameters = await process_cv(cv_data_base64, provider, "openai")
55    print(f"CV Parameters: {json.dumps(cv_parameters, indent=2)}")
56    
57    # Process prompt if provided
58    prompt_parameters = {}
59    if prompt:
60        print("\nProcessing prompt...")
61        prompt_parameters = await process_prompt(prompt, provider)
62        print(f"Prompt Parameters: {json.dumps(prompt_parameters, indent=2)}")
63    
64    # Merge and apply defaults
65    parameters = {**cv_parameters, **prompt_parameters}
66    final_parameters = apply_parameter_defaults(parameters)
67    
68    print("\nFinal LinkedIn Search Parameters:")
69    print(json.dumps(final_parameters, indent=2))
70    
71    # Note: This test doesn't actually call the LinkedIn scraper
72    print("\nTest complete. To perform a real LinkedIn search, upload this Actor to Apify.")
73
74if __name__ == "__main__":
75    asyncio.run(test_cv_processing())

{
    "actorSpecification": 1,
    "name": "ai-job-finder",
    "title": "AI Job Finder",
    "description": "An AI-powered tool that reads a CV and/or prompt to find relevant jobs on LinkedIn",
    "version": "0.1",
    "buildTag": "latest",
    "meta": {
        "templateId": "python-apify"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

# First specify the base Docker image.
FROM apify/actor-python:3.12

# Copy requirements.txt into the Actor image
COPY requirements.txt ./

# Install the packages specified in requirements.txt
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Copy the remaining files and directories with the source code
COPY . ./

# Use compileall to ensure the runnability of the Actor Python code
RUN python3 -m compileall -q .

# Specify how to launch the source code of your Actor
CMD ["python3", "-m", "src"]

{
    "title": "AI Job Finder",
    "description": "An AI-powered tool that reads a CV and/or prompt to find relevant jobs on LinkedIn",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "cv": {
            "title": "CV/Resume",
            "type": "object",
            "description": "Upload your CV/resume (PDF, DOCX, TXT formats supported) as Base64 encoded string",
            "editor": "file",
            "nullable": true
        },
        "prompt": {
            "title": "Job Search Query",
            "type": "string",
            "description": "Describe the job you're looking for (e.g., 'Senior Python Developer in New York')",
            "editor": "textarea",
            "default": "I'm looking for remote senior software engineering roles in AI companies. I have 5 years of experience with Python and machine learning.",
            "nullable": true
        },
        "llm_settings": {
            "title": "LLM Provider Settings",
            "type": "object",
            "description": "Configure which LLM provider to use",
            "editor": "json",
            "default": {
                "provider": "gemini",
                "model": "gemini-1.5-pro"
            },
            "prefill": {
                "provider": "gemini",
                "model": "gemini-1.5-pro"
            }
        },
        "api_keys": {
            "title": "API Keys",
            "type": "object",
            "description": "API keys for LLM providers (optional - defaults to environment variables)",
            "editor": "json",
            "default": {},
            "prefill": {
                "openai": "",
                "claude": "",
                "gemini": ""
            }
        },
        "linkedin_search_params": {
            "title": "Additional LinkedIn Search Parameters",
            "type": "object",
            "description": "Override specific LinkedIn search parameters",
            "editor": "json",
            "nullable": true
        },
        "proxy": {
            "title": "Proxy Configuration",
            "type": "object",
            "description": "Configure Apify proxy for LinkedIn scraping",
            "editor": "proxy",
            "default": {
                "useApifyProxy": true,
                "apifyProxyGroups": ["RESIDENTIAL"]
            }
        }
    },
    "required": []
}

1import logging
2import base64
3import json
4import re
5from typing import Dict, Any, Optional
6
7logger = logging.getLogger(__name__)
8
9async def process_cv(cv_data: str, llm_provider, provider_name: str) -> Dict[str, Any]:
10    """
11    Process CV data using the appropriate LLM provider
12    
13    Args:
14        cv_data: CV data (either base64 encoded file or plain text)
15        llm_provider: The LLM provider instance to use
16        provider_name: Name of the provider ('openai', 'claude', or 'gemini')
17        
18    Returns:
19        Dictionary of extracted parameters for LinkedIn job search
20    """
21    try:
22        logger.info(f"Processing CV with {provider_name} provider")
23        
24        # Process CV with the provider
25        cv_parameters = await llm_provider.process_cv(cv_data)
26        
27        # Validate and clean the parameters
28        cv_parameters = validate_cv_parameters(cv_parameters)
29        
30        logger.info(f"Successfully extracted parameters from CV: {json.dumps(cv_parameters, indent=2)}")
31        return cv_parameters
32        
33    except Exception as e:
34        logger.error(f"Error processing CV: {str(e)}")
35        # Return empty parameters, which will use defaults later
36        return {}
37
38def validate_cv_parameters(parameters: Dict[str, Any]) -> Dict[str, Any]:
39    """
40    Validate and clean the parameters extracted from the CV
41    
42    Args:
43        parameters: Raw parameters extracted by the LLM
44        
45    Returns:
46        Cleaned and validated parameters
47    """
48    cleaned = {}
49    
50    # Clean and validate title
51    if "title" in parameters and parameters["title"]:
52        cleaned["title"] = str(parameters["title"]).strip()
53    
54    # Clean and validate location
55    if "location" in parameters and parameters["location"]:
56        cleaned["location"] = str(parameters["location"]).strip()
57    
58    # Clean and validate experienceLevel
59    if "experienceLevel" in parameters and parameters["experienceLevel"]:
60        exp_level = str(parameters["experienceLevel"]).strip()
61        # Ensure it's a number from 1-5
62        if exp_level in ["1", "2", "3", "4", "5"]:
63            cleaned["experienceLevel"] = exp_level
64    
65    # Clean and validate workType
66    if "workType" in parameters and parameters["workType"]:
67        work_type = str(parameters["workType"]).strip()
68        # Ensure it's a valid work type (1, 2, or 3)
69        if work_type in ["1", "2", "3"]:
70            cleaned["workType"] = work_type
71    
72    # Clean and validate contractType
73    if "contractType" in parameters and parameters["contractType"]:
74        contract_type = str(parameters["contractType"]).strip().upper()
75        # Ensure it's a valid contract type (F, P, C, T, I, or V)
76        if contract_type in ["F", "P", "C", "T", "I", "V"]:
77            cleaned["contractType"] = contract_type
78    
79    # Clean and validate skills (might be used for custom filtering later)
80    if "skills" in parameters and isinstance(parameters["skills"], list):
81        cleaned["skills"] = [str(skill).strip() for skill in parameters["skills"] if skill]
82    
83    return cleaned

1#!/usr/bin/env python3
2from apify import Actor
3import logging
4import json
5import base64
6import re
7import os
8from typing import Dict, List, Any, Optional
9
10# Import providers
11from .llm_providers.factory import create_llm_provider
12from .cv_processor import process_cv
13from .prompt_processor import process_prompt
14from .parameter_handler import apply_parameter_defaults
15
16# Set up logging
17logging.basicConfig(level=logging.INFO)
18logger = logging.getLogger(__name__)
19
20async def main():
21    """Main entry point for the Actor"""
22    # Initialize the Actor
23    await Actor.init()
24    
25    # Get input from the actor
26    actor_input = await Actor.get_input() or {}
27    
28    # Validate input - require at least CV or prompt
29    cv_data = actor_input.get("cv")
30    prompt = actor_input.get("prompt")
31    
32    if not cv_data and not prompt:
33        raise ValueError("At least one of CV or prompt must be provided")
34    
35    # Get LLM settings
36    llm_settings = actor_input.get("llm_settings", {"provider": "gemini", "model": "gemini-1.5-pro"})
37    provider_name = llm_settings.get("provider", "gemini")
38    
39    # Get API key - first from input, then from environment variables
40    api_keys = actor_input.get("api_keys", {})
41    api_key = api_keys.get(provider_name)
42    
43    # If no API key in input, try to get from environment variables
44    if not api_key:
45        if provider_name == "openai":
46            api_key = os.getenv("OPENAI_API_KEY")
47        elif provider_name == "gemini":
48            api_key = os.getenv("GEMINI_API_KEY")
49        elif provider_name == "claude":
50            api_key = os.getenv("CLAUDE_API_KEY")
51    
52    # If no API key was found, we can't proceed with LLM processing
53    if not api_key:
54        logger.warning(f"No API key provided for {provider_name}")
55        await Actor.push_data([{
56            "title": "LLM API KEY IS NEEDED",
57            "description": f"Please provide an API key for {provider_name.upper()} to use this Actor",
58            "instructions": f"Set the {provider_name.upper()}_API_KEY environment variable or provide it in the api_keys input parameter",
59            "location": "N/A",
60            "companyName": "AI Job Finder",
61            "experienceLevel": "N/A",
62            "workType": "N/A",
63            "contractType": "N/A",
64            "publishedAt": "N/A",
65            "message": f"API key for {provider_name} is required to get real results"
66        }])
67        logger.info("Returned message indicating API key is needed")
68        return
69    
70    # Create LLM provider for processing
71    model = llm_settings.get("model")
72    if provider_name == "gemini" and not model:
73        model = "gemini-1.5-pro"
74        
75    logger.info(f"Using LLM provider: {provider_name} with model: {model}")
76    llm_provider = create_llm_provider(provider_name, api_key, model)
77    
78    # Process parameters
79    parameters = {}
80    
81    # Extract parameters from CV and/or prompt
82    if cv_data:
83        logger.info("Processing CV...")
84        cv_parameters = await process_cv(cv_data, llm_provider, provider_name)
85        parameters.update(cv_parameters)
86    
87    if prompt:
88        logger.info("Processing prompt...")
89        try:
90            prompt_parameters = await process_prompt(prompt, llm_provider)
91            # Prompt parameters override CV parameters
92            parameters.update(prompt_parameters)
93        except Exception as e:
94            logger.error(f"Error processing prompt: {str(e)}")
95            # Continue with default parameters
96    
97    # Apply any explicit parameters from input
98    linkedin_params = actor_input.get("linkedin_search_params", {})
99    if linkedin_params:
100        parameters.update(linkedin_params)
101    
102    # Apply defaults for missing parameters
103    parameters = apply_parameter_defaults(parameters)
104    
105    # Set proxy configuration
106    if "proxy_configuration" in actor_input:
107        parameters["proxy"] = actor_input["proxy_configuration"]
108    elif "proxy" in actor_input:
109        parameters["proxy"] = actor_input["proxy"]
110    
111    # Log the parameters we'll use
112    logger.info(f"Using LinkedIn search parameters: {json.dumps(parameters, indent=2)}")
113    
114    # Call LinkedIn scraper
115    logger.info("Calling LinkedIn scraper with parameters")
116    try:
117        jobs = await call_linkedin_scraper(parameters)
118        
119        # Save output
120        await Actor.push_data(jobs)
121        logger.info(f"Found {len(jobs)} matching jobs")
122    except Exception as e:
123        logger.error(f"Error calling LinkedIn scraper: {str(e)}")
124        # Return a meaningful error to the user
125        await Actor.push_data([{
126            "title": "Error Connecting to LinkedIn Scraper",
127            "description": f"An error occurred while trying to connect to the LinkedIn Jobs Scraper: {str(e)}",
128            "error": True,
129            "parameters": parameters
130        }])
131
132async def call_linkedin_scraper(parameters):
133    """Call the LinkedIn scraper with the given parameters"""
134    # Prepare the Actor input
135    run_input = {
136        "title": parameters.get("title", ""),
137        "location": parameters.get("location", ""),
138        "companyName": parameters.get("companyName", []),
139        "companyId": parameters.get("companyId", []),
140        "workType": parameters.get("workType", ""),
141        "experienceLevel": parameters.get("experienceLevel", ""),
142        "contractType": parameters.get("contractType", ""),
143        "publishedAt": parameters.get("publishedAt", ""),
144        "rows": parameters.get("rows", 10),
145        "proxy": parameters.get("proxy", {
146            "useApifyProxy": True,
147            "apifyProxyGroups": ["RESIDENTIAL"]
148        })
149    }
150    
151    # Run the Actor and wait for it to finish using Actor.apify_client
152    # This automatically handles the authentication - no need for explicit API key
153    run = await Actor.apify_client.actor("BHzefUZlZRKWxkTck").call(run_input=run_input)
154    
155    # Fetch and return the Actor's output
156    dataset_items = await Actor.apify_client.dataset(run["defaultDatasetId"]).list_items()
157    return dataset_items.items

1import logging
2from typing import Dict, Any
3
4logger = logging.getLogger(__name__)
5
6def apply_parameter_defaults(parameters: Dict[str, Any]) -> Dict[str, Any]:
7    """
8    Apply default values for missing parameters
9
10    Args:
11        parameters: Current set of parameters
12
13    Returns:
14        Parameters with defaults applied
15    """
16    # Create a copy of the parameters to avoid modifying the original
17    final_params = parameters.copy()
18
19    # Check for title (required parameter)
20    if "title" not in final_params or not final_params["title"]:
21        final_params["title"] = "Software Engineer"  # Default job title
22        logger.info("Using default job title: 'Software Engineer'")
23
24    # Set default location if not provided
25    if "location" not in final_params or not final_params["location"]:
26        final_params["location"] = "United States"  # Country is required, default to United States
27        logger.info("Using default location: United States")
28
29    # Set default experience level if not provided
30    if "experienceLevel" not in final_params or not final_params["experienceLevel"]:
31        final_params["experienceLevel"] = "3"  # Associate
32        logger.info("Using default experience level: 3 (Associate)")
33
34    # Set default work type if not provided
35    if "workType" not in final_params or not final_params["workType"]:
36        final_params["workType"] = ""  # Empty string means any work type
37        logger.info("Using default work type: any")
38
39    # Set default contract type if not provided
40    if "contractType" not in final_params or not final_params["contractType"]:
41        final_params["contractType"] = "F"  # Full-time
42        logger.info("Using default contract type: F (Full-Time)")
43
44    # Set default published at if not provided
45    if "publishedAt" not in final_params or not final_params["publishedAt"]:
46        final_params["publishedAt"] = ""  # Empty string means any time
47        logger.info("Using default time frame: any time")
48
49    # Set default company name if not provided
50    if "companyName" not in final_params or not final_params["companyName"]:
51        final_params["companyName"] = []  # Empty list means any company
52        logger.info("Using default company name: any company")
53
54    # Set default company ID if not provided
55    if "companyId" not in final_params or not final_params["companyId"]:
56        final_params["companyId"] = []  # Empty list means any company ID
57        logger.info("Using default company ID: any company ID")
58
59    # Set default rows if not provided
60    if "rows" not in final_params or not final_params["rows"]:
61        final_params["rows"] = 10  # Default to 10 results
62        logger.info("Using default rows: 10")
63
64    # Ensure we have proper proxy configuration
65    if "proxy" not in final_params or not final_params["proxy"]:
66        final_params["proxy"] = {
67            "useApifyProxy": True,
68            "apifyProxyGroups": ["RESIDENTIAL"]
69        }
70        logger.info("Using default proxy configuration")
71
72    return final_params

1import logging
2import json
3from typing import Dict, Any, Optional
4
5logger = logging.getLogger(__name__)
6
7async def process_prompt(prompt: str, llm_provider) -> Dict[str, Any]:
8    """
9    Process user prompt and extract job search parameters
10    
11    Args:
12        prompt: User's job search query
13        llm_provider: The LLM provider instance to use
14        
15    Returns:
16        Dictionary of extracted parameters for LinkedIn job search
17    """
18    try:
19        logger.info("Processing user prompt")
20        
21        # Process prompt with the provider
22        prompt_parameters = await llm_provider.process_prompt(prompt)
23        
24        # Validate and clean the parameters
25        prompt_parameters = validate_prompt_parameters(prompt_parameters)
26        
27        logger.info(f"Successfully extracted parameters from prompt: {json.dumps(prompt_parameters, indent=2)}")
28        return prompt_parameters
29        
30    except Exception as e:
31        logger.error(f"Error processing prompt: {str(e)}")
32        # Return empty parameters, which will use defaults later
33        return {}
34
35def validate_prompt_parameters(parameters: Dict[str, Any]) -> Dict[str, Any]:
36    """
37    Validate and clean the parameters extracted from the prompt
38    
39    Args:
40        parameters: Raw parameters extracted by the LLM
41        
42    Returns:
43        Cleaned and validated parameters
44    """
45    cleaned = {}
46    
47    # Clean and validate title
48    if "title" in parameters and parameters["title"]:
49        cleaned["title"] = str(parameters["title"]).strip()
50    
51    # Clean and validate location
52    if "location" in parameters and parameters["location"]:
53        cleaned["location"] = str(parameters["location"]).strip()
54    
55    # Clean and validate experienceLevel
56    if "experienceLevel" in parameters and parameters["experienceLevel"]:
57        exp_level = str(parameters["experienceLevel"]).strip()
58        # Ensure it's a number from 1-5
59        if exp_level in ["1", "2", "3", "4", "5"]:
60            cleaned["experienceLevel"] = exp_level
61    
62    # Clean and validate workType
63    if "workType" in parameters and parameters["workType"]:
64        work_type = str(parameters["workType"]).strip()
65        # Ensure it's a valid work type (1, 2, or 3)
66        if work_type in ["1", "2", "3"]:
67            cleaned["workType"] = work_type
68    
69    # Clean and validate contractType
70    if "contractType" in parameters and parameters["contractType"]:
71        contract_type = str(parameters["contractType"]).strip().upper()
72        # Ensure it's a valid contract type (F, P, C, T, I, or V)
73        if contract_type in ["F", "P", "C", "T", "I", "V"]:
74            cleaned["contractType"] = contract_type
75    
76    # Clean and validate publishedAt
77    if "publishedAt" in parameters and parameters["publishedAt"]:
78        published_at = str(parameters["publishedAt"]).strip()
79        # Ensure it's a valid time frame
80        if published_at in ["r86400", "r604800", "r2592000", ""]:
81            cleaned["publishedAt"] = published_at
82    
83    # Clean and validate rows
84    if "rows" in parameters and parameters["rows"]:
85        try:
86            rows = int(parameters["rows"])
87            if rows > 0:
88                cleaned["rows"] = rows
89        except (ValueError, TypeError):
90            pass
91    
92    # Clean and validate companyName
93    if "companyName" in parameters and isinstance(parameters["companyName"], list):
94        cleaned["companyName"] = [str(company).strip() for company in parameters["companyName"] if company]
95    
96    # Clean and validate companyId
97    if "companyId" in parameters and isinstance(parameters["companyId"], list):
98        cleaned["companyId"] = [str(company_id).strip() for company_id in parameters["companyId"] if company_id]
99    
100    return cleaned

1# AI Job Finder package

1import asyncio
2
3from .main import main
4
5# Execute the Actor entrypoint
6asyncio.run(main())

.git
.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

# Added by Apify CLI
node_modules

1# 🚀 Advanced Reddit Scraper for Apify Actors - Lightning Fast & SEO Optimized
2
3Unlock the full potential of Reddit data with our Advanced Reddit Scraper designed for the Apify platform. This high-performance tool uses lightning-fast requests to extract extensive subreddit information, providing researchers, marketers, and data enthusiasts with unparalleled social media insights.
4
5## 📊 Comprehensive Reddit Data Extraction
6
7Our Reddit Scraper offers a robust set of features that allow you to collect detailed data from any subreddit using Apify's powerful actor architecture. Enjoy rapid scraping with optimal performance while taking advantage of customizable settings tailored for your data requirements.
8
9### 🔥 Key SEO-Optimized Features
10
11- **Full Subreddit Scraping**: Extract every detail from the target subreddits, capturing posts, comments, and metadata.
12- **Customizable Data Fields**: Configure exactly what you're after, ensuring that you only get the data that matters.
13- **Lightning Fast Performance**: Utilizes Python requests for rapid data retrieval, so you never miss a trending topic.
14- **Scalable Data Collection**: Effortlessly scrape multiple subreddits simultaneously, ideal for large-scale data mining.
15- **Real-Time Insights**: Obtain the most current Reddit information, perfect for real-time analytics and trend monitoring.
16- **Easy Integration with Data Pipelines**: Seamlessly export data in various formats (JSON, CSV, etc.) for immediate analysis.
17
18### 🌟 Use Cases for Maximum Impact
19
201. **Market Research & Trend Analysis**: Monitor public opinion and identify trending topics across subreddits.
212. **Content Creation & Optimization**: Discover viral posts and themes to inspire your content strategy.
223. **Sentiment Analysis**: Analyze user reactions and sentiments using detailed comment extraction.
234. **Competitive Intelligence**: Stay ahead by tracking competitor mentions and industry-specific discussions.
245. **Academic & Social Media Research**: Gather comprehensive data for scholarly studies and social behavior analysis.
25
26### 🛠️ How It Works
27
281. **Input Parameters**: 
29   - **Queries**: Provide one or more subreddit URLs in the format `https://reddit.com/r/<subreddit>`.
30   - **Post Sorting**: Choose how posts are sorted (e.g., `hot`, `new`, `top`, or `rising`).  
31   - **Top Period**: Specify the period for top posts (e.g., `day`, `week`, or `all`).
32   - **Max Posts**: Set the maximum number of posts to scrape per subreddit.
33   - **Comment Sorting**: Define the method to sort comments (e.g., `best`, `top`, `new`).
34   - **Number of Comments**: Determine how many comments (and a few nested replies) to extract per post.
352. **Execution**: Our scraper efficiently navigates Reddit using HTTP requests, ensuring quick and reliable data extraction while strictly following Reddit's guidelines.
363. **Output**: Receive clean, structured data ready for analysis and integration into your existing workflows.
37
38### 📈 Why Our Reddit Scraper Stands Out
39
40- **Comprehensive Data Collection**: Capture every available piece of information from the subreddits you track.
41- **High-Speed Requests**: Leveraging the fastest possible scraping techniques to give you immediate insights.
42- **Customizable & Flexible**: Tailor the scraping process to meet diverse and specific data needs.
43- **Enterprise-Grade Scalability**: Perfect for both small-scale projects and large-scale data operations.
44- **Ethical & Compliant**: Adheres to Reddit’s data usage policies, including respecting robots.txt and API guidelines.
45
46### 🔗 Essential Resources
47
48- [Apify Platform](https://apify.com)
49- [Actor Documentation](https://docs.apify.com/actors)
50- [API Reference](https://docs.apify.com/api/v2)
51
52### 📞 Expert Support When You Need It
53
54For further assistance or inquiries, feel free to reach out:
55- 📧 Email: tnot2652@gmail.com
56
57### 🚀 Ready to Upgrade Your Data Game?
58
59Don't miss out on vital Reddit insights. Enhance your data strategy and make informed decisions with our Advanced Reddit Scraper. Start scraping smarter and faster on Apify today!

1# Feel free to add your Python dependencies below. For formatting guidelines, see:
2# https://pip.pypa.io/en/latest/reference/requirements-file-format/
3
4apify == 2.2.1
5beautifulsoup4[lxml]
6httpx
7types-beautifulsoup4

1from abc import ABC, abstractmethod
2from typing import Dict, Any, Optional
3
4class LLMProvider(ABC):
5    """Base abstract class for LLM providers"""
6    
7    def __init__(self, api_key: str, model: Optional[str] = None):
8        """
9        Initialize the LLM provider
10        
11        Args:
12            api_key: API key for the provider
13            model: Optional specific model to use
14        """
15        self.api_key = api_key
16        self.model = model
17    
18    @abstractmethod
19    async def process_cv(self, cv_data: str) -> Dict[str, Any]:
20        """
21        Process CV data and extract job search parameters
22        
23        Args:
24            cv_data: CV content (could be base64 encoded file or text)
25            
26        Returns:
27            Dictionary of extracted parameters
28        """
29        pass
30    
31    @abstractmethod
32    async def process_prompt(self, prompt: str) -> Dict[str, Any]:
33        """
34        Process user prompt and extract job search parameters
35        
36        Args:
37            prompt: User's job search query
38            
39        Returns:
40            Dictionary of extracted parameters
41        """
42        pass
43    
44    @abstractmethod
45    async def validate_api_key(self) -> bool:
46        """
47        Validate that the API key is correct
48        
49        Returns:
50            True if valid, False otherwise
51        """
52        pass
53    
54    def supports_document_processing(self) -> bool:
55        """
56        Check if the provider and model support direct document processing
57        
58        Returns:
59            True if document processing is supported, False otherwise
60        """
61        return False

1import json
2import logging
3import re
4import base64
5from typing import Dict, Any, Optional, List
6
7from anthropic import AsyncAnthropic
8from src.llm_providers.base_provider import LLMProvider
9
10logger = logging.getLogger(__name__)
11
12class ClaudeProvider(LLMProvider):
13    """Implementation of LLM provider for Anthropic Claude"""
14    
15    def __init__(self, api_key: str, model: Optional[str] = None):
16        """Initialize the Claude provider"""
17        super().__init__(api_key, model)
18        self.client = AsyncAnthropic(api_key=api_key)
19        self.model = model or "claude-3-opus-20240229"  # Default to most capable model
20    
21    def supports_document_processing(self) -> bool:
22        """Check if this provider/model supports direct document processing"""
23        return "claude-3" in self.model  # All Claude 3 models support document processing
24    
25    async def validate_api_key(self) -> bool:
26        """Validate the API key by making a simple models call"""
27        try:
28            # There's no direct way to validate the key without making a message request
29            # Use a minimal request to check if the key works
30            await self.client.messages.create(
31                model=self.model,
32                max_tokens=10,
33                messages=[{"role": "user", "content": "Hello"}]
34            )
35            return True
36        except Exception as e:
37            logger.error(f"Claude API key validation failed: {str(e)}")
38            return False
39    
40    async def process_cv(self, cv_data: str) -> Dict[str, Any]:
41        """
42        Process CV with Claude
43        
44        Args:
45            cv_data: CV content (could be base64 encoded file or text)
46            
47        Returns:
48            Dictionary of extracted parameters
49        """
50        if self.supports_document_processing() and cv_data.startswith("data:"):
51            return await self._process_cv_with_document_api(cv_data)
52        else:
53            # Assume it's already text
54            return await self._process_cv_text(cv_data)
55    
56    async def _process_cv_with_document_api(self, cv_data: str) -> Dict[str, Any]:
57        """Process CV using Claude's document capabilities"""
58        try:
59            # Extract the mime type and base64 data
60            mime_type, encoded_data = cv_data.split(';base64,', 1)
61            mime_type = mime_type.replace('data:', '')
62            
63            response = await self.client.messages.create(
64                model=self.model,
65                max_tokens=4000,
66                system="Extract job search parameters from this CV/resume.",
67                messages=[
68                    {"role": "user", "content": [
69                        {"type": "text", "text": self._get_cv_prompt()},
70                        {"type": "image", "source": {
71                            "type": "base64",
72                            "media_type": mime_type,
73                            "data": encoded_data
74                        }}
75                    ]}
76                ]
77            )
78            
79            # Extract JSON from response
80            content = response.content[0].text
81            # Find JSON in the content (handle potential text wrapping)
82            json_match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
83            if json_match:
84                return json.loads(json_match.group(1))
85            
86            # If no JSON block, try to parse the entire content
87            return json.loads(content)
88        except Exception as e:
89            logger.error(f"Claude document processing failed: {str(e)}")
90            raise
91    
92    async def _process_cv_text(self, cv_text: str) -> Dict[str, Any]:
93        """Process CV text with Claude"""
94        try:
95            response = await self.client.messages.create(
96                model=self.model,
97                max_tokens=4000,
98                system="Extract job search parameters from this CV/resume.",
99                messages=[
100                    {"role": "user", "content": self._get_cv_prompt() + f"\n\nCV TEXT:\n{cv_text}"}
101                ]
102            )
103            
104            # Extract JSON from response
105            content = response.content[0].text
106            # Find JSON in the content (handle potential text wrapping)
107            json_match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
108            if json_match:
109                return json.loads(json_match.group(1))
110            
111            # If no JSON block, try to parse the entire content
112            return json.loads(content)
113        except Exception as e:
114            logger.error(f"Claude text processing failed: {str(e)}")
115            raise
116    
117    async def process_prompt(self, prompt: str) -> Dict[str, Any]:
118        """Process user prompt and extract job search parameters"""
119        try:
120            response = await self.client.messages.create(
121                model=self.model,
122                max_tokens=4000,
123                system="Extract job search parameters from this query.",
124                messages=[
125                    {"role": "user", "content": self._get_prompt_extraction_prompt() + f"\n\nUSER QUERY:\n{prompt}"}
126                ]
127            )
128            
129            # Extract JSON from response
130            content = response.content[0].text
131            # Find JSON in the content (handle potential text wrapping)
132            json_match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
133            if json_match:
134                return json.loads(json_match.group(1))
135            
136            # If no JSON block, try to parse the entire content
137            return json.loads(content)
138        except Exception as e:
139            logger.error(f"Claude prompt processing failed: {str(e)}")
140            raise
141    
142    def _get_cv_prompt(self) -> str:
143        """Get the prompt for CV analysis"""
144        return """
145        Extract the following job search parameters from this CV/resume.
146        
147        Return your response as valid JSON object inside ```json code blocks with the following structure:
148        
149        ```json
150        {
151          "title": "The most recent job title or professional role",
152          "location": "Current or preferred location",
153          "experienceLevel": "A numeric value from 1-5 where:
154            1 = Internship
155            2 = Entry Level
156            3 = Associate
157            4 = Mid-Senior Level
158            5 = Director",
159          "workType": "Either:
160            1 = On-Site
161            2 = Remote
162            3 = Hybrid
163           Based on any workstyle preferences found in the CV",
164          "contractType": "A single letter representing employment type preference:
165            F = Full-Time
166            P = Part-Time
167            C = Contract
168            T = Temporary
169            I = Internship
170            V = Volunteer",
171          "skills": ["list", "of", "key", "technical", "and", "soft", "skills"]
172        }
173        ```
174        
175        If a piece of information is not clearly stated in the CV, make a reasonable inference based on the available information. If inference is not possible, use null.
176        
177        IMPORTANT: Your output must be a valid JSON object wrapped in ```json code blocks.
178        """
179    
180    def _get_prompt_extraction_prompt(self) -> str:
181        """Get the prompt for extracting parameters from user query"""
182        return """
183        Extract LinkedIn job search parameters from this query.
184        
185        Return your response as valid JSON object inside ```json code blocks with the following structure:
186        
187        ```json
188        {
189          "title": "Job title or role to search for",
190          "location": "Geographic location for job search",
191          "companyName": ["array of specific companies mentioned"],
192          "companyId": ["array of LinkedIn company IDs if mentioned"],
193          "workType": "Either:
194            1 = On-Site
195            2 = Remote
196            3 = Hybrid",
197          "experienceLevel": "A numeric value from 1-5 where:
198            1 = Internship
199            2 = Entry Level
200            3 = Associate
201            4 = Mid-Senior Level
202            5 = Director",
203          "contractType": "A single letter representing employment type:
204            F = Full-Time
205            P = Part-Time
206            C = Contract
207            T = Temporary
208            I = Internship
209            V = Volunteer",
210          "publishedAt": "Time frame:
211            r86400 = Last 24 hours
212            r604800 = Last week
213            r2592000 = Last month
214            empty string = Any time",
215          "rows": "Number of job listings to return (integer)"
216        }
217        ```
218        
219        For any parameters not explicitly mentioned in the query, use null.
220        
221        IMPORTANT: Your output must be a valid JSON object wrapped in ```json code blocks.
222        """