Free Proxy Scraper

.git
.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

# Added by Apify CLI
node_modules

1annotated-types==0.7.0
2anyio==4.8.0
3apify==2.4.0
4apify_client==1.9.2
5apify_shared==1.3.1
6Brotli==1.1.0
7browserforge==1.2.3
8cachetools==5.5.2
9certifi==2025.1.31
10cffi==1.17.1
11charset-normalizer==3.4.1
12click==8.1.8
13colorama==0.4.6
14crawlee==0.6.3
15cryptography==44.0.2
16docutils==0.21.2
17eval_type_backport==0.2.2
18filelock==3.17.0
19h11==0.14.0
20h2==4.2.0
21hpack==4.1.0
22httpcore==1.0.7
23httpx==0.28.1
24hyperframe==6.1.0
25idna==3.10
26lazy-object-proxy==1.10.0
27markdown-it-py==3.0.0
28mdurl==0.1.2
29more-itertools==10.6.0
30multidict==6.1.0
31numpy==2.2.3
32pandas==2.2.3
33propcache==0.3.0
34psutil==7.0.0
35pycountry==24.6.1
36pycparser==2.22
37pydantic==2.10.6
38pydantic-settings==2.6.1
39pydantic_core==2.27.2
40pyee==12.1.1
41Pygments==2.19.1
42python-dateutil==2.9.0.post0
43python-dotenv==1.0.1
44pytz==2025.1
45requests==2.32.3
46requests-file==2.1.0
47rich==13.9.4
48setuptools==75.8.2
49six==1.17.0
50sniffio==1.3.1
51sortedcollections==2.1.0
52sortedcontainers==2.4.0
53tldextract==5.1.3
54typing_extensions==4.12.2
55tzdata==2025.1
56urllib3==2.3.0
57websockets==15.0.1
58wheel==0.45.1
59yarl==1.18.3
60zstandard==0.23.0

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.12

# Second, copy just requirements.txt into the Actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
    && python --version \
    && echo "Pip version:" \
    && pip --version \
    && echo "Installing dependencies:" \
    && pip install -r requirements.txt \
    && echo "All installed Python packages:" \
    && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Use compileall to ensure the runnability of the Actor Python code.
RUN python3 -m compileall -q .

# Create and run as a non-root user.
RUN useradd --create-home apify && \
    chown -R apify:apify ./  && \
    chown -R apify:apify /usr/local/lib/python*
USER apify


# Specify how to launch the source code of your Actor.
# By default, the "python3 -m ." command is run
CMD ["python3", "-m", "src"]

{
  "title": "Proxy Fetcher",
  "description": "Fetches free proxies with optional filtering by country codes, protocol support, and sorting options.",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "country_codes": {
      "title": "Country Codes",
      "type": "array",
      "description": "List of country codes to filter proxies (e.g., ['US', 'UK'])",
      "editor": "stringList"
    },
    "http": {
      "title": "HTTP",
      "type": "string",
      "enum": ["true", "false", "unset"],
      "default": "unset",
      "editor": "select",
      "description": "Filter proxies based on HTTP support: 'true' includes only proxies that support HTTP, 'false' includes only those that do not, 'unset' applies no filter."
    },
    "ssl": {
      "title": "SSL",
      "type": "string",
      "enum": ["true", "false", "unset"],
      "default": "unset",
      "editor": "select",
      "description": "Filter proxies based on SSL support: 'true' includes only proxies that support SSL, 'false' includes only those that do not, 'unset' applies no filter."
    },
    "socks4": {
      "title": "SOCKS4",
      "type": "string",
      "enum": ["true", "false", "unset"],
      "default": "unset",
      "editor": "select",
      "description": "Filter proxies based on SOCKS4 support: 'true' includes only proxies that support SOCKS4, 'false' includes only those that do not, 'unset' applies no filter."
    },
    "socks5": {
      "title": "SOCKS5",
      "type": "string",
      "enum": ["true", "false", "unset"],
      "default": "unset",
      "editor": "select",
      "description": "Filter proxies based on SOCKS5 support: 'true' includes only proxies that support SOCKS5, 'false' includes only those that do not, 'unset' applies no filter."
    },
    "sort_by": {
      "title": "Sort By",
      "type": "string",
      "description": "The field name to sort results by (e.g., 'delay', 'lastseen', 'checks_up', 'checks_down').",
      "enum": ["delay", "lastseen", "checks_up", "checks_down"],
      "default": "delay",
      "editor": "select"
    },
    "sort_order": {
      "title": "Sort Order",
      "type": "string",
      "description": "Sorting order: 'asc' for ascending, 'desc' for descending. Defaults to 'asc' if not provided.",
      "enum": ["asc", "desc"],
      "default": "asc",
      "editor": "select"
    }
  }
}

{
  "actorSpecification": 1,
  "name": "free-proxy-fetcher",
  "title": "Free Proxy Scraper",
  "description": "Scrapes and filters free proxy servers with protocol support mapping, country filtering, and sorting capabilities",
  "version": "1.0",
  "buildTag": "latest",
  "environmentVariables": {
    "PROXIES_SOURCE_URL": "https://vakhov.github.io/fresh-proxy-list/proxylist.json"
  },
  "dockerfile": "./Dockerfile",
  "storages": {
    "dataset": {
      "actorSpecification": 1,
      "fields": {
        "type": "object",
        "properties": {
          "proxies": {
            "type": "array",
            "items": {
              "type": "object",
              "description": "Processed proxy data"
            }
          },
          "error": {
            "type": "string",
            "description": "Error details"
          }
        },
        "required": ["proxies"]
      },
      "views": {
        "overview": {
          "title": "Overview",
          "transformation": {
            "fields": ["proxies", "error"]
          },
          "display": {
            "component": "table",
            "properties": {
              "proxies": {
                "label": "Proxies",
                "format": "json"
              },
              "error": {
                "label": "Error",
                "format": "text"
              }
            }
          }
        }
      }
    }
  }
}

1

1import asyncio
2
3from .main import main
4
5# Execute the Actor entry point.
6asyncio.run(main())

1from apify import Actor
2from os import getenv
3import requests
4from .models import ProtocolFilter, ProxyFetcherInput, ProxyFetcherOutput
5from .utils import map_anonymity_level
6
7
8async def main():
9    async with Actor:
10        try:
11            input_data = await Actor.get_input()
12            params = (
13                ProxyFetcherInput(**input_data) if input_data else ProxyFetcherInput()
14            )
15
16            url = getenv("PROXIES_SOURCE_URL")
17            Actor.log.info(f"Fetching proxies")
18            response = requests.get(url, timeout=10)
19            response.raise_for_status()
20            raw_proxies = response.json()
21
22            # Process proxy data
23            processed = []
24            for proxy in raw_proxies:
25                if not proxy.get("country_code"):
26                    proxy["country_code"] = "Unknown"
27                try:
28                    processed_proxy = {
29                        "ip": proxy["ip"],
30                        "port": int(proxy["port"]),
31                        "http": proxy["http"] == "1",
32                        "ssl": proxy["ssl"] == "1",
33                        "socks4": proxy["socks4"] == "1",
34                        "socks5": proxy["socks5"] == "1",
35                        "anon": map_anonymity_level(proxy["anon"]),
36                        "country_code": proxy.get("country_code").upper(),
37                        "delay": float(proxy.get("delay", 0)),
38                        "lastseen": proxy.get("lastseen"),
39                        "checks_up": proxy.get("checks_up"),
40                        "checks_down": proxy.get("checks_down"),
41                    }
42                    processed.append(processed_proxy)
43                except (KeyError, ValueError) as e:
44                    Actor.log.warning(
45                        f"Skipping invalid proxy entry: {proxy}. Error: {str(e)}"
46                    )
47
48            # Apply filters with None safety
49            filtered = [
50                p
51                for p in processed
52                if (
53                    params.country_codes is None
54                    or (
55                        isinstance(p.get("country_code"), str)
56                        and p["country_code"] in params.country_codes
57                    )
58                )
59                and (
60                    params.http == ProtocolFilter.UNSET
61                    or p["http"] == (params.http == ProtocolFilter.TRUE)
62                )
63                and (
64                    params.ssl == ProtocolFilter.UNSET
65                    or p["ssl"] == (params.ssl == ProtocolFilter.TRUE)
66                )
67                and (
68                    params.socks4 == ProtocolFilter.UNSET
69                    or p["socks4"] == (params.socks4 == ProtocolFilter.TRUE)
70                )
71                and (
72                    params.socks5 == ProtocolFilter.UNSET
73                    or p["socks5"] == (params.socks5 == ProtocolFilter.TRUE)
74                )
75            ]
76
77            # Safe sorting implementation
78            if params.sort_by:
79                reverse = params.sort_order.lower() == "desc"
80                try:
81                    filtered.sort(
82                        key=lambda x: x.get(params.sort_by, 0), reverse=reverse
83                    )
84                except Exception as e:
85                    Actor.log.error(f"Sorting failed: {str(e)}")
86
87            await Actor.push_data(
88                ProxyFetcherOutput(proxies=filtered).model_dump(
89                    exclude_none=True, exclude_unset=True
90                )
91            )
92
93        except Exception as e:
94            error_msg = f"Actor failed: {str(e)}"
95            Actor.log.error(error_msg)
96            await Actor.fail(exception=e, status_message=error_msg)

1from pydantic import BaseModel, Field, field_validator
2from typing import List, Optional
3from enum import Enum
4import pycountry
5
6
7class ProtocolFilter(str, Enum):
8    TRUE = "true"
9    FALSE = "false"
10    UNSET = "unset"
11
12
13class ProxyFetcherInput(BaseModel):
14    country_codes: Optional[List[str]] = Field(
15        None, description="List of ISO country codes (e.g., ['US', 'DE'])"
16    )
17    http: ProtocolFilter = ProtocolFilter.UNSET
18    ssl: ProtocolFilter = ProtocolFilter.UNSET
19    socks4: ProtocolFilter = ProtocolFilter.UNSET
20    socks5: ProtocolFilter = ProtocolFilter.UNSET
21    sort_by: str = "lastseen"
22    sort_order: str = "asc"
23
24    @field_validator("country_codes")
25    @classmethod
26    def validate_country_codes(cls, v: Optional[List[str]]) -> Optional[List[str]]:
27        if v is None:
28            return v
29
30        valid_alpha2 = {country.alpha_2.upper() for country in pycountry.countries}
31        valid_alpha3 = {country.alpha_3.upper() for country in pycountry.countries}
32
33        valid = []
34        invalid = []
35        for code in v:
36            if not isinstance(code, str):
37                invalid.append(str(code))
38                continue
39            normalized = code.strip().upper()
40            if normalized in valid_alpha2 or normalized in valid_alpha3:
41                valid.append(code)
42            else:
43                invalid.append(code)
44
45        if invalid:
46            raise ValueError(f"Invalid country codes: {', '.join(invalid)}")
47        return valid
48
49
50class ProxyFetcherOutput(BaseModel):
51    proxies: List[dict] = Field(..., description="Processed proxy data")
52    error: Optional[str] = Field(None, description="Error details")

1def map_anonymity_level(level: str) -> str:
2    level_map = {
3        "1": "No anonymity",
4        "2": "Low anonymity",
5        "3": "Average anonymity",
6        "4": "High anonymity",
7    }
8    return level_map.get(level, "Unknown")