Scrapy Proxies: Middleware, Rotation and Retry Logic That Holds Up

# The entire built-in mechanism, in one line.
yield scrapy.Request(url, meta={"proxy": "http://USERNAME:PASSWORD@PROXY_HOST:PROXY_PORT"})

# middlewares.py
import uuid
from urllib.parse import quote
from w3lib.http import basic_auth_header


class SeamlessProxyMiddleware:
    """Assigns an exit identity to every outgoing request."""

    def __init__(self, host, user, password):
        self.host = host
        self.user = quote(user, safe="")
        self.password = quote(password, safe="")

    @classmethod
    def from_crawler(cls, crawler):
        s = crawler.settings
        return cls(
            s.get("PROXY_HOST"),
            s.get("PROXY_USER"),
            s.get("PROXY_PASSWORD"),
        )

    def process_request(self, request, spider):
        if "proxy" in request.meta:
            return  # a retry keeping its original exit

        # A session tag per domain keeps one identity per site.
        tag = request.meta.get("session_tag") or uuid.uuid4().hex[:8]
        user = f"{self.user}-session-{tag}"

        request.meta["proxy"] = f"http://{self.host}"
        request.meta["session_tag"] = tag
        request.headers["Proxy-Authorization"] = basic_auth_header(
            user, self.password
        )

# settings.py
DOWNLOADER_MIDDLEWARES = {
    "myproject.middlewares.SeamlessProxyMiddleware": 350,
    "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 400,
}

# settings.py
CONCURRENT_REQUESTS = 32
CONCURRENT_REQUESTS_PER_DOMAIN = 8      # the number that actually matters

AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1.0
AUTOTHROTTLE_MAX_DELAY = 30.0
AUTOTHROTTLE_TARGET_CONCURRENCY = 4.0
AUTOTHROTTLE_DEBUG = False

DOWNLOAD_TIMEOUT = 45                    # residential exits add latency
RANDOMIZE_DOWNLOAD_DELAY = True

# settings.py
RETRY_ENABLED = True
RETRY_TIMES = 4
# 403 and 407 are deliberately absent: neither improves on repetition.
RETRY_HTTP_CODES = [429, 500, 502, 503, 504, 522, 524]

class RetryOnNewExitMiddleware:
    """One retry for a 403, on a fresh identity, to isolate IP from request."""

    def process_response(self, request, response, spider):
        if response.status != 403 or request.meta.get("exit_retried"):
            return response

        new = request.copy()
        new.meta.pop("proxy", None)
        new.meta.pop("session_tag", None)
        new.meta["exit_retried"] = True
        new.dont_filter = True
        spider.logger.info("403 on %s — retrying on a new exit", request.url)
        return new

def process_response(self, request, response, spider):
    if response.status >= 400:
        spider.logger.warning(
            "%s %s session=%s",
            response.status,
            request.url,
            request.meta.get("session_tag"),
        )
    return response