API Reference

unparallel

unparallel.RequestError dataclass

A dataclass wrapping an exception that was raised during a web request.

Besides the exception itself, this contains the URL, method, and (optional) payload of the failed request.

Source code in unparallel/unparallel.py

@dataclass
class RequestError:
    """A dataclass wrapping an exception that was raised during a web request.

    Besides the exception itself, this contains the URL, method, and (optional) payload
    of the failed request.
    """

    url: str
    method: str
    payload: Optional[Any]
    exception: Exception

unparallel.up async

up(urls, method='GET', base_url=None, headers=None, payloads=None, response_fn=DEFAULT_JSON_FN, flatten_result=False, max_connections=100, timeout=10, max_retries_on_timeout=3, raise_for_status=True, limits=None, timeouts=None, progress=True)

Creates async web requests to the specified URL(s) using asyncio and httpx.

Parameters:

Name	Type	Description	Default
urls	Union[str, List[str]]	A list of URLs as the targets for the requests. If only one URL but multiple payloads are supplied, that URL is used for all requests. If a base_url is supplied, this can also be a list of paths (or one path).	required
method	str	HTTP method to use - one of GET, OPTIONS, HEAD, POST, PUT, PATCH, or DELETE. Defaults to GET.	'GET'
base_url	Optional[str]	The base URL of the target API/service. Defaults to None.	None
headers	Optional[Dict[str, Any]]	A dictionary of headers to use. Defaults to None.	None
payloads	Optional[Any]	A list of JSON payloads (dictionaries) e.g. for HTTP post requests. Used together with urls. If one payload but multiple URLs are supplied, that payload is used for all requests. Defaults to None.	None
response_fn	Optional[Callable[[Response], Any]]	The function (callback) to apply on every response of the HTTP requests. This can be an existing function of httpx.Response like .json() or .read(), or a custom function which takes the httpx.Response as the argument returns Any. If you set this to None, you will get the raw httpx.Response. Defaults to httpx.Response.json.	DEFAULT_JSON_FN
flatten_result	bool	If True and the response per request is a list, flatten that list of lists. This is useful when using paging. Defaults to False.	False
max_connections	int	The total number of simultaneous TCP connections. Defaults to 100. This is passed into httpx.Limits.	100
timeout	int	The timeout for requests in seconds. Defaults to 10. This is passed into httpx.Timeout.	10
max_retries_on_timeout	int	The maximum number retries if the requests fails due to a timeout (httpx.TimeoutException). Defauls to 3.	3
raise_for_status	bool	If True, .raise_for_status() is called on overy response.	True
limits	Optional[Limits]	The limits configuration for httpx. If specified, this overrides the max_connections parameter.	None
timeouts	Optional[Timeout]	The timeout configuration for httpx. If specified, this overrides the timeout parameter.	None
progress	bool	If set to True, progress bar is shown. Defaults to True.	True

Raises:

Type	Description
ValueError	If the HTTP method is not valid.
ValueError	If the number of URLs provided does not match the number of payloads (except if there is only one URL).

Returns:

Type	Description
List[Any]	List[Any]: A list of the response data per request in the same order as the
List[Any]	input (URLs/payloads).

Source code in unparallel/unparallel.py

async def up(
    urls: Union[str, List[str]],
    method: str = "GET",
    base_url: Optional[str] = None,
    headers: Optional[Dict[str, Any]] = None,
    payloads: Optional[Any] = None,
    response_fn: Optional[Callable[[httpx.Response], Any]] = DEFAULT_JSON_FN,
    flatten_result: bool = False,
    max_connections: Optional[int] = 100,
    timeout: Optional[int] = 10,
    max_retries_on_timeout: int = 3,
    raise_for_status: bool = True,
    limits: Optional[httpx.Limits] = None,
    timeouts: Optional[httpx.Timeout] = None,
    progress: bool = True,
) -> List[Any]:
    """Creates async web requests to the specified URL(s) using ``asyncio``
    and ``httpx``.

    Args:
        urls (Union[str, List[str]]): A list of URLs as the targets for the requests.
            If only one URL but multiple payloads are supplied, that URL is used for
            all requests.
            If a ``base_url`` is supplied, this can also be a list of paths
            (or one path).
        method (str): HTTP method to use - one of ``GET``, ``OPTIONS``, ``HEAD``,
            ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. Defaults to ``GET``.
        base_url (Optional[str]):  The base URL of the target API/service. Defaults to
            None.
        headers (Optional[Dict[str, Any]], optional): A dictionary of headers to use.
            Defaults to None.
        payloads (Optional[Any], optional): A list of JSON payloads (dictionaries) e.g.
            for HTTP post requests. Used together with ``urls``. If one payload but
            multiple URLs are supplied, that payload is used for all requests.
            Defaults to None.
        response_fn (Optional[Callable[[httpx.Response], Any]]): The function (callback)
            to apply on every response of the HTTP requests. This can be an existing
            function of ``httpx.Response`` like ``.json()`` or ``.read()``, or a custom
            function which takes the ``httpx.Response`` as the argument returns ``Any``.
            If you set this to ``None``, you will get the raw ``httpx.Response``.
            Defaults to ``httpx.Response.json``.
        flatten_result (bool): If True and the response per request is a list,
            flatten that list of lists. This is useful when using paging.
            Defaults to False.
        max_connections (int): The total number of simultaneous TCP
            connections. Defaults to 100. This is passed into ``httpx.Limits``.
        timeout (int): The timeout for requests in seconds. Defaults to 10.
            This is passed into ``httpx.Timeout``.
        max_retries_on_timeout (int): The maximum number retries if the requests fails
            due to a timeout (``httpx.TimeoutException``). Defauls to 3.
        raise_for_status (bool): If True, ``.raise_for_status()`` is called on overy
            response.
        limits (Optional[httpx.Limits]): The limits configuration for ``httpx``.
            If specified, this overrides the ``max_connections`` parameter.
        timeouts (Optional[httpx.Timeout]): The timeout configuration for ``httpx``.
            If specified, this overrides the ``timeout`` parameter.
        progress (bool): If set to True, progress bar is shown.
            Defaults to True.

    Raises:
        ValueError: If the HTTP method is not valid.
        ValueError: If the number of URLs provided does not match the number of
            payloads (except if there is only one URL).

    Returns:
        List[Any]:  A list of the response data per request in the same order as the
        input (URLs/payloads).
    """
    # Check if method it valid
    if method.upper() not in VALID_HTTP_METHODS:
        raise ValueError(
            f"The method '{method}' is not a supported HTTP method. "
            f"Supported methods: {VALID_HTTP_METHODS}"
        )

    # Wrap single URL into list to check for alignment with payload
    if isinstance(urls, str):
        urls = [urls]

    # Check if payloads align with URLs
    if payloads:
        if not isinstance(payloads, list):
            payloads = [payloads]
        if len(urls) == 1 and len(payloads) > 1:
            logging.info(f"Using URL '{urls[0]}' for all {len(payloads)} payloads")
            urls = urls * len(payloads)
        if len(payloads) == 1 and len(urls) > 1:
            logging.info(f"Using payload '{payloads[0]}' for all {len(urls)} URLs")
            payloads = payloads * len(urls)
        if len(urls) != len(payloads):
            raise ValueError(
                f"The number of URLs does not match the number of payloads: "
                f"{len(urls)} != {len(payloads)}"
            )

    if timeouts is None:
        timeouts = httpx.Timeout(timeout)
    if limits is None:
        if max_connections != DEFAULT_LIMITS.max_connections:
            limits = httpx.Limits(max_connections=max_connections)
            limits.max_keepalive_connections = DEFAULT_LIMITS.max_keepalive_connections
        else:
            limits = DEFAULT_LIMITS

    return await request_urls(
        urls=urls,
        method=method,
        base_url=base_url,
        headers=headers,
        payloads=payloads,
        response_fn=response_fn,
        flatten_result=flatten_result,
        max_retries_on_timeout=max_retries_on_timeout,
        raise_for_status=raise_for_status,
        progress=progress,
        limits=limits,
        timeouts=timeouts,
    )