initial commit
This commit is contained in:
Executable
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,22 @@
|
||||
Copyright 2006 Dan-Haim. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
3. Neither the name of Dan Haim nor the names of his contributors may be used
|
||||
to endorse or promote products derived from this software without specific
|
||||
prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
|
||||
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
|
||||
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
|
||||
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
|
||||
@@ -0,0 +1,321 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: PySocks
|
||||
Version: 1.7.1
|
||||
Summary: A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information.
|
||||
Home-page: https://github.com/Anorov/PySocks
|
||||
Author: Anorov
|
||||
Author-email: anorov.vorona@gmail.com
|
||||
License: BSD
|
||||
Keywords: socks,proxy
|
||||
Platform: UNKNOWN
|
||||
Classifier: Programming Language :: Python :: 2
|
||||
Classifier: Programming Language :: Python :: 2.7
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.4
|
||||
Classifier: Programming Language :: Python :: 3.5
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
PySocks
|
||||
=======
|
||||
|
||||
PySocks lets you send traffic through SOCKS and HTTP proxy servers. It is a modern fork of [SocksiPy](http://socksipy.sourceforge.net/) with bug fixes and extra features.
|
||||
|
||||
Acts as a drop-in replacement to the socket module. Seamlessly configure SOCKS proxies for any socket object by calling `socket_object.set_proxy()`.
|
||||
|
||||
----------------
|
||||
|
||||
Features
|
||||
========
|
||||
|
||||
* SOCKS proxy client for Python 2.7 and 3.4+
|
||||
* TCP supported
|
||||
* UDP mostly supported (issues may occur in some edge cases)
|
||||
* HTTP proxy client included but not supported or recommended (you should use urllib2's or requests' own HTTP proxy interface)
|
||||
* urllib2 handler included. `pip install` / `setup.py install` will automatically install the `sockshandler` module.
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
pip install PySocks
|
||||
|
||||
Or download the tarball / `git clone` and...
|
||||
|
||||
python setup.py install
|
||||
|
||||
These will install both the `socks` and `sockshandler` modules.
|
||||
|
||||
Alternatively, include just `socks.py` in your project.
|
||||
|
||||
--------------------------------------------
|
||||
|
||||
*Warning:* PySocks/SocksiPy only supports HTTP proxies that use CONNECT tunneling. Certain HTTP proxies may not work with this library. If you wish to use HTTP (not SOCKS) proxies, it is recommended that you rely on your HTTP client's native proxy support (`proxies` dict for `requests`, or `urllib2.ProxyHandler` for `urllib2`) instead.
|
||||
|
||||
--------------------------------------------
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
## socks.socksocket ##
|
||||
|
||||
import socks
|
||||
|
||||
s = socks.socksocket() # Same API as socket.socket in the standard lib
|
||||
|
||||
s.set_proxy(socks.SOCKS5, "localhost") # SOCKS4 and SOCKS5 use port 1080 by default
|
||||
# Or
|
||||
s.set_proxy(socks.SOCKS4, "localhost", 4444)
|
||||
# Or
|
||||
s.set_proxy(socks.HTTP, "5.5.5.5", 8888)
|
||||
|
||||
# Can be treated identical to a regular socket object
|
||||
s.connect(("www.somesite.com", 80))
|
||||
s.sendall("GET / HTTP/1.1 ...")
|
||||
print s.recv(4096)
|
||||
|
||||
## Monkeypatching ##
|
||||
|
||||
To monkeypatch the entire standard library with a single default proxy:
|
||||
|
||||
import urllib2
|
||||
import socket
|
||||
import socks
|
||||
|
||||
socks.set_default_proxy(socks.SOCKS5, "localhost")
|
||||
socket.socket = socks.socksocket
|
||||
|
||||
urllib2.urlopen("http://www.somesite.com/") # All requests will pass through the SOCKS proxy
|
||||
|
||||
Note that monkeypatching may not work for all standard modules or for all third party modules, and generally isn't recommended. Monkeypatching is usually an anti-pattern in Python.
|
||||
|
||||
## urllib2 Handler ##
|
||||
|
||||
Example use case with the `sockshandler` urllib2 handler. Note that you must import both `socks` and `sockshandler`, as the handler is its own module separate from PySocks. The module is included in the PyPI package.
|
||||
|
||||
import urllib2
|
||||
import socks
|
||||
from sockshandler import SocksiPyHandler
|
||||
|
||||
opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 9050))
|
||||
print opener.open("http://www.somesite.com/") # All requests made by the opener will pass through the SOCKS proxy
|
||||
|
||||
--------------------------------------------
|
||||
|
||||
Original SocksiPy README attached below, amended to reflect API changes.
|
||||
|
||||
--------------------------------------------
|
||||
|
||||
SocksiPy
|
||||
|
||||
A Python SOCKS module.
|
||||
|
||||
(C) 2006 Dan-Haim. All rights reserved.
|
||||
|
||||
See LICENSE file for details.
|
||||
|
||||
|
||||
*WHAT IS A SOCKS PROXY?*
|
||||
|
||||
A SOCKS proxy is a proxy server at the TCP level. In other words, it acts as
|
||||
a tunnel, relaying all traffic going through it without modifying it.
|
||||
SOCKS proxies can be used to relay traffic using any network protocol that
|
||||
uses TCP.
|
||||
|
||||
*WHAT IS SOCKSIPY?*
|
||||
|
||||
This Python module allows you to create TCP connections through a SOCKS
|
||||
proxy without any special effort.
|
||||
It also supports relaying UDP packets with a SOCKS5 proxy.
|
||||
|
||||
*PROXY COMPATIBILITY*
|
||||
|
||||
SocksiPy is compatible with three different types of proxies:
|
||||
|
||||
1. SOCKS Version 4 (SOCKS4), including the SOCKS4a extension.
|
||||
2. SOCKS Version 5 (SOCKS5).
|
||||
3. HTTP Proxies which support tunneling using the CONNECT method.
|
||||
|
||||
*SYSTEM REQUIREMENTS*
|
||||
|
||||
Being written in Python, SocksiPy can run on any platform that has a Python
|
||||
interpreter and TCP/IP support.
|
||||
This module has been tested with Python 2.3 and should work with greater versions
|
||||
just as well.
|
||||
|
||||
|
||||
INSTALLATION
|
||||
-------------
|
||||
|
||||
Simply copy the file "socks.py" to your Python's `lib/site-packages` directory,
|
||||
and you're ready to go. [Editor's note: it is better to use `python setup.py install` for PySocks]
|
||||
|
||||
|
||||
USAGE
|
||||
------
|
||||
|
||||
First load the socks module with the command:
|
||||
|
||||
>>> import socks
|
||||
>>>
|
||||
|
||||
The socks module provides a class called `socksocket`, which is the base to all of the module's functionality.
|
||||
|
||||
The `socksocket` object has the same initialization parameters as the normal socket
|
||||
object to ensure maximal compatibility, however it should be noted that `socksocket` will only function with family being `AF_INET` and
|
||||
type being either `SOCK_STREAM` or `SOCK_DGRAM`.
|
||||
Generally, it is best to initialize the `socksocket` object with no parameters
|
||||
|
||||
>>> s = socks.socksocket()
|
||||
>>>
|
||||
|
||||
The `socksocket` object has an interface which is very similiar to socket's (in fact
|
||||
the `socksocket` class is derived from socket) with a few extra methods.
|
||||
To select the proxy server you would like to use, use the `set_proxy` method, whose
|
||||
syntax is:
|
||||
|
||||
set_proxy(proxy_type, addr[, port[, rdns[, username[, password]]]])
|
||||
|
||||
Explanation of the parameters:
|
||||
|
||||
`proxy_type` - The type of the proxy server. This can be one of three possible
|
||||
choices: `PROXY_TYPE_SOCKS4`, `PROXY_TYPE_SOCKS5` and `PROXY_TYPE_HTTP` for SOCKS4,
|
||||
SOCKS5 and HTTP servers respectively. `SOCKS4`, `SOCKS5`, and `HTTP` are all aliases, respectively.
|
||||
|
||||
`addr` - The IP address or DNS name of the proxy server.
|
||||
|
||||
`port` - The port of the proxy server. Defaults to 1080 for socks and 8080 for http.
|
||||
|
||||
`rdns` - This is a boolean flag than modifies the behavior regarding DNS resolving.
|
||||
If it is set to True, DNS resolving will be preformed remotely, on the server.
|
||||
If it is set to False, DNS resolving will be preformed locally. Please note that
|
||||
setting this to True with SOCKS4 servers actually use an extension to the protocol,
|
||||
called SOCKS4a, which may not be supported on all servers (SOCKS5 and http servers
|
||||
always support DNS). The default is True.
|
||||
|
||||
`username` - For SOCKS5 servers, this allows simple username / password authentication
|
||||
with the server. For SOCKS4 servers, this parameter will be sent as the userid.
|
||||
This parameter is ignored if an HTTP server is being used. If it is not provided,
|
||||
authentication will not be used (servers may accept unauthenticated requests).
|
||||
|
||||
`password` - This parameter is valid only for SOCKS5 servers and specifies the
|
||||
respective password for the username provided.
|
||||
|
||||
Example of usage:
|
||||
|
||||
>>> s.set_proxy(socks.SOCKS5, "socks.example.com") # uses default port 1080
|
||||
>>> s.set_proxy(socks.SOCKS4, "socks.test.com", 1081)
|
||||
|
||||
After the set_proxy method has been called, simply call the connect method with the
|
||||
traditional parameters to establish a connection through the proxy:
|
||||
|
||||
>>> s.connect(("www.sourceforge.net", 80))
|
||||
>>>
|
||||
|
||||
Connection will take a bit longer to allow negotiation with the proxy server.
|
||||
Please note that calling connect without calling `set_proxy` earlier will connect
|
||||
without a proxy (just like a regular socket).
|
||||
|
||||
Errors: Any errors in the connection process will trigger exceptions. The exception
|
||||
may either be generated by the underlying socket layer or may be custom module
|
||||
exceptions, whose details follow:
|
||||
|
||||
class `ProxyError` - This is a base exception class. It is not raised directly but
|
||||
rather all other exception classes raised by this module are derived from it.
|
||||
This allows an easy way to catch all proxy-related errors. It descends from `IOError`.
|
||||
|
||||
All `ProxyError` exceptions have an attribute `socket_err`, which will contain either a
|
||||
caught `socket.error` exception, or `None` if there wasn't any.
|
||||
|
||||
class `GeneralProxyError` - When thrown, it indicates a problem which does not fall
|
||||
into another category.
|
||||
|
||||
* `Sent invalid data` - This error means that unexpected data has been received from
|
||||
the server. The most common reason is that the server specified as the proxy is
|
||||
not really a SOCKS4/SOCKS5/HTTP proxy, or maybe the proxy type specified is wrong.
|
||||
|
||||
* `Connection closed unexpectedly` - The proxy server unexpectedly closed the connection.
|
||||
This may indicate that the proxy server is experiencing network or software problems.
|
||||
|
||||
* `Bad proxy type` - This will be raised if the type of the proxy supplied to the
|
||||
set_proxy function was not one of `SOCKS4`/`SOCKS5`/`HTTP`.
|
||||
|
||||
* `Bad input` - This will be raised if the `connect()` method is called with bad input
|
||||
parameters.
|
||||
|
||||
class `SOCKS5AuthError` - This indicates that the connection through a SOCKS5 server
|
||||
failed due to an authentication problem.
|
||||
|
||||
* `Authentication is required` - This will happen if you use a SOCKS5 server which
|
||||
requires authentication without providing a username / password at all.
|
||||
|
||||
* `All offered authentication methods were rejected` - This will happen if the proxy
|
||||
requires a special authentication method which is not supported by this module.
|
||||
|
||||
* `Unknown username or invalid password` - Self descriptive.
|
||||
|
||||
class `SOCKS5Error` - This will be raised for SOCKS5 errors which are not related to
|
||||
authentication.
|
||||
The parameter is a tuple containing a code, as given by the server,
|
||||
and a description of the
|
||||
error. The possible errors, according to the RFC, are:
|
||||
|
||||
* `0x01` - General SOCKS server failure - If for any reason the proxy server is unable to
|
||||
fulfill your request (internal server error).
|
||||
* `0x02` - connection not allowed by ruleset - If the address you're trying to connect to
|
||||
is blacklisted on the server or requires authentication.
|
||||
* `0x03` - Network unreachable - The target could not be contacted. A router on the network
|
||||
had replied with a destination net unreachable error.
|
||||
* `0x04` - Host unreachable - The target could not be contacted. A router on the network
|
||||
had replied with a destination host unreachable error.
|
||||
* `0x05` - Connection refused - The target server has actively refused the connection
|
||||
(the requested port is closed).
|
||||
* `0x06` - TTL expired - The TTL value of the SYN packet from the proxy to the target server
|
||||
has expired. This usually means that there are network problems causing the packet
|
||||
to be caught in a router-to-router "ping-pong".
|
||||
* `0x07` - Command not supported - For instance if the server does not support UDP.
|
||||
* `0x08` - Address type not supported - The client has provided an invalid address type.
|
||||
When using this module, this error should not occur.
|
||||
|
||||
class `SOCKS4Error` - This will be raised for SOCKS4 errors. The parameter is a tuple
|
||||
containing a code and a description of the error, as given by the server. The
|
||||
possible error, according to the specification are:
|
||||
|
||||
* `0x5B` - Request rejected or failed - Will be raised in the event of an failure for any
|
||||
reason other then the two mentioned next.
|
||||
* `0x5C` - request rejected because SOCKS server cannot connect to identd on the client -
|
||||
The Socks server had tried an ident lookup on your computer and has failed. In this
|
||||
case you should run an identd server and/or configure your firewall to allow incoming
|
||||
connections to local port 113 from the remote server.
|
||||
* `0x5D` - request rejected because the client program and identd report different user-ids -
|
||||
The Socks server had performed an ident lookup on your computer and has received a
|
||||
different userid than the one you have provided. Change your userid (through the
|
||||
username parameter of the set_proxy method) to match and try again.
|
||||
|
||||
class `HTTPError` - This will be raised for HTTP errors. The message will contain
|
||||
the HTTP status code and provided error message.
|
||||
|
||||
After establishing the connection, the object behaves like a standard socket.
|
||||
|
||||
Methods like `makefile()` and `settimeout()` should behave just like regular sockets.
|
||||
Call the `close()` method to close the connection.
|
||||
|
||||
In addition to the `socksocket` class, an additional function worth mentioning is the
|
||||
`set_default_proxy` function. The parameters are the same as the `set_proxy` method.
|
||||
This function will set default proxy settings for newly created `socksocket` objects,
|
||||
in which the proxy settings haven't been changed via the `set_proxy` method.
|
||||
This is quite useful if you wish to force 3rd party modules to use a SOCKS proxy,
|
||||
by overriding the socket object.
|
||||
For example:
|
||||
|
||||
>>> socks.set_default_proxy(socks.SOCKS5, "socks.example.com")
|
||||
>>> socket.socket = socks.socksocket
|
||||
>>> urllib.urlopen("http://www.sourceforge.net/")
|
||||
|
||||
|
||||
PROBLEMS
|
||||
---------
|
||||
|
||||
Please open a GitHub issue at https://github.com/Anorov/PySocks
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
PySocks-1.7.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
PySocks-1.7.1.dist-info/LICENSE,sha256=cCfiFOAU63i3rcwc7aWspxOnn8T2oMUsnaWz5wfm_-k,1401
|
||||
PySocks-1.7.1.dist-info/METADATA,sha256=zbQMizjPOOP4DhEiEX24XXjNrYuIxF9UGUpN0uFDB6Y,13235
|
||||
PySocks-1.7.1.dist-info/RECORD,,
|
||||
PySocks-1.7.1.dist-info/WHEEL,sha256=t_MpApv386-8PVts2R6wsTifdIn0vbUDTVv61IbqFC8,92
|
||||
PySocks-1.7.1.dist-info/top_level.txt,sha256=TKSOIfCFBoK9EY8FBYbYqC3PWd3--G15ph9n8-QHPDk,19
|
||||
__pycache__/socks.cpython-312.pyc,,
|
||||
__pycache__/sockshandler.cpython-312.pyc,,
|
||||
socks.py,sha256=xOYn27t9IGrbTBzWsUUuPa0YBuplgiUykzkOB5V5iFY,31086
|
||||
sockshandler.py,sha256=2SYGj-pwt1kjgLoZAmyeaEXCeZDWRmfVS_QG6kErGtY,3966
|
||||
@@ -0,0 +1,5 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: bdist_wheel (0.33.3)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
socks
|
||||
sockshandler
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Classes Without Boilerplate
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Literal, Protocol
|
||||
|
||||
from . import converters, exceptions, filters, setters, validators
|
||||
from ._cmp import cmp_using
|
||||
from ._config import get_run_validators, set_run_validators
|
||||
from ._funcs import asdict, assoc, astuple, has, resolve_types
|
||||
from ._make import (
|
||||
NOTHING,
|
||||
Attribute,
|
||||
Converter,
|
||||
Factory,
|
||||
_Nothing,
|
||||
attrib,
|
||||
attrs,
|
||||
evolve,
|
||||
fields,
|
||||
fields_dict,
|
||||
make_class,
|
||||
validate,
|
||||
)
|
||||
from ._next_gen import define, field, frozen, mutable
|
||||
from ._version_info import VersionInfo
|
||||
|
||||
|
||||
s = attributes = attrs
|
||||
ib = attr = attrib
|
||||
dataclass = partial(attrs, auto_attribs=True) # happy Easter ;)
|
||||
|
||||
|
||||
class AttrsInstance(Protocol):
|
||||
pass
|
||||
|
||||
|
||||
NothingType = Literal[_Nothing.NOTHING]
|
||||
|
||||
__all__ = [
|
||||
"NOTHING",
|
||||
"Attribute",
|
||||
"AttrsInstance",
|
||||
"Converter",
|
||||
"Factory",
|
||||
"NothingType",
|
||||
"asdict",
|
||||
"assoc",
|
||||
"astuple",
|
||||
"attr",
|
||||
"attrib",
|
||||
"attributes",
|
||||
"attrs",
|
||||
"cmp_using",
|
||||
"converters",
|
||||
"define",
|
||||
"evolve",
|
||||
"exceptions",
|
||||
"field",
|
||||
"fields",
|
||||
"fields_dict",
|
||||
"filters",
|
||||
"frozen",
|
||||
"get_run_validators",
|
||||
"has",
|
||||
"ib",
|
||||
"make_class",
|
||||
"mutable",
|
||||
"resolve_types",
|
||||
"s",
|
||||
"set_run_validators",
|
||||
"setters",
|
||||
"validate",
|
||||
"validators",
|
||||
]
|
||||
|
||||
|
||||
def _make_getattr(mod_name: str) -> Callable:
|
||||
"""
|
||||
Create a metadata proxy for packaging information that uses *mod_name* in
|
||||
its warnings and errors.
|
||||
"""
|
||||
|
||||
def __getattr__(name: str) -> str:
|
||||
if name not in ("__version__", "__version_info__"):
|
||||
msg = f"module {mod_name} has no attribute {name}"
|
||||
raise AttributeError(msg)
|
||||
|
||||
from importlib.metadata import metadata
|
||||
|
||||
meta = metadata("attrs")
|
||||
|
||||
if name == "__version_info__":
|
||||
return VersionInfo._from_version_string(meta["version"])
|
||||
|
||||
return meta["version"]
|
||||
|
||||
return __getattr__
|
||||
|
||||
|
||||
__getattr__ = _make_getattr(__name__)
|
||||
@@ -0,0 +1,389 @@
|
||||
import enum
|
||||
import sys
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Generic,
|
||||
Literal,
|
||||
Mapping,
|
||||
Protocol,
|
||||
Sequence,
|
||||
TypeVar,
|
||||
overload,
|
||||
)
|
||||
|
||||
# `import X as X` is required to make these public
|
||||
from . import converters as converters
|
||||
from . import exceptions as exceptions
|
||||
from . import filters as filters
|
||||
from . import setters as setters
|
||||
from . import validators as validators
|
||||
from ._cmp import cmp_using as cmp_using
|
||||
from ._typing_compat import AttrsInstance_
|
||||
from ._version_info import VersionInfo
|
||||
from attrs import (
|
||||
define as define,
|
||||
field as field,
|
||||
mutable as mutable,
|
||||
frozen as frozen,
|
||||
_EqOrderType,
|
||||
_ValidatorType,
|
||||
_ConverterType,
|
||||
_ReprArgType,
|
||||
_OnSetAttrType,
|
||||
_OnSetAttrArgType,
|
||||
_FieldTransformer,
|
||||
_ValidatorArgType,
|
||||
)
|
||||
|
||||
if sys.version_info >= (3, 10):
|
||||
from typing import TypeGuard, TypeAlias
|
||||
else:
|
||||
from typing_extensions import TypeGuard, TypeAlias
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
from typing import dataclass_transform
|
||||
else:
|
||||
from typing_extensions import dataclass_transform
|
||||
|
||||
__version__: str
|
||||
__version_info__: VersionInfo
|
||||
__title__: str
|
||||
__description__: str
|
||||
__url__: str
|
||||
__uri__: str
|
||||
__author__: str
|
||||
__email__: str
|
||||
__license__: str
|
||||
__copyright__: str
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_C = TypeVar("_C", bound=type)
|
||||
|
||||
_FilterType = Callable[["Attribute[_T]", _T], bool]
|
||||
|
||||
# We subclass this here to keep the protocol's qualified name clean.
|
||||
class AttrsInstance(AttrsInstance_, Protocol):
|
||||
pass
|
||||
|
||||
_A = TypeVar("_A", bound=type[AttrsInstance])
|
||||
|
||||
class _Nothing(enum.Enum):
|
||||
NOTHING = enum.auto()
|
||||
|
||||
NOTHING = _Nothing.NOTHING
|
||||
NothingType: TypeAlias = Literal[_Nothing.NOTHING]
|
||||
|
||||
# NOTE: Factory lies about its return type to make this possible:
|
||||
# `x: List[int] # = Factory(list)`
|
||||
# Work around mypy issue #4554 in the common case by using an overload.
|
||||
|
||||
@overload
|
||||
def Factory(factory: Callable[[], _T]) -> _T: ...
|
||||
@overload
|
||||
def Factory(
|
||||
factory: Callable[[Any], _T],
|
||||
takes_self: Literal[True],
|
||||
) -> _T: ...
|
||||
@overload
|
||||
def Factory(
|
||||
factory: Callable[[], _T],
|
||||
takes_self: Literal[False],
|
||||
) -> _T: ...
|
||||
|
||||
In = TypeVar("In")
|
||||
Out = TypeVar("Out")
|
||||
|
||||
class Converter(Generic[In, Out]):
|
||||
@overload
|
||||
def __init__(self, converter: Callable[[In], Out]) -> None: ...
|
||||
@overload
|
||||
def __init__(
|
||||
self,
|
||||
converter: Callable[[In, AttrsInstance, Attribute], Out],
|
||||
*,
|
||||
takes_self: Literal[True],
|
||||
takes_field: Literal[True],
|
||||
) -> None: ...
|
||||
@overload
|
||||
def __init__(
|
||||
self,
|
||||
converter: Callable[[In, Attribute], Out],
|
||||
*,
|
||||
takes_field: Literal[True],
|
||||
) -> None: ...
|
||||
@overload
|
||||
def __init__(
|
||||
self,
|
||||
converter: Callable[[In, AttrsInstance], Out],
|
||||
*,
|
||||
takes_self: Literal[True],
|
||||
) -> None: ...
|
||||
|
||||
class Attribute(Generic[_T]):
|
||||
name: str
|
||||
default: _T | None
|
||||
validator: _ValidatorType[_T] | None
|
||||
repr: _ReprArgType
|
||||
cmp: _EqOrderType
|
||||
eq: _EqOrderType
|
||||
order: _EqOrderType
|
||||
hash: bool | None
|
||||
init: bool
|
||||
converter: Converter | None
|
||||
metadata: dict[Any, Any]
|
||||
type: type[_T] | None
|
||||
kw_only: bool
|
||||
on_setattr: _OnSetAttrType
|
||||
alias: str | None
|
||||
|
||||
def evolve(self, **changes: Any) -> "Attribute[Any]": ...
|
||||
|
||||
# NOTE: We had several choices for the annotation to use for type arg:
|
||||
# 1) Type[_T]
|
||||
# - Pros: Handles simple cases correctly
|
||||
# - Cons: Might produce less informative errors in the case of conflicting
|
||||
# TypeVars e.g. `attr.ib(default='bad', type=int)`
|
||||
# 2) Callable[..., _T]
|
||||
# - Pros: Better error messages than #1 for conflicting TypeVars
|
||||
# - Cons: Terrible error messages for validator checks.
|
||||
# e.g. attr.ib(type=int, validator=validate_str)
|
||||
# -> error: Cannot infer function type argument
|
||||
# 3) type (and do all of the work in the mypy plugin)
|
||||
# - Pros: Simple here, and we could customize the plugin with our own errors.
|
||||
# - Cons: Would need to write mypy plugin code to handle all the cases.
|
||||
# We chose option #1.
|
||||
|
||||
# `attr` lies about its return type to make the following possible:
|
||||
# attr() -> Any
|
||||
# attr(8) -> int
|
||||
# attr(validator=<some callable>) -> Whatever the callable expects.
|
||||
# This makes this type of assignments possible:
|
||||
# x: int = attr(8)
|
||||
#
|
||||
# This form catches explicit None or no default but with no other arguments
|
||||
# returns Any.
|
||||
@overload
|
||||
def attrib(
|
||||
default: None = ...,
|
||||
validator: None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
type: None = ...,
|
||||
converter: None = ...,
|
||||
factory: None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
) -> Any: ...
|
||||
|
||||
# This form catches an explicit None or no default and infers the type from the
|
||||
# other arguments.
|
||||
@overload
|
||||
def attrib(
|
||||
default: None = ...,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
type: type[_T] | None = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
) -> _T: ...
|
||||
|
||||
# This form catches an explicit default argument.
|
||||
@overload
|
||||
def attrib(
|
||||
default: _T,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
type: type[_T] | None = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
) -> _T: ...
|
||||
|
||||
# This form covers type=non-Type: e.g. forward references (str), Any
|
||||
@overload
|
||||
def attrib(
|
||||
default: _T | None = ...,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
type: object = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
) -> Any: ...
|
||||
@overload
|
||||
@dataclass_transform(order_default=True, field_specifiers=(attrib, field))
|
||||
def attrs(
|
||||
maybe_cls: _C,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr_ns: str | None = ...,
|
||||
repr: bool = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
collect_by_mro: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
) -> _C: ...
|
||||
@overload
|
||||
@dataclass_transform(order_default=True, field_specifiers=(attrib, field))
|
||||
def attrs(
|
||||
maybe_cls: None = ...,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr_ns: str | None = ...,
|
||||
repr: bool = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
collect_by_mro: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
) -> Callable[[_C], _C]: ...
|
||||
def fields(cls: type[AttrsInstance] | AttrsInstance) -> Any: ...
|
||||
def fields_dict(cls: type[AttrsInstance]) -> dict[str, Attribute[Any]]: ...
|
||||
def validate(inst: AttrsInstance) -> None: ...
|
||||
def resolve_types(
|
||||
cls: _A,
|
||||
globalns: dict[str, Any] | None = ...,
|
||||
localns: dict[str, Any] | None = ...,
|
||||
attribs: list[Attribute[Any]] | None = ...,
|
||||
include_extras: bool = ...,
|
||||
) -> _A: ...
|
||||
|
||||
# TODO: add support for returning a proper attrs class from the mypy plugin
|
||||
# we use Any instead of _CountingAttr so that e.g. `make_class('Foo',
|
||||
# [attr.ib()])` is valid
|
||||
def make_class(
|
||||
name: str,
|
||||
attrs: list[str] | tuple[str, ...] | dict[str, Any],
|
||||
bases: tuple[type, ...] = ...,
|
||||
class_body: dict[str, Any] | None = ...,
|
||||
repr_ns: str | None = ...,
|
||||
repr: bool = ...,
|
||||
cmp: _EqOrderType | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
collect_by_mro: bool = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
) -> type: ...
|
||||
|
||||
# _funcs --
|
||||
|
||||
# TODO: add support for returning TypedDict from the mypy plugin
|
||||
# FIXME: asdict/astuple do not honor their factory args. Waiting on one of
|
||||
# these:
|
||||
# https://github.com/python/mypy/issues/4236
|
||||
# https://github.com/python/typing/issues/253
|
||||
# XXX: remember to fix attrs.asdict/astuple too!
|
||||
def asdict(
|
||||
inst: AttrsInstance,
|
||||
recurse: bool = ...,
|
||||
filter: _FilterType[Any] | None = ...,
|
||||
dict_factory: type[Mapping[Any, Any]] = ...,
|
||||
retain_collection_types: bool = ...,
|
||||
value_serializer: Callable[[type, Attribute[Any], Any], Any] | None = ...,
|
||||
tuple_keys: bool | None = ...,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
# TODO: add support for returning NamedTuple from the mypy plugin
|
||||
def astuple(
|
||||
inst: AttrsInstance,
|
||||
recurse: bool = ...,
|
||||
filter: _FilterType[Any] | None = ...,
|
||||
tuple_factory: type[Sequence[Any]] = ...,
|
||||
retain_collection_types: bool = ...,
|
||||
) -> tuple[Any, ...]: ...
|
||||
def has(cls: type) -> TypeGuard[type[AttrsInstance]]: ...
|
||||
def assoc(inst: _T, **changes: Any) -> _T: ...
|
||||
def evolve(inst: _T, **changes: Any) -> _T: ...
|
||||
|
||||
# _config --
|
||||
|
||||
def set_run_validators(run: bool) -> None: ...
|
||||
def get_run_validators() -> bool: ...
|
||||
|
||||
# aliases --
|
||||
|
||||
s = attributes = attrs
|
||||
ib = attr = attrib
|
||||
dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
import functools
|
||||
import types
|
||||
|
||||
from ._make import __ne__
|
||||
|
||||
|
||||
_operation_names = {"eq": "==", "lt": "<", "le": "<=", "gt": ">", "ge": ">="}
|
||||
|
||||
|
||||
def cmp_using(
|
||||
eq=None,
|
||||
lt=None,
|
||||
le=None,
|
||||
gt=None,
|
||||
ge=None,
|
||||
require_same_type=True,
|
||||
class_name="Comparable",
|
||||
):
|
||||
"""
|
||||
Create a class that can be passed into `attrs.field`'s ``eq``, ``order``,
|
||||
and ``cmp`` arguments to customize field comparison.
|
||||
|
||||
The resulting class will have a full set of ordering methods if at least
|
||||
one of ``{lt, le, gt, ge}`` and ``eq`` are provided.
|
||||
|
||||
Args:
|
||||
eq (typing.Callable | None):
|
||||
Callable used to evaluate equality of two objects.
|
||||
|
||||
lt (typing.Callable | None):
|
||||
Callable used to evaluate whether one object is less than another
|
||||
object.
|
||||
|
||||
le (typing.Callable | None):
|
||||
Callable used to evaluate whether one object is less than or equal
|
||||
to another object.
|
||||
|
||||
gt (typing.Callable | None):
|
||||
Callable used to evaluate whether one object is greater than
|
||||
another object.
|
||||
|
||||
ge (typing.Callable | None):
|
||||
Callable used to evaluate whether one object is greater than or
|
||||
equal to another object.
|
||||
|
||||
require_same_type (bool):
|
||||
When `True`, equality and ordering methods will return
|
||||
`NotImplemented` if objects are not of the same type.
|
||||
|
||||
class_name (str | None): Name of class. Defaults to "Comparable".
|
||||
|
||||
See `comparison` for more details.
|
||||
|
||||
.. versionadded:: 21.1.0
|
||||
"""
|
||||
|
||||
body = {
|
||||
"__slots__": ["value"],
|
||||
"__init__": _make_init(),
|
||||
"_requirements": [],
|
||||
"_is_comparable_to": _is_comparable_to,
|
||||
}
|
||||
|
||||
# Add operations.
|
||||
num_order_functions = 0
|
||||
has_eq_function = False
|
||||
|
||||
if eq is not None:
|
||||
has_eq_function = True
|
||||
body["__eq__"] = _make_operator("eq", eq)
|
||||
body["__ne__"] = __ne__
|
||||
|
||||
if lt is not None:
|
||||
num_order_functions += 1
|
||||
body["__lt__"] = _make_operator("lt", lt)
|
||||
|
||||
if le is not None:
|
||||
num_order_functions += 1
|
||||
body["__le__"] = _make_operator("le", le)
|
||||
|
||||
if gt is not None:
|
||||
num_order_functions += 1
|
||||
body["__gt__"] = _make_operator("gt", gt)
|
||||
|
||||
if ge is not None:
|
||||
num_order_functions += 1
|
||||
body["__ge__"] = _make_operator("ge", ge)
|
||||
|
||||
type_ = types.new_class(
|
||||
class_name, (object,), {}, lambda ns: ns.update(body)
|
||||
)
|
||||
|
||||
# Add same type requirement.
|
||||
if require_same_type:
|
||||
type_._requirements.append(_check_same_type)
|
||||
|
||||
# Add total ordering if at least one operation was defined.
|
||||
if 0 < num_order_functions < 4:
|
||||
if not has_eq_function:
|
||||
# functools.total_ordering requires __eq__ to be defined,
|
||||
# so raise early error here to keep a nice stack.
|
||||
msg = "eq must be define is order to complete ordering from lt, le, gt, ge."
|
||||
raise ValueError(msg)
|
||||
type_ = functools.total_ordering(type_)
|
||||
|
||||
return type_
|
||||
|
||||
|
||||
def _make_init():
|
||||
"""
|
||||
Create __init__ method.
|
||||
"""
|
||||
|
||||
def __init__(self, value):
|
||||
"""
|
||||
Initialize object with *value*.
|
||||
"""
|
||||
self.value = value
|
||||
|
||||
return __init__
|
||||
|
||||
|
||||
def _make_operator(name, func):
|
||||
"""
|
||||
Create operator method.
|
||||
"""
|
||||
|
||||
def method(self, other):
|
||||
if not self._is_comparable_to(other):
|
||||
return NotImplemented
|
||||
|
||||
result = func(self.value, other.value)
|
||||
if result is NotImplemented:
|
||||
return NotImplemented
|
||||
|
||||
return result
|
||||
|
||||
method.__name__ = f"__{name}__"
|
||||
method.__doc__ = (
|
||||
f"Return a {_operation_names[name]} b. Computed by attrs."
|
||||
)
|
||||
|
||||
return method
|
||||
|
||||
|
||||
def _is_comparable_to(self, other):
|
||||
"""
|
||||
Check whether `other` is comparable to `self`.
|
||||
"""
|
||||
return all(func(self, other) for func in self._requirements)
|
||||
|
||||
|
||||
def _check_same_type(self, other):
|
||||
"""
|
||||
Return True if *self* and *other* are of the same type, False otherwise.
|
||||
"""
|
||||
return other.value.__class__ is self.value.__class__
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import Any, Callable
|
||||
|
||||
_CompareWithType = Callable[[Any, Any], bool]
|
||||
|
||||
def cmp_using(
|
||||
eq: _CompareWithType | None = ...,
|
||||
lt: _CompareWithType | None = ...,
|
||||
le: _CompareWithType | None = ...,
|
||||
gt: _CompareWithType | None = ...,
|
||||
ge: _CompareWithType | None = ...,
|
||||
require_same_type: bool = ...,
|
||||
class_name: str = ...,
|
||||
) -> type: ...
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import inspect
|
||||
import platform
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from collections.abc import Mapping, Sequence # noqa: F401
|
||||
from typing import _GenericAlias
|
||||
|
||||
|
||||
PYPY = platform.python_implementation() == "PyPy"
|
||||
PY_3_10_PLUS = sys.version_info[:2] >= (3, 10)
|
||||
PY_3_11_PLUS = sys.version_info[:2] >= (3, 11)
|
||||
PY_3_12_PLUS = sys.version_info[:2] >= (3, 12)
|
||||
PY_3_13_PLUS = sys.version_info[:2] >= (3, 13)
|
||||
PY_3_14_PLUS = sys.version_info[:2] >= (3, 14)
|
||||
|
||||
|
||||
if PY_3_14_PLUS:
|
||||
import annotationlib
|
||||
|
||||
# We request forward-ref annotations to not break in the presence of
|
||||
# forward references.
|
||||
|
||||
def _get_annotations(cls):
|
||||
return annotationlib.get_annotations(
|
||||
cls, format=annotationlib.Format.FORWARDREF
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def _get_annotations(cls):
|
||||
"""
|
||||
Get annotations for *cls*.
|
||||
"""
|
||||
return cls.__dict__.get("__annotations__", {})
|
||||
|
||||
|
||||
class _AnnotationExtractor:
|
||||
"""
|
||||
Extract type annotations from a callable, returning None whenever there
|
||||
is none.
|
||||
"""
|
||||
|
||||
__slots__ = ["sig"]
|
||||
|
||||
def __init__(self, callable):
|
||||
try:
|
||||
self.sig = inspect.signature(callable)
|
||||
except (ValueError, TypeError): # inspect failed
|
||||
self.sig = None
|
||||
|
||||
def get_first_param_type(self):
|
||||
"""
|
||||
Return the type annotation of the first argument if it's not empty.
|
||||
"""
|
||||
if not self.sig:
|
||||
return None
|
||||
|
||||
params = list(self.sig.parameters.values())
|
||||
if params and params[0].annotation is not inspect.Parameter.empty:
|
||||
return params[0].annotation
|
||||
|
||||
return None
|
||||
|
||||
def get_return_type(self):
|
||||
"""
|
||||
Return the return type if it's not empty.
|
||||
"""
|
||||
if (
|
||||
self.sig
|
||||
and self.sig.return_annotation is not inspect.Signature.empty
|
||||
):
|
||||
return self.sig.return_annotation
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Thread-local global to track attrs instances which are already being repr'd.
|
||||
# This is needed because there is no other (thread-safe) way to pass info
|
||||
# about the instances that are already being repr'd through the call stack
|
||||
# in order to ensure we don't perform infinite recursion.
|
||||
#
|
||||
# For instance, if an instance contains a dict which contains that instance,
|
||||
# we need to know that we're already repr'ing the outside instance from within
|
||||
# the dict's repr() call.
|
||||
#
|
||||
# This lives here rather than in _make.py so that the functions in _make.py
|
||||
# don't have a direct reference to the thread-local in their globals dict.
|
||||
# If they have such a reference, it breaks cloudpickle.
|
||||
repr_context = threading.local()
|
||||
|
||||
|
||||
def get_generic_base(cl):
|
||||
"""If this is a generic class (A[str]), return the generic base for it."""
|
||||
if cl.__class__ is _GenericAlias:
|
||||
return cl.__origin__
|
||||
return None
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
__all__ = ["get_run_validators", "set_run_validators"]
|
||||
|
||||
_run_validators = True
|
||||
|
||||
|
||||
def set_run_validators(run):
|
||||
"""
|
||||
Set whether or not validators are run. By default, they are run.
|
||||
|
||||
.. deprecated:: 21.3.0 It will not be removed, but it also will not be
|
||||
moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()`
|
||||
instead.
|
||||
"""
|
||||
if not isinstance(run, bool):
|
||||
msg = "'run' must be bool."
|
||||
raise TypeError(msg)
|
||||
global _run_validators
|
||||
_run_validators = run
|
||||
|
||||
|
||||
def get_run_validators():
|
||||
"""
|
||||
Return whether or not validators are run.
|
||||
|
||||
.. deprecated:: 21.3.0 It will not be removed, but it also will not be
|
||||
moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()`
|
||||
instead.
|
||||
"""
|
||||
return _run_validators
|
||||
@@ -0,0 +1,497 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
import copy
|
||||
|
||||
from ._compat import get_generic_base
|
||||
from ._make import _OBJ_SETATTR, NOTHING, fields
|
||||
from .exceptions import AttrsAttributeNotFoundError
|
||||
|
||||
|
||||
_ATOMIC_TYPES = frozenset(
|
||||
{
|
||||
type(None),
|
||||
bool,
|
||||
int,
|
||||
float,
|
||||
str,
|
||||
complex,
|
||||
bytes,
|
||||
type(...),
|
||||
type,
|
||||
range,
|
||||
property,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def asdict(
|
||||
inst,
|
||||
recurse=True,
|
||||
filter=None,
|
||||
dict_factory=dict,
|
||||
retain_collection_types=False,
|
||||
value_serializer=None,
|
||||
):
|
||||
"""
|
||||
Return the *attrs* attribute values of *inst* as a dict.
|
||||
|
||||
Optionally recurse into other *attrs*-decorated classes.
|
||||
|
||||
Args:
|
||||
inst: Instance of an *attrs*-decorated class.
|
||||
|
||||
recurse (bool): Recurse into classes that are also *attrs*-decorated.
|
||||
|
||||
filter (~typing.Callable):
|
||||
A callable whose return code determines whether an attribute or
|
||||
element is included (`True`) or dropped (`False`). Is called with
|
||||
the `attrs.Attribute` as the first argument and the value as the
|
||||
second argument.
|
||||
|
||||
dict_factory (~typing.Callable):
|
||||
A callable to produce dictionaries from. For example, to produce
|
||||
ordered dictionaries instead of normal Python dictionaries, pass in
|
||||
``collections.OrderedDict``.
|
||||
|
||||
retain_collection_types (bool):
|
||||
Do not convert to `list` when encountering an attribute whose type
|
||||
is `tuple` or `set`. Only meaningful if *recurse* is `True`.
|
||||
|
||||
value_serializer (typing.Callable | None):
|
||||
A hook that is called for every attribute or dict key/value. It
|
||||
receives the current instance, field and value and must return the
|
||||
(updated) value. The hook is run *after* the optional *filter* has
|
||||
been applied.
|
||||
|
||||
Returns:
|
||||
Return type of *dict_factory*.
|
||||
|
||||
Raises:
|
||||
attrs.exceptions.NotAnAttrsClassError:
|
||||
If *cls* is not an *attrs* class.
|
||||
|
||||
.. versionadded:: 16.0.0 *dict_factory*
|
||||
.. versionadded:: 16.1.0 *retain_collection_types*
|
||||
.. versionadded:: 20.3.0 *value_serializer*
|
||||
.. versionadded:: 21.3.0
|
||||
If a dict has a collection for a key, it is serialized as a tuple.
|
||||
"""
|
||||
attrs = fields(inst.__class__)
|
||||
rv = dict_factory()
|
||||
for a in attrs:
|
||||
v = getattr(inst, a.name)
|
||||
if filter is not None and not filter(a, v):
|
||||
continue
|
||||
|
||||
if value_serializer is not None:
|
||||
v = value_serializer(inst, a, v)
|
||||
|
||||
if recurse is True:
|
||||
value_type = type(v)
|
||||
if value_type in _ATOMIC_TYPES:
|
||||
rv[a.name] = v
|
||||
elif has(value_type):
|
||||
rv[a.name] = asdict(
|
||||
v,
|
||||
recurse=True,
|
||||
filter=filter,
|
||||
dict_factory=dict_factory,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
)
|
||||
elif issubclass(value_type, (tuple, list, set, frozenset)):
|
||||
cf = value_type if retain_collection_types is True else list
|
||||
items = [
|
||||
_asdict_anything(
|
||||
i,
|
||||
is_key=False,
|
||||
filter=filter,
|
||||
dict_factory=dict_factory,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
)
|
||||
for i in v
|
||||
]
|
||||
try:
|
||||
rv[a.name] = cf(items)
|
||||
except TypeError:
|
||||
if not issubclass(cf, tuple):
|
||||
raise
|
||||
# Workaround for TypeError: cf.__new__() missing 1 required
|
||||
# positional argument (which appears, for a namedturle)
|
||||
rv[a.name] = cf(*items)
|
||||
elif issubclass(value_type, dict):
|
||||
df = dict_factory
|
||||
rv[a.name] = df(
|
||||
(
|
||||
_asdict_anything(
|
||||
kk,
|
||||
is_key=True,
|
||||
filter=filter,
|
||||
dict_factory=df,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
),
|
||||
_asdict_anything(
|
||||
vv,
|
||||
is_key=False,
|
||||
filter=filter,
|
||||
dict_factory=df,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
),
|
||||
)
|
||||
for kk, vv in v.items()
|
||||
)
|
||||
else:
|
||||
rv[a.name] = v
|
||||
else:
|
||||
rv[a.name] = v
|
||||
return rv
|
||||
|
||||
|
||||
def _asdict_anything(
|
||||
val,
|
||||
is_key,
|
||||
filter,
|
||||
dict_factory,
|
||||
retain_collection_types,
|
||||
value_serializer,
|
||||
):
|
||||
"""
|
||||
``asdict`` only works on attrs instances, this works on anything.
|
||||
"""
|
||||
val_type = type(val)
|
||||
if val_type in _ATOMIC_TYPES:
|
||||
rv = val
|
||||
if value_serializer is not None:
|
||||
rv = value_serializer(None, None, rv)
|
||||
elif getattr(val_type, "__attrs_attrs__", None) is not None:
|
||||
# Attrs class.
|
||||
rv = asdict(
|
||||
val,
|
||||
recurse=True,
|
||||
filter=filter,
|
||||
dict_factory=dict_factory,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
)
|
||||
elif issubclass(val_type, (tuple, list, set, frozenset)):
|
||||
if retain_collection_types is True:
|
||||
cf = val.__class__
|
||||
elif is_key:
|
||||
cf = tuple
|
||||
else:
|
||||
cf = list
|
||||
|
||||
rv = cf(
|
||||
[
|
||||
_asdict_anything(
|
||||
i,
|
||||
is_key=False,
|
||||
filter=filter,
|
||||
dict_factory=dict_factory,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
)
|
||||
for i in val
|
||||
]
|
||||
)
|
||||
elif issubclass(val_type, dict):
|
||||
df = dict_factory
|
||||
rv = df(
|
||||
(
|
||||
_asdict_anything(
|
||||
kk,
|
||||
is_key=True,
|
||||
filter=filter,
|
||||
dict_factory=df,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
),
|
||||
_asdict_anything(
|
||||
vv,
|
||||
is_key=False,
|
||||
filter=filter,
|
||||
dict_factory=df,
|
||||
retain_collection_types=retain_collection_types,
|
||||
value_serializer=value_serializer,
|
||||
),
|
||||
)
|
||||
for kk, vv in val.items()
|
||||
)
|
||||
else:
|
||||
rv = val
|
||||
if value_serializer is not None:
|
||||
rv = value_serializer(None, None, rv)
|
||||
|
||||
return rv
|
||||
|
||||
|
||||
def astuple(
|
||||
inst,
|
||||
recurse=True,
|
||||
filter=None,
|
||||
tuple_factory=tuple,
|
||||
retain_collection_types=False,
|
||||
):
|
||||
"""
|
||||
Return the *attrs* attribute values of *inst* as a tuple.
|
||||
|
||||
Optionally recurse into other *attrs*-decorated classes.
|
||||
|
||||
Args:
|
||||
inst: Instance of an *attrs*-decorated class.
|
||||
|
||||
recurse (bool):
|
||||
Recurse into classes that are also *attrs*-decorated.
|
||||
|
||||
filter (~typing.Callable):
|
||||
A callable whose return code determines whether an attribute or
|
||||
element is included (`True`) or dropped (`False`). Is called with
|
||||
the `attrs.Attribute` as the first argument and the value as the
|
||||
second argument.
|
||||
|
||||
tuple_factory (~typing.Callable):
|
||||
A callable to produce tuples from. For example, to produce lists
|
||||
instead of tuples.
|
||||
|
||||
retain_collection_types (bool):
|
||||
Do not convert to `list` or `dict` when encountering an attribute
|
||||
which type is `tuple`, `dict` or `set`. Only meaningful if
|
||||
*recurse* is `True`.
|
||||
|
||||
Returns:
|
||||
Return type of *tuple_factory*
|
||||
|
||||
Raises:
|
||||
attrs.exceptions.NotAnAttrsClassError:
|
||||
If *cls* is not an *attrs* class.
|
||||
|
||||
.. versionadded:: 16.2.0
|
||||
"""
|
||||
attrs = fields(inst.__class__)
|
||||
rv = []
|
||||
retain = retain_collection_types # Very long. :/
|
||||
for a in attrs:
|
||||
v = getattr(inst, a.name)
|
||||
if filter is not None and not filter(a, v):
|
||||
continue
|
||||
value_type = type(v)
|
||||
if recurse is True:
|
||||
if value_type in _ATOMIC_TYPES:
|
||||
rv.append(v)
|
||||
elif has(value_type):
|
||||
rv.append(
|
||||
astuple(
|
||||
v,
|
||||
recurse=True,
|
||||
filter=filter,
|
||||
tuple_factory=tuple_factory,
|
||||
retain_collection_types=retain,
|
||||
)
|
||||
)
|
||||
elif issubclass(value_type, (tuple, list, set, frozenset)):
|
||||
cf = v.__class__ if retain is True else list
|
||||
items = [
|
||||
(
|
||||
astuple(
|
||||
j,
|
||||
recurse=True,
|
||||
filter=filter,
|
||||
tuple_factory=tuple_factory,
|
||||
retain_collection_types=retain,
|
||||
)
|
||||
if has(j.__class__)
|
||||
else j
|
||||
)
|
||||
for j in v
|
||||
]
|
||||
try:
|
||||
rv.append(cf(items))
|
||||
except TypeError:
|
||||
if not issubclass(cf, tuple):
|
||||
raise
|
||||
# Workaround for TypeError: cf.__new__() missing 1 required
|
||||
# positional argument (which appears, for a namedturle)
|
||||
rv.append(cf(*items))
|
||||
elif issubclass(value_type, dict):
|
||||
df = value_type if retain is True else dict
|
||||
rv.append(
|
||||
df(
|
||||
(
|
||||
(
|
||||
astuple(
|
||||
kk,
|
||||
tuple_factory=tuple_factory,
|
||||
retain_collection_types=retain,
|
||||
)
|
||||
if has(kk.__class__)
|
||||
else kk
|
||||
),
|
||||
(
|
||||
astuple(
|
||||
vv,
|
||||
tuple_factory=tuple_factory,
|
||||
retain_collection_types=retain,
|
||||
)
|
||||
if has(vv.__class__)
|
||||
else vv
|
||||
),
|
||||
)
|
||||
for kk, vv in v.items()
|
||||
)
|
||||
)
|
||||
else:
|
||||
rv.append(v)
|
||||
else:
|
||||
rv.append(v)
|
||||
|
||||
return rv if tuple_factory is list else tuple_factory(rv)
|
||||
|
||||
|
||||
def has(cls):
|
||||
"""
|
||||
Check whether *cls* is a class with *attrs* attributes.
|
||||
|
||||
Args:
|
||||
cls (type): Class to introspect.
|
||||
|
||||
Raises:
|
||||
TypeError: If *cls* is not a class.
|
||||
|
||||
Returns:
|
||||
bool:
|
||||
"""
|
||||
attrs = getattr(cls, "__attrs_attrs__", None)
|
||||
if attrs is not None:
|
||||
return True
|
||||
|
||||
# No attrs, maybe it's a specialized generic (A[str])?
|
||||
generic_base = get_generic_base(cls)
|
||||
if generic_base is not None:
|
||||
generic_attrs = getattr(generic_base, "__attrs_attrs__", None)
|
||||
if generic_attrs is not None:
|
||||
# Stick it on here for speed next time.
|
||||
cls.__attrs_attrs__ = generic_attrs
|
||||
return generic_attrs is not None
|
||||
return False
|
||||
|
||||
|
||||
def assoc(inst, **changes):
|
||||
"""
|
||||
Copy *inst* and apply *changes*.
|
||||
|
||||
This is different from `evolve` that applies the changes to the arguments
|
||||
that create the new instance.
|
||||
|
||||
`evolve`'s behavior is preferable, but there are `edge cases`_ where it
|
||||
doesn't work. Therefore `assoc` is deprecated, but will not be removed.
|
||||
|
||||
.. _`edge cases`: https://github.com/python-attrs/attrs/issues/251
|
||||
|
||||
Args:
|
||||
inst: Instance of a class with *attrs* attributes.
|
||||
|
||||
changes: Keyword changes in the new copy.
|
||||
|
||||
Returns:
|
||||
A copy of inst with *changes* incorporated.
|
||||
|
||||
Raises:
|
||||
attrs.exceptions.AttrsAttributeNotFoundError:
|
||||
If *attr_name* couldn't be found on *cls*.
|
||||
|
||||
attrs.exceptions.NotAnAttrsClassError:
|
||||
If *cls* is not an *attrs* class.
|
||||
|
||||
.. deprecated:: 17.1.0
|
||||
Use `attrs.evolve` instead if you can. This function will not be
|
||||
removed du to the slightly different approach compared to
|
||||
`attrs.evolve`, though.
|
||||
"""
|
||||
new = copy.copy(inst)
|
||||
attrs = fields(inst.__class__)
|
||||
for k, v in changes.items():
|
||||
a = getattr(attrs, k, NOTHING)
|
||||
if a is NOTHING:
|
||||
msg = f"{k} is not an attrs attribute on {new.__class__}."
|
||||
raise AttrsAttributeNotFoundError(msg)
|
||||
_OBJ_SETATTR(new, k, v)
|
||||
return new
|
||||
|
||||
|
||||
def resolve_types(
|
||||
cls, globalns=None, localns=None, attribs=None, include_extras=True
|
||||
):
|
||||
"""
|
||||
Resolve any strings and forward annotations in type annotations.
|
||||
|
||||
This is only required if you need concrete types in :class:`Attribute`'s
|
||||
*type* field. In other words, you don't need to resolve your types if you
|
||||
only use them for static type checking.
|
||||
|
||||
With no arguments, names will be looked up in the module in which the class
|
||||
was created. If this is not what you want, for example, if the name only
|
||||
exists inside a method, you may pass *globalns* or *localns* to specify
|
||||
other dictionaries in which to look up these names. See the docs of
|
||||
`typing.get_type_hints` for more details.
|
||||
|
||||
Args:
|
||||
cls (type): Class to resolve.
|
||||
|
||||
globalns (dict | None): Dictionary containing global variables.
|
||||
|
||||
localns (dict | None): Dictionary containing local variables.
|
||||
|
||||
attribs (list | None):
|
||||
List of attribs for the given class. This is necessary when calling
|
||||
from inside a ``field_transformer`` since *cls* is not an *attrs*
|
||||
class yet.
|
||||
|
||||
include_extras (bool):
|
||||
Resolve more accurately, if possible. Pass ``include_extras`` to
|
||||
``typing.get_hints``, if supported by the typing module. On
|
||||
supported Python versions (3.9+), this resolves the types more
|
||||
accurately.
|
||||
|
||||
Raises:
|
||||
TypeError: If *cls* is not a class.
|
||||
|
||||
attrs.exceptions.NotAnAttrsClassError:
|
||||
If *cls* is not an *attrs* class and you didn't pass any attribs.
|
||||
|
||||
NameError: If types cannot be resolved because of missing variables.
|
||||
|
||||
Returns:
|
||||
*cls* so you can use this function also as a class decorator. Please
|
||||
note that you have to apply it **after** `attrs.define`. That means the
|
||||
decorator has to come in the line **before** `attrs.define`.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
.. versionadded:: 21.1.0 *attribs*
|
||||
.. versionadded:: 23.1.0 *include_extras*
|
||||
"""
|
||||
# Since calling get_type_hints is expensive we cache whether we've
|
||||
# done it already.
|
||||
if getattr(cls, "__attrs_types_resolved__", None) != cls:
|
||||
import typing
|
||||
|
||||
kwargs = {
|
||||
"globalns": globalns,
|
||||
"localns": localns,
|
||||
"include_extras": include_extras,
|
||||
}
|
||||
|
||||
hints = typing.get_type_hints(cls, **kwargs)
|
||||
for field in fields(cls) if attribs is None else attribs:
|
||||
if field.name in hints:
|
||||
# Since fields have been frozen we must work around it.
|
||||
_OBJ_SETATTR(field, "type", hints[field.name])
|
||||
# We store the class we resolved so that subclasses know they haven't
|
||||
# been resolved.
|
||||
cls.__attrs_types_resolved__ = cls
|
||||
|
||||
# Return the class so you can use it as a decorator too.
|
||||
return cls
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,674 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
These are keyword-only APIs that call `attr.s` and `attr.ib` with different
|
||||
default values.
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
|
||||
from . import setters
|
||||
from ._funcs import asdict as _asdict
|
||||
from ._funcs import astuple as _astuple
|
||||
from ._make import (
|
||||
_DEFAULT_ON_SETATTR,
|
||||
NOTHING,
|
||||
_frozen_setattrs,
|
||||
attrib,
|
||||
attrs,
|
||||
)
|
||||
from .exceptions import NotAnAttrsClassError, UnannotatedAttributeError
|
||||
|
||||
|
||||
def define(
|
||||
maybe_cls=None,
|
||||
*,
|
||||
these=None,
|
||||
repr=None,
|
||||
unsafe_hash=None,
|
||||
hash=None,
|
||||
init=None,
|
||||
slots=True,
|
||||
frozen=False,
|
||||
weakref_slot=True,
|
||||
str=False,
|
||||
auto_attribs=None,
|
||||
kw_only=False,
|
||||
cache_hash=False,
|
||||
auto_exc=True,
|
||||
eq=None,
|
||||
order=False,
|
||||
auto_detect=True,
|
||||
getstate_setstate=None,
|
||||
on_setattr=None,
|
||||
field_transformer=None,
|
||||
match_args=True,
|
||||
force_kw_only=False,
|
||||
):
|
||||
r"""
|
||||
A class decorator that adds :term:`dunder methods` according to
|
||||
:term:`fields <field>` specified using :doc:`type annotations <types>`,
|
||||
`field()` calls, or the *these* argument.
|
||||
|
||||
Since *attrs* patches or replaces an existing class, you cannot use
|
||||
`object.__init_subclass__` with *attrs* classes, because it runs too early.
|
||||
As a replacement, you can define ``__attrs_init_subclass__`` on your class.
|
||||
It will be called by *attrs* classes that subclass it after they're
|
||||
created. See also :ref:`init-subclass`.
|
||||
|
||||
Args:
|
||||
slots (bool):
|
||||
Create a :term:`slotted class <slotted classes>` that's more
|
||||
memory-efficient. Slotted classes are generally superior to the
|
||||
default dict classes, but have some gotchas you should know about,
|
||||
so we encourage you to read the :term:`glossary entry <slotted
|
||||
classes>`.
|
||||
|
||||
auto_detect (bool):
|
||||
Instead of setting the *init*, *repr*, *eq*, and *hash* arguments
|
||||
explicitly, assume they are set to True **unless any** of the
|
||||
involved methods for one of the arguments is implemented in the
|
||||
*current* class (meaning, it is *not* inherited from some base
|
||||
class).
|
||||
|
||||
So, for example by implementing ``__eq__`` on a class yourself,
|
||||
*attrs* will deduce ``eq=False`` and will create *neither*
|
||||
``__eq__`` *nor* ``__ne__`` (but Python classes come with a
|
||||
sensible ``__ne__`` by default, so it *should* be enough to only
|
||||
implement ``__eq__`` in most cases).
|
||||
|
||||
Passing :data:`True` or :data:`False` to *init*, *repr*, *eq*, or *hash*
|
||||
overrides whatever *auto_detect* would determine.
|
||||
|
||||
auto_exc (bool):
|
||||
If the class subclasses `BaseException` (which implicitly includes
|
||||
any subclass of any exception), the following happens to behave
|
||||
like a well-behaved Python exception class:
|
||||
|
||||
- the values for *eq*, *order*, and *hash* are ignored and the
|
||||
instances compare and hash by the instance's ids [#]_ ,
|
||||
- all attributes that are either passed into ``__init__`` or have a
|
||||
default value are additionally available as a tuple in the
|
||||
``args`` attribute,
|
||||
- the value of *str* is ignored leaving ``__str__`` to base
|
||||
classes.
|
||||
|
||||
.. [#]
|
||||
Note that *attrs* will *not* remove existing implementations of
|
||||
``__hash__`` or the equality methods. It just won't add own
|
||||
ones.
|
||||
|
||||
on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]):
|
||||
A callable that is run whenever the user attempts to set an
|
||||
attribute (either by assignment like ``i.x = 42`` or by using
|
||||
`setattr` like ``setattr(i, "x", 42)``). It receives the same
|
||||
arguments as validators: the instance, the attribute that is being
|
||||
modified, and the new value.
|
||||
|
||||
If no exception is raised, the attribute is set to the return value
|
||||
of the callable.
|
||||
|
||||
If a list of callables is passed, they're automatically wrapped in
|
||||
an `attrs.setters.pipe`.
|
||||
|
||||
If left None, the default behavior is to run converters and
|
||||
validators whenever an attribute is set.
|
||||
|
||||
init (bool):
|
||||
Create a ``__init__`` method that initializes the *attrs*
|
||||
attributes. Leading underscores are stripped for the argument name,
|
||||
unless an alias is set on the attribute.
|
||||
|
||||
.. seealso::
|
||||
`init` shows advanced ways to customize the generated
|
||||
``__init__`` method, including executing code before and after.
|
||||
|
||||
repr(bool):
|
||||
Create a ``__repr__`` method with a human readable representation
|
||||
of *attrs* attributes.
|
||||
|
||||
str (bool):
|
||||
Create a ``__str__`` method that is identical to ``__repr__``. This
|
||||
is usually not necessary except for `Exception`\ s.
|
||||
|
||||
eq (bool | None):
|
||||
If True or None (default), add ``__eq__`` and ``__ne__`` methods
|
||||
that check two instances for equality.
|
||||
|
||||
.. seealso::
|
||||
`comparison` describes how to customize the comparison behavior
|
||||
going as far comparing NumPy arrays.
|
||||
|
||||
order (bool | None):
|
||||
If True, add ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__``
|
||||
methods that behave like *eq* above and allow instances to be
|
||||
ordered.
|
||||
|
||||
They compare the instances as if they were tuples of their *attrs*
|
||||
attributes if and only if the types of both classes are
|
||||
*identical*.
|
||||
|
||||
If `None` mirror value of *eq*.
|
||||
|
||||
.. seealso:: `comparison`
|
||||
|
||||
unsafe_hash (bool | None):
|
||||
If None (default), the ``__hash__`` method is generated according
|
||||
how *eq* and *frozen* are set.
|
||||
|
||||
1. If *both* are True, *attrs* will generate a ``__hash__`` for
|
||||
you.
|
||||
2. If *eq* is True and *frozen* is False, ``__hash__`` will be set
|
||||
to None, marking it unhashable (which it is).
|
||||
3. If *eq* is False, ``__hash__`` will be left untouched meaning
|
||||
the ``__hash__`` method of the base class will be used. If the
|
||||
base class is `object`, this means it will fall back to id-based
|
||||
hashing.
|
||||
|
||||
Although not recommended, you can decide for yourself and force
|
||||
*attrs* to create one (for example, if the class is immutable even
|
||||
though you didn't freeze it programmatically) by passing True or
|
||||
not. Both of these cases are rather special and should be used
|
||||
carefully.
|
||||
|
||||
.. seealso::
|
||||
|
||||
- Our documentation on `hashing`,
|
||||
- Python's documentation on `object.__hash__`,
|
||||
- and the `GitHub issue that led to the default \ behavior
|
||||
<https://github.com/python-attrs/attrs/issues/136>`_ for more
|
||||
details.
|
||||
|
||||
hash (bool | None):
|
||||
Deprecated alias for *unsafe_hash*. *unsafe_hash* takes precedence.
|
||||
|
||||
cache_hash (bool):
|
||||
Ensure that the object's hash code is computed only once and stored
|
||||
on the object. If this is set to True, hashing must be either
|
||||
explicitly or implicitly enabled for this class. If the hash code
|
||||
is cached, avoid any reassignments of fields involved in hash code
|
||||
computation or mutations of the objects those fields point to after
|
||||
object creation. If such changes occur, the behavior of the
|
||||
object's hash code is undefined.
|
||||
|
||||
frozen (bool):
|
||||
Make instances immutable after initialization. If someone attempts
|
||||
to modify a frozen instance, `attrs.exceptions.FrozenInstanceError`
|
||||
is raised.
|
||||
|
||||
.. note::
|
||||
|
||||
1. This is achieved by installing a custom ``__setattr__``
|
||||
method on your class, so you can't implement your own.
|
||||
|
||||
2. True immutability is impossible in Python.
|
||||
|
||||
3. This *does* have a minor a runtime performance `impact
|
||||
<how-frozen>` when initializing new instances. In other
|
||||
words: ``__init__`` is slightly slower with ``frozen=True``.
|
||||
|
||||
4. If a class is frozen, you cannot modify ``self`` in
|
||||
``__attrs_post_init__`` or a self-written ``__init__``. You
|
||||
can circumvent that limitation by using
|
||||
``object.__setattr__(self, "attribute_name", value)``.
|
||||
|
||||
5. Subclasses of a frozen class are frozen too.
|
||||
|
||||
kw_only (bool):
|
||||
Make attributes keyword-only in the generated ``__init__`` (if
|
||||
*init* is False, this parameter is ignored). Attributes that
|
||||
explicitly set ``kw_only=False`` are not affected; base class
|
||||
attributes are also not affected.
|
||||
|
||||
Also see *force_kw_only*.
|
||||
|
||||
weakref_slot (bool):
|
||||
Make instances weak-referenceable. This has no effect unless
|
||||
*slots* is True.
|
||||
|
||||
field_transformer (~typing.Callable | None):
|
||||
A function that is called with the original class object and all
|
||||
fields right before *attrs* finalizes the class. You can use this,
|
||||
for example, to automatically add converters or validators to
|
||||
fields based on their types.
|
||||
|
||||
.. seealso:: `transform-fields`
|
||||
|
||||
match_args (bool):
|
||||
If True (default), set ``__match_args__`` on the class to support
|
||||
:pep:`634` (*Structural Pattern Matching*). It is a tuple of all
|
||||
non-keyword-only ``__init__`` parameter names on Python 3.10 and
|
||||
later. Ignored on older Python versions.
|
||||
|
||||
collect_by_mro (bool):
|
||||
If True, *attrs* collects attributes from base classes correctly
|
||||
according to the `method resolution order
|
||||
<https://docs.python.org/3/howto/mro.html>`_. If False, *attrs*
|
||||
will mimic the (wrong) behavior of `dataclasses` and :pep:`681`.
|
||||
|
||||
See also `issue #428
|
||||
<https://github.com/python-attrs/attrs/issues/428>`_.
|
||||
|
||||
force_kw_only (bool):
|
||||
A back-compat flag for restoring pre-25.4.0 behavior. If True and
|
||||
``kw_only=True``, all attributes are made keyword-only, including
|
||||
base class attributes, and those set to ``kw_only=False`` at the
|
||||
attribute level. Defaults to False.
|
||||
|
||||
See also `issue #980
|
||||
<https://github.com/python-attrs/attrs/issues/980>`_.
|
||||
|
||||
getstate_setstate (bool | None):
|
||||
.. note::
|
||||
|
||||
This is usually only interesting for slotted classes and you
|
||||
should probably just set *auto_detect* to True.
|
||||
|
||||
If True, ``__getstate__`` and ``__setstate__`` are generated and
|
||||
attached to the class. This is necessary for slotted classes to be
|
||||
pickleable. If left None, it's True by default for slotted classes
|
||||
and False for dict classes.
|
||||
|
||||
If *auto_detect* is True, and *getstate_setstate* is left None, and
|
||||
**either** ``__getstate__`` or ``__setstate__`` is detected
|
||||
directly on the class (meaning: not inherited), it is set to False
|
||||
(this is usually what you want).
|
||||
|
||||
auto_attribs (bool | None):
|
||||
If True, look at type annotations to determine which attributes to
|
||||
use, like `dataclasses`. If False, it will only look for explicit
|
||||
:func:`field` class attributes, like classic *attrs*.
|
||||
|
||||
If left None, it will guess:
|
||||
|
||||
1. If any attributes are annotated and no unannotated
|
||||
`attrs.field`\ s are found, it assumes *auto_attribs=True*.
|
||||
2. Otherwise it assumes *auto_attribs=False* and tries to collect
|
||||
`attrs.field`\ s.
|
||||
|
||||
If *attrs* decides to look at type annotations, **all** fields
|
||||
**must** be annotated. If *attrs* encounters a field that is set to
|
||||
a :func:`field` / `attr.ib` but lacks a type annotation, an
|
||||
`attrs.exceptions.UnannotatedAttributeError` is raised. Use
|
||||
``field_name: typing.Any = field(...)`` if you don't want to set a
|
||||
type.
|
||||
|
||||
.. warning::
|
||||
|
||||
For features that use the attribute name to create decorators
|
||||
(for example, :ref:`validators <validators>`), you still *must*
|
||||
assign :func:`field` / `attr.ib` to them. Otherwise Python will
|
||||
either not find the name or try to use the default value to
|
||||
call, for example, ``validator`` on it.
|
||||
|
||||
Attributes annotated as `typing.ClassVar`, and attributes that are
|
||||
neither annotated nor set to an `field()` are **ignored**.
|
||||
|
||||
these (dict[str, object]):
|
||||
A dictionary of name to the (private) return value of `field()`
|
||||
mappings. This is useful to avoid the definition of your attributes
|
||||
within the class body because you can't (for example, if you want
|
||||
to add ``__repr__`` methods to Django models) or don't want to.
|
||||
|
||||
If *these* is not `None`, *attrs* will *not* search the class body
|
||||
for attributes and will *not* remove any attributes from it.
|
||||
|
||||
The order is deduced from the order of the attributes inside
|
||||
*these*.
|
||||
|
||||
Arguably, this is a rather obscure feature.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
.. versionchanged:: 21.3.0 Converters are also run ``on_setattr``.
|
||||
.. versionadded:: 22.2.0
|
||||
*unsafe_hash* as an alias for *hash* (for :pep:`681` compliance).
|
||||
.. versionchanged:: 24.1.0
|
||||
Instances are not compared as tuples of attributes anymore, but using a
|
||||
big ``and`` condition. This is faster and has more correct behavior for
|
||||
uncomparable values like `math.nan`.
|
||||
.. versionadded:: 24.1.0
|
||||
If a class has an *inherited* classmethod called
|
||||
``__attrs_init_subclass__``, it is executed after the class is created.
|
||||
.. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*.
|
||||
.. versionadded:: 24.3.0
|
||||
Unless already present, a ``__replace__`` method is automatically
|
||||
created for `copy.replace` (Python 3.13+ only).
|
||||
.. versionchanged:: 25.4.0
|
||||
*kw_only* now only applies to attributes defined in the current class,
|
||||
and respects attribute-level ``kw_only=False`` settings.
|
||||
.. versionadded:: 25.4.0
|
||||
Added *force_kw_only* to go back to the previous *kw_only* behavior.
|
||||
|
||||
.. note::
|
||||
|
||||
The main differences to the classic `attr.s` are:
|
||||
|
||||
- Automatically detect whether or not *auto_attribs* should be `True`
|
||||
(c.f. *auto_attribs* parameter).
|
||||
- Converters and validators run when attributes are set by default --
|
||||
if *frozen* is `False`.
|
||||
- *slots=True*
|
||||
|
||||
Usually, this has only upsides and few visible effects in everyday
|
||||
programming. But it *can* lead to some surprising behaviors, so
|
||||
please make sure to read :term:`slotted classes`.
|
||||
|
||||
- *auto_exc=True*
|
||||
- *auto_detect=True*
|
||||
- *order=False*
|
||||
- *force_kw_only=False*
|
||||
- Some options that were only relevant on Python 2 or were kept around
|
||||
for backwards-compatibility have been removed.
|
||||
|
||||
"""
|
||||
|
||||
def do_it(cls, auto_attribs):
|
||||
return attrs(
|
||||
maybe_cls=cls,
|
||||
these=these,
|
||||
repr=repr,
|
||||
hash=hash,
|
||||
unsafe_hash=unsafe_hash,
|
||||
init=init,
|
||||
slots=slots,
|
||||
frozen=frozen,
|
||||
weakref_slot=weakref_slot,
|
||||
str=str,
|
||||
auto_attribs=auto_attribs,
|
||||
kw_only=kw_only,
|
||||
cache_hash=cache_hash,
|
||||
auto_exc=auto_exc,
|
||||
eq=eq,
|
||||
order=order,
|
||||
auto_detect=auto_detect,
|
||||
collect_by_mro=True,
|
||||
getstate_setstate=getstate_setstate,
|
||||
on_setattr=on_setattr,
|
||||
field_transformer=field_transformer,
|
||||
match_args=match_args,
|
||||
force_kw_only=force_kw_only,
|
||||
)
|
||||
|
||||
def wrap(cls):
|
||||
"""
|
||||
Making this a wrapper ensures this code runs during class creation.
|
||||
|
||||
We also ensure that frozen-ness of classes is inherited.
|
||||
"""
|
||||
nonlocal frozen, on_setattr
|
||||
|
||||
had_on_setattr = on_setattr not in (None, setters.NO_OP)
|
||||
|
||||
# By default, mutable classes convert & validate on setattr.
|
||||
if frozen is False and on_setattr is None:
|
||||
on_setattr = _DEFAULT_ON_SETATTR
|
||||
|
||||
# However, if we subclass a frozen class, we inherit the immutability
|
||||
# and disable on_setattr.
|
||||
for base_cls in cls.__bases__:
|
||||
if base_cls.__setattr__ is _frozen_setattrs:
|
||||
if had_on_setattr:
|
||||
msg = "Frozen classes can't use on_setattr (frozen-ness was inherited)."
|
||||
raise ValueError(msg)
|
||||
|
||||
on_setattr = setters.NO_OP
|
||||
break
|
||||
|
||||
if auto_attribs is not None:
|
||||
return do_it(cls, auto_attribs)
|
||||
|
||||
try:
|
||||
return do_it(cls, True)
|
||||
except UnannotatedAttributeError:
|
||||
return do_it(cls, False)
|
||||
|
||||
# maybe_cls's type depends on the usage of the decorator. It's a class
|
||||
# if it's used as `@attrs` but `None` if used as `@attrs()`.
|
||||
if maybe_cls is None:
|
||||
return wrap
|
||||
|
||||
return wrap(maybe_cls)
|
||||
|
||||
|
||||
mutable = define
|
||||
frozen = partial(define, frozen=True, on_setattr=None)
|
||||
|
||||
|
||||
def field(
|
||||
*,
|
||||
default=NOTHING,
|
||||
validator=None,
|
||||
repr=True,
|
||||
hash=None,
|
||||
init=True,
|
||||
metadata=None,
|
||||
type=None,
|
||||
converter=None,
|
||||
factory=None,
|
||||
kw_only=None,
|
||||
eq=None,
|
||||
order=None,
|
||||
on_setattr=None,
|
||||
alias=None,
|
||||
):
|
||||
"""
|
||||
Create a new :term:`field` / :term:`attribute` on a class.
|
||||
|
||||
.. warning::
|
||||
|
||||
Does **nothing** unless the class is also decorated with
|
||||
`attrs.define` (or similar)!
|
||||
|
||||
Args:
|
||||
default:
|
||||
A value that is used if an *attrs*-generated ``__init__`` is used
|
||||
and no value is passed while instantiating or the attribute is
|
||||
excluded using ``init=False``.
|
||||
|
||||
If the value is an instance of `attrs.Factory`, its callable will
|
||||
be used to construct a new value (useful for mutable data types
|
||||
like lists or dicts).
|
||||
|
||||
If a default is not set (or set manually to `attrs.NOTHING`), a
|
||||
value *must* be supplied when instantiating; otherwise a
|
||||
`TypeError` will be raised.
|
||||
|
||||
.. seealso:: `defaults`
|
||||
|
||||
factory (~typing.Callable):
|
||||
Syntactic sugar for ``default=attr.Factory(factory)``.
|
||||
|
||||
validator (~typing.Callable | list[~typing.Callable]):
|
||||
Callable that is called by *attrs*-generated ``__init__`` methods
|
||||
after the instance has been initialized. They receive the
|
||||
initialized instance, the :func:`~attrs.Attribute`, and the passed
|
||||
value.
|
||||
|
||||
The return value is *not* inspected so the validator has to throw
|
||||
an exception itself.
|
||||
|
||||
If a `list` is passed, its items are treated as validators and must
|
||||
all pass.
|
||||
|
||||
Validators can be globally disabled and re-enabled using
|
||||
`attrs.validators.get_disabled` / `attrs.validators.set_disabled`.
|
||||
|
||||
The validator can also be set using decorator notation as shown
|
||||
below.
|
||||
|
||||
.. seealso:: :ref:`validators`
|
||||
|
||||
repr (bool | ~typing.Callable):
|
||||
Include this attribute in the generated ``__repr__`` method. If
|
||||
True, include the attribute; if False, omit it. By default, the
|
||||
built-in ``repr()`` function is used. To override how the attribute
|
||||
value is formatted, pass a ``callable`` that takes a single value
|
||||
and returns a string. Note that the resulting string is used as-is,
|
||||
which means it will be used directly *instead* of calling
|
||||
``repr()`` (the default).
|
||||
|
||||
eq (bool | ~typing.Callable):
|
||||
If True (default), include this attribute in the generated
|
||||
``__eq__`` and ``__ne__`` methods that check two instances for
|
||||
equality. To override how the attribute value is compared, pass a
|
||||
callable that takes a single value and returns the value to be
|
||||
compared.
|
||||
|
||||
.. seealso:: `comparison`
|
||||
|
||||
order (bool | ~typing.Callable):
|
||||
If True (default), include this attributes in the generated
|
||||
``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. To
|
||||
override how the attribute value is ordered, pass a callable that
|
||||
takes a single value and returns the value to be ordered.
|
||||
|
||||
.. seealso:: `comparison`
|
||||
|
||||
hash (bool | None):
|
||||
Include this attribute in the generated ``__hash__`` method. If
|
||||
None (default), mirror *eq*'s value. This is the correct behavior
|
||||
according the Python spec. Setting this value to anything else
|
||||
than None is *discouraged*.
|
||||
|
||||
.. seealso:: `hashing`
|
||||
|
||||
init (bool):
|
||||
Include this attribute in the generated ``__init__`` method.
|
||||
|
||||
It is possible to set this to False and set a default value. In
|
||||
that case this attributed is unconditionally initialized with the
|
||||
specified default value or factory.
|
||||
|
||||
.. seealso:: `init`
|
||||
|
||||
converter (typing.Callable | Converter):
|
||||
A callable that is called by *attrs*-generated ``__init__`` methods
|
||||
to convert attribute's value to the desired format.
|
||||
|
||||
If a vanilla callable is passed, it is given the passed-in value as
|
||||
the only positional argument. It is possible to receive additional
|
||||
arguments by wrapping the callable in a `Converter`.
|
||||
|
||||
Either way, the returned value will be used as the new value of the
|
||||
attribute. The value is converted before being passed to the
|
||||
validator, if any.
|
||||
|
||||
.. seealso:: :ref:`converters`
|
||||
|
||||
metadata (dict | None):
|
||||
An arbitrary mapping, to be used by third-party code.
|
||||
|
||||
.. seealso:: `extending-metadata`.
|
||||
|
||||
type (type):
|
||||
The type of the attribute. Nowadays, the preferred method to
|
||||
specify the type is using a variable annotation (see :pep:`526`).
|
||||
This argument is provided for backwards-compatibility and for usage
|
||||
with `make_class`. Regardless of the approach used, the type will
|
||||
be stored on ``Attribute.type``.
|
||||
|
||||
Please note that *attrs* doesn't do anything with this metadata by
|
||||
itself. You can use it as part of your own code or for `static type
|
||||
checking <types>`.
|
||||
|
||||
kw_only (bool | None):
|
||||
Make this attribute keyword-only in the generated ``__init__`` (if
|
||||
*init* is False, this parameter is ignored). If None (default),
|
||||
mirror the setting from `attrs.define`.
|
||||
|
||||
on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]):
|
||||
Allows to overwrite the *on_setattr* setting from `attr.s`. If left
|
||||
None, the *on_setattr* value from `attr.s` is used. Set to
|
||||
`attrs.setters.NO_OP` to run **no** `setattr` hooks for this
|
||||
attribute -- regardless of the setting in `define()`.
|
||||
|
||||
alias (str | None):
|
||||
Override this attribute's parameter name in the generated
|
||||
``__init__`` method. If left None, default to ``name`` stripped
|
||||
of leading underscores. See `private-attributes`.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
.. versionchanged:: 21.1.0
|
||||
*eq*, *order*, and *cmp* also accept a custom callable
|
||||
.. versionadded:: 22.2.0 *alias*
|
||||
.. versionadded:: 23.1.0
|
||||
The *type* parameter has been re-added; mostly for `attrs.make_class`.
|
||||
Please note that type checkers ignore this metadata.
|
||||
.. versionchanged:: 25.4.0
|
||||
*kw_only* can now be None, and its default is also changed from False to
|
||||
None.
|
||||
|
||||
.. seealso::
|
||||
|
||||
`attr.ib`
|
||||
"""
|
||||
return attrib(
|
||||
default=default,
|
||||
validator=validator,
|
||||
repr=repr,
|
||||
hash=hash,
|
||||
init=init,
|
||||
metadata=metadata,
|
||||
type=type,
|
||||
converter=converter,
|
||||
factory=factory,
|
||||
kw_only=kw_only,
|
||||
eq=eq,
|
||||
order=order,
|
||||
on_setattr=on_setattr,
|
||||
alias=alias,
|
||||
)
|
||||
|
||||
|
||||
def asdict(inst, *, recurse=True, filter=None, value_serializer=None):
|
||||
"""
|
||||
Same as `attr.asdict`, except that collections types are always retained
|
||||
and dict is always used as *dict_factory*.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _asdict(
|
||||
inst=inst,
|
||||
recurse=recurse,
|
||||
filter=filter,
|
||||
value_serializer=value_serializer,
|
||||
retain_collection_types=True,
|
||||
)
|
||||
|
||||
|
||||
def astuple(inst, *, recurse=True, filter=None):
|
||||
"""
|
||||
Same as `attr.astuple`, except that collections types are always retained
|
||||
and `tuple` is always used as the *tuple_factory*.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _astuple(
|
||||
inst=inst, recurse=recurse, filter=filter, retain_collection_types=True
|
||||
)
|
||||
|
||||
|
||||
def inspect(cls):
|
||||
"""
|
||||
Inspect the class and return its effective build parameters.
|
||||
|
||||
Warning:
|
||||
This feature is currently **experimental** and is not covered by our
|
||||
strict backwards-compatibility guarantees.
|
||||
|
||||
Args:
|
||||
cls: The *attrs*-decorated class to inspect.
|
||||
|
||||
Returns:
|
||||
The effective build parameters of the class.
|
||||
|
||||
Raises:
|
||||
NotAnAttrsClassError: If the class is not an *attrs*-decorated class.
|
||||
|
||||
.. versionadded:: 25.4.0
|
||||
"""
|
||||
try:
|
||||
return cls.__dict__["__attrs_props__"]
|
||||
except KeyError:
|
||||
msg = f"{cls!r} is not an attrs-decorated class."
|
||||
raise NotAnAttrsClassError(msg) from None
|
||||
@@ -0,0 +1,15 @@
|
||||
from typing import Any, ClassVar, Protocol
|
||||
|
||||
# MYPY is a special constant in mypy which works the same way as `TYPE_CHECKING`.
|
||||
MYPY = False
|
||||
|
||||
if MYPY:
|
||||
# A protocol to be able to statically accept an attrs class.
|
||||
class AttrsInstance_(Protocol):
|
||||
__attrs_attrs__: ClassVar[Any]
|
||||
|
||||
else:
|
||||
# For type checkers without plug-in support use an empty protocol that
|
||||
# will (hopefully) be combined into a union.
|
||||
class AttrsInstance_(Protocol):
|
||||
pass
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
from functools import total_ordering
|
||||
|
||||
from ._funcs import astuple
|
||||
from ._make import attrib, attrs
|
||||
|
||||
|
||||
@total_ordering
|
||||
@attrs(eq=False, order=False, slots=True, frozen=True)
|
||||
class VersionInfo:
|
||||
"""
|
||||
A version object that can be compared to tuple of length 1--4:
|
||||
|
||||
>>> attr.VersionInfo(19, 1, 0, "final") <= (19, 2)
|
||||
True
|
||||
>>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1)
|
||||
True
|
||||
>>> vi = attr.VersionInfo(19, 2, 0, "final")
|
||||
>>> vi < (19, 1, 1)
|
||||
False
|
||||
>>> vi < (19,)
|
||||
False
|
||||
>>> vi == (19, 2,)
|
||||
True
|
||||
>>> vi == (19, 2, 1)
|
||||
False
|
||||
|
||||
.. versionadded:: 19.2
|
||||
"""
|
||||
|
||||
year = attrib(type=int)
|
||||
minor = attrib(type=int)
|
||||
micro = attrib(type=int)
|
||||
releaselevel = attrib(type=str)
|
||||
|
||||
@classmethod
|
||||
def _from_version_string(cls, s):
|
||||
"""
|
||||
Parse *s* and return a _VersionInfo.
|
||||
"""
|
||||
v = s.split(".")
|
||||
if len(v) == 3:
|
||||
v.append("final")
|
||||
|
||||
return cls(
|
||||
year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3]
|
||||
)
|
||||
|
||||
def _ensure_tuple(self, other):
|
||||
"""
|
||||
Ensure *other* is a tuple of a valid length.
|
||||
|
||||
Returns a possibly transformed *other* and ourselves as a tuple of
|
||||
the same length as *other*.
|
||||
"""
|
||||
|
||||
if self.__class__ is other.__class__:
|
||||
other = astuple(other)
|
||||
|
||||
if not isinstance(other, tuple):
|
||||
raise NotImplementedError
|
||||
|
||||
if not (1 <= len(other) <= 4):
|
||||
raise NotImplementedError
|
||||
|
||||
return astuple(self)[: len(other)], other
|
||||
|
||||
def __eq__(self, other):
|
||||
try:
|
||||
us, them = self._ensure_tuple(other)
|
||||
except NotImplementedError:
|
||||
return NotImplemented
|
||||
|
||||
return us == them
|
||||
|
||||
def __lt__(self, other):
|
||||
try:
|
||||
us, them = self._ensure_tuple(other)
|
||||
except NotImplementedError:
|
||||
return NotImplemented
|
||||
|
||||
# Since alphabetically "dev0" < "final" < "post1" < "post2", we don't
|
||||
# have to do anything special with releaselevel for now.
|
||||
return us < them
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.year, self.minor, self.micro, self.releaselevel))
|
||||
@@ -0,0 +1,9 @@
|
||||
class VersionInfo:
|
||||
@property
|
||||
def year(self) -> int: ...
|
||||
@property
|
||||
def minor(self) -> int: ...
|
||||
@property
|
||||
def micro(self) -> int: ...
|
||||
@property
|
||||
def releaselevel(self) -> str: ...
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Commonly useful converters.
|
||||
"""
|
||||
|
||||
import typing
|
||||
|
||||
from ._compat import _AnnotationExtractor
|
||||
from ._make import NOTHING, Converter, Factory, pipe
|
||||
|
||||
|
||||
__all__ = [
|
||||
"default_if_none",
|
||||
"optional",
|
||||
"pipe",
|
||||
"to_bool",
|
||||
]
|
||||
|
||||
|
||||
def optional(converter):
|
||||
"""
|
||||
A converter that allows an attribute to be optional. An optional attribute
|
||||
is one which can be set to `None`.
|
||||
|
||||
Type annotations will be inferred from the wrapped converter's, if it has
|
||||
any.
|
||||
|
||||
Args:
|
||||
converter (typing.Callable):
|
||||
the converter that is used for non-`None` values.
|
||||
|
||||
.. versionadded:: 17.1.0
|
||||
"""
|
||||
|
||||
if isinstance(converter, Converter):
|
||||
|
||||
def optional_converter(val, inst, field):
|
||||
if val is None:
|
||||
return None
|
||||
return converter(val, inst, field)
|
||||
|
||||
else:
|
||||
|
||||
def optional_converter(val):
|
||||
if val is None:
|
||||
return None
|
||||
return converter(val)
|
||||
|
||||
xtr = _AnnotationExtractor(converter)
|
||||
|
||||
t = xtr.get_first_param_type()
|
||||
if t:
|
||||
optional_converter.__annotations__["val"] = typing.Optional[t]
|
||||
|
||||
rt = xtr.get_return_type()
|
||||
if rt:
|
||||
optional_converter.__annotations__["return"] = typing.Optional[rt]
|
||||
|
||||
if isinstance(converter, Converter):
|
||||
return Converter(optional_converter, takes_self=True, takes_field=True)
|
||||
|
||||
return optional_converter
|
||||
|
||||
|
||||
def default_if_none(default=NOTHING, factory=None):
|
||||
"""
|
||||
A converter that allows to replace `None` values by *default* or the result
|
||||
of *factory*.
|
||||
|
||||
Args:
|
||||
default:
|
||||
Value to be used if `None` is passed. Passing an instance of
|
||||
`attrs.Factory` is supported, however the ``takes_self`` option is
|
||||
*not*.
|
||||
|
||||
factory (typing.Callable):
|
||||
A callable that takes no parameters whose result is used if `None`
|
||||
is passed.
|
||||
|
||||
Raises:
|
||||
TypeError: If **neither** *default* or *factory* is passed.
|
||||
|
||||
TypeError: If **both** *default* and *factory* are passed.
|
||||
|
||||
ValueError:
|
||||
If an instance of `attrs.Factory` is passed with
|
||||
``takes_self=True``.
|
||||
|
||||
.. versionadded:: 18.2.0
|
||||
"""
|
||||
if default is NOTHING and factory is None:
|
||||
msg = "Must pass either `default` or `factory`."
|
||||
raise TypeError(msg)
|
||||
|
||||
if default is not NOTHING and factory is not None:
|
||||
msg = "Must pass either `default` or `factory` but not both."
|
||||
raise TypeError(msg)
|
||||
|
||||
if factory is not None:
|
||||
default = Factory(factory)
|
||||
|
||||
if isinstance(default, Factory):
|
||||
if default.takes_self:
|
||||
msg = "`takes_self` is not supported by default_if_none."
|
||||
raise ValueError(msg)
|
||||
|
||||
def default_if_none_converter(val):
|
||||
if val is not None:
|
||||
return val
|
||||
|
||||
return default.factory()
|
||||
|
||||
else:
|
||||
|
||||
def default_if_none_converter(val):
|
||||
if val is not None:
|
||||
return val
|
||||
|
||||
return default
|
||||
|
||||
return default_if_none_converter
|
||||
|
||||
|
||||
def to_bool(val):
|
||||
"""
|
||||
Convert "boolean" strings (for example, from environment variables) to real
|
||||
booleans.
|
||||
|
||||
Values mapping to `True`:
|
||||
|
||||
- ``True``
|
||||
- ``"true"`` / ``"t"``
|
||||
- ``"yes"`` / ``"y"``
|
||||
- ``"on"``
|
||||
- ``"1"``
|
||||
- ``1``
|
||||
|
||||
Values mapping to `False`:
|
||||
|
||||
- ``False``
|
||||
- ``"false"`` / ``"f"``
|
||||
- ``"no"`` / ``"n"``
|
||||
- ``"off"``
|
||||
- ``"0"``
|
||||
- ``0``
|
||||
|
||||
Raises:
|
||||
ValueError: For any other value.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
if isinstance(val, str):
|
||||
val = val.lower()
|
||||
|
||||
if val in (True, "true", "t", "yes", "y", "on", "1", 1):
|
||||
return True
|
||||
if val in (False, "false", "f", "no", "n", "off", "0", 0):
|
||||
return False
|
||||
|
||||
msg = f"Cannot convert value to bool: {val!r}"
|
||||
raise ValueError(msg)
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import Callable, Any, overload
|
||||
|
||||
from attrs import _ConverterType, _CallableConverterType
|
||||
|
||||
@overload
|
||||
def pipe(*validators: _CallableConverterType) -> _CallableConverterType: ...
|
||||
@overload
|
||||
def pipe(*validators: _ConverterType) -> _ConverterType: ...
|
||||
@overload
|
||||
def optional(converter: _CallableConverterType) -> _CallableConverterType: ...
|
||||
@overload
|
||||
def optional(converter: _ConverterType) -> _ConverterType: ...
|
||||
@overload
|
||||
def default_if_none(default: Any) -> _CallableConverterType: ...
|
||||
@overload
|
||||
def default_if_none(
|
||||
*, factory: Callable[[], Any]
|
||||
) -> _CallableConverterType: ...
|
||||
def to_bool(val: str | int | bool) -> bool: ...
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class FrozenError(AttributeError):
|
||||
"""
|
||||
A frozen/immutable instance or attribute have been attempted to be
|
||||
modified.
|
||||
|
||||
It mirrors the behavior of ``namedtuples`` by using the same error message
|
||||
and subclassing `AttributeError`.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
msg = "can't set attribute"
|
||||
super().__init__(msg)
|
||||
self.msg = msg
|
||||
|
||||
|
||||
class FrozenInstanceError(FrozenError):
|
||||
"""
|
||||
A frozen instance has been attempted to be modified.
|
||||
|
||||
.. versionadded:: 16.1.0
|
||||
"""
|
||||
|
||||
|
||||
class FrozenAttributeError(FrozenError):
|
||||
"""
|
||||
A frozen attribute has been attempted to be modified.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
|
||||
|
||||
class AttrsAttributeNotFoundError(ValueError):
|
||||
"""
|
||||
An *attrs* function couldn't find an attribute that the user asked for.
|
||||
|
||||
.. versionadded:: 16.2.0
|
||||
"""
|
||||
|
||||
|
||||
class NotAnAttrsClassError(ValueError):
|
||||
"""
|
||||
A non-*attrs* class has been passed into an *attrs* function.
|
||||
|
||||
.. versionadded:: 16.2.0
|
||||
"""
|
||||
|
||||
|
||||
class DefaultAlreadySetError(RuntimeError):
|
||||
"""
|
||||
A default has been set when defining the field and is attempted to be reset
|
||||
using the decorator.
|
||||
|
||||
.. versionadded:: 17.1.0
|
||||
"""
|
||||
|
||||
|
||||
class UnannotatedAttributeError(RuntimeError):
|
||||
"""
|
||||
A class with ``auto_attribs=True`` has a field without a type annotation.
|
||||
|
||||
.. versionadded:: 17.3.0
|
||||
"""
|
||||
|
||||
|
||||
class PythonTooOldError(RuntimeError):
|
||||
"""
|
||||
It was attempted to use an *attrs* feature that requires a newer Python
|
||||
version.
|
||||
|
||||
.. versionadded:: 18.2.0
|
||||
"""
|
||||
|
||||
|
||||
class NotCallableError(TypeError):
|
||||
"""
|
||||
A field requiring a callable has been set with a value that is not
|
||||
callable.
|
||||
|
||||
.. versionadded:: 19.2.0
|
||||
"""
|
||||
|
||||
def __init__(self, msg, value):
|
||||
super(TypeError, self).__init__(msg, value)
|
||||
self.msg = msg
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return str(self.msg)
|
||||
@@ -0,0 +1,17 @@
|
||||
from typing import Any
|
||||
|
||||
class FrozenError(AttributeError):
|
||||
msg: str = ...
|
||||
|
||||
class FrozenInstanceError(FrozenError): ...
|
||||
class FrozenAttributeError(FrozenError): ...
|
||||
class AttrsAttributeNotFoundError(ValueError): ...
|
||||
class NotAnAttrsClassError(ValueError): ...
|
||||
class DefaultAlreadySetError(RuntimeError): ...
|
||||
class UnannotatedAttributeError(RuntimeError): ...
|
||||
class PythonTooOldError(RuntimeError): ...
|
||||
|
||||
class NotCallableError(TypeError):
|
||||
msg: str = ...
|
||||
value: Any = ...
|
||||
def __init__(self, msg: str, value: Any) -> None: ...
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Commonly useful filters for `attrs.asdict` and `attrs.astuple`.
|
||||
"""
|
||||
|
||||
from ._make import Attribute
|
||||
|
||||
|
||||
def _split_what(what):
|
||||
"""
|
||||
Returns a tuple of `frozenset`s of classes and attributes.
|
||||
"""
|
||||
return (
|
||||
frozenset(cls for cls in what if isinstance(cls, type)),
|
||||
frozenset(cls for cls in what if isinstance(cls, str)),
|
||||
frozenset(cls for cls in what if isinstance(cls, Attribute)),
|
||||
)
|
||||
|
||||
|
||||
def include(*what):
|
||||
"""
|
||||
Create a filter that only allows *what*.
|
||||
|
||||
Args:
|
||||
what (list[type, str, attrs.Attribute]):
|
||||
What to include. Can be a type, a name, or an attribute.
|
||||
|
||||
Returns:
|
||||
Callable:
|
||||
A callable that can be passed to `attrs.asdict`'s and
|
||||
`attrs.astuple`'s *filter* argument.
|
||||
|
||||
.. versionchanged:: 23.1.0 Accept strings with field names.
|
||||
"""
|
||||
cls, names, attrs = _split_what(what)
|
||||
|
||||
def include_(attribute, value):
|
||||
return (
|
||||
value.__class__ in cls
|
||||
or attribute.name in names
|
||||
or attribute in attrs
|
||||
)
|
||||
|
||||
return include_
|
||||
|
||||
|
||||
def exclude(*what):
|
||||
"""
|
||||
Create a filter that does **not** allow *what*.
|
||||
|
||||
Args:
|
||||
what (list[type, str, attrs.Attribute]):
|
||||
What to exclude. Can be a type, a name, or an attribute.
|
||||
|
||||
Returns:
|
||||
Callable:
|
||||
A callable that can be passed to `attrs.asdict`'s and
|
||||
`attrs.astuple`'s *filter* argument.
|
||||
|
||||
.. versionchanged:: 23.3.0 Accept field name string as input argument
|
||||
"""
|
||||
cls, names, attrs = _split_what(what)
|
||||
|
||||
def exclude_(attribute, value):
|
||||
return not (
|
||||
value.__class__ in cls
|
||||
or attribute.name in names
|
||||
or attribute in attrs
|
||||
)
|
||||
|
||||
return exclude_
|
||||
@@ -0,0 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
from . import Attribute, _FilterType
|
||||
|
||||
def include(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ...
|
||||
def exclude(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ...
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Commonly used hooks for on_setattr.
|
||||
"""
|
||||
|
||||
from . import _config
|
||||
from .exceptions import FrozenAttributeError
|
||||
|
||||
|
||||
def pipe(*setters):
|
||||
"""
|
||||
Run all *setters* and return the return value of the last one.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
|
||||
def wrapped_pipe(instance, attrib, new_value):
|
||||
rv = new_value
|
||||
|
||||
for setter in setters:
|
||||
rv = setter(instance, attrib, rv)
|
||||
|
||||
return rv
|
||||
|
||||
return wrapped_pipe
|
||||
|
||||
|
||||
def frozen(_, __, ___):
|
||||
"""
|
||||
Prevent an attribute to be modified.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
raise FrozenAttributeError
|
||||
|
||||
|
||||
def validate(instance, attrib, new_value):
|
||||
"""
|
||||
Run *attrib*'s validator on *new_value* if it has one.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
if _config._run_validators is False:
|
||||
return new_value
|
||||
|
||||
v = attrib.validator
|
||||
if not v:
|
||||
return new_value
|
||||
|
||||
v(instance, attrib, new_value)
|
||||
|
||||
return new_value
|
||||
|
||||
|
||||
def convert(instance, attrib, new_value):
|
||||
"""
|
||||
Run *attrib*'s converter -- if it has one -- on *new_value* and return the
|
||||
result.
|
||||
|
||||
.. versionadded:: 20.1.0
|
||||
"""
|
||||
c = attrib.converter
|
||||
if c:
|
||||
# This can be removed once we drop 3.8 and use attrs.Converter instead.
|
||||
from ._make import Converter
|
||||
|
||||
if not isinstance(c, Converter):
|
||||
return c(new_value)
|
||||
|
||||
return c(new_value, instance, attrib)
|
||||
|
||||
return new_value
|
||||
|
||||
|
||||
# Sentinel for disabling class-wide *on_setattr* hooks for certain attributes.
|
||||
# Sphinx's autodata stopped working, so the docstring is inlined in the API
|
||||
# docs.
|
||||
NO_OP = object()
|
||||
@@ -0,0 +1,20 @@
|
||||
from typing import Any, NewType, NoReturn, TypeVar
|
||||
|
||||
from . import Attribute
|
||||
from attrs import _OnSetAttrType
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
def frozen(
|
||||
instance: Any, attribute: Attribute[Any], new_value: Any
|
||||
) -> NoReturn: ...
|
||||
def pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ...
|
||||
def validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ...
|
||||
|
||||
# convert is allowed to return Any, because they can be chained using pipe.
|
||||
def convert(
|
||||
instance: Any, attribute: Attribute[Any], new_value: Any
|
||||
) -> Any: ...
|
||||
|
||||
_NoOpType = NewType("_NoOpType", object)
|
||||
NO_OP: _NoOpType
|
||||
@@ -0,0 +1,750 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Commonly useful validators.
|
||||
"""
|
||||
|
||||
import operator
|
||||
import re
|
||||
|
||||
from contextlib import contextmanager
|
||||
from re import Pattern
|
||||
|
||||
from ._config import get_run_validators, set_run_validators
|
||||
from ._make import _AndValidator, and_, attrib, attrs
|
||||
from .converters import default_if_none
|
||||
from .exceptions import NotCallableError
|
||||
|
||||
|
||||
__all__ = [
|
||||
"and_",
|
||||
"deep_iterable",
|
||||
"deep_mapping",
|
||||
"disabled",
|
||||
"ge",
|
||||
"get_disabled",
|
||||
"gt",
|
||||
"in_",
|
||||
"instance_of",
|
||||
"is_callable",
|
||||
"le",
|
||||
"lt",
|
||||
"matches_re",
|
||||
"max_len",
|
||||
"min_len",
|
||||
"not_",
|
||||
"optional",
|
||||
"or_",
|
||||
"set_disabled",
|
||||
]
|
||||
|
||||
|
||||
def set_disabled(disabled):
|
||||
"""
|
||||
Globally disable or enable running validators.
|
||||
|
||||
By default, they are run.
|
||||
|
||||
Args:
|
||||
disabled (bool): If `True`, disable running all validators.
|
||||
|
||||
.. warning::
|
||||
|
||||
This function is not thread-safe!
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
set_run_validators(not disabled)
|
||||
|
||||
|
||||
def get_disabled():
|
||||
"""
|
||||
Return a bool indicating whether validators are currently disabled or not.
|
||||
|
||||
Returns:
|
||||
bool:`True` if validators are currently disabled.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return not get_run_validators()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def disabled():
|
||||
"""
|
||||
Context manager that disables running validators within its context.
|
||||
|
||||
.. warning::
|
||||
|
||||
This context manager is not thread-safe!
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
.. versionchanged:: 26.1.0 The contextmanager is nestable.
|
||||
"""
|
||||
prev = get_run_validators()
|
||||
set_run_validators(False)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
set_run_validators(prev)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _InstanceOfValidator:
|
||||
type = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if not isinstance(value, self.type):
|
||||
msg = f"'{attr.name}' must be {self.type!r} (got {value!r} that is a {value.__class__!r})."
|
||||
raise TypeError(
|
||||
msg,
|
||||
attr,
|
||||
self.type,
|
||||
value,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<instance_of validator for type {self.type!r}>"
|
||||
|
||||
|
||||
def instance_of(type):
|
||||
"""
|
||||
A validator that raises a `TypeError` if the initializer is called with a
|
||||
wrong type for this particular attribute (checks are performed using
|
||||
`isinstance` therefore it's also valid to pass a tuple of types).
|
||||
|
||||
Args:
|
||||
type (type | tuple[type]): The type to check for.
|
||||
|
||||
Raises:
|
||||
TypeError:
|
||||
With a human readable error message, the attribute (of type
|
||||
`attrs.Attribute`), the expected type, and the value it got.
|
||||
"""
|
||||
return _InstanceOfValidator(type)
|
||||
|
||||
|
||||
@attrs(repr=False, frozen=True, slots=True)
|
||||
class _MatchesReValidator:
|
||||
pattern = attrib()
|
||||
match_func = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if not self.match_func(value):
|
||||
msg = f"'{attr.name}' must match regex {self.pattern.pattern!r} ({value!r} doesn't)"
|
||||
raise ValueError(
|
||||
msg,
|
||||
attr,
|
||||
self.pattern,
|
||||
value,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<matches_re validator for pattern {self.pattern!r}>"
|
||||
|
||||
|
||||
def matches_re(regex, flags=0, func=None):
|
||||
r"""
|
||||
A validator that raises `ValueError` if the initializer is called with a
|
||||
string that doesn't match *regex*.
|
||||
|
||||
Args:
|
||||
regex (str, re.Pattern):
|
||||
A regex string or precompiled pattern to match against
|
||||
|
||||
flags (int):
|
||||
Flags that will be passed to the underlying re function (default 0)
|
||||
|
||||
func (typing.Callable):
|
||||
Which underlying `re` function to call. Valid options are
|
||||
`re.fullmatch`, `re.search`, and `re.match`; the default `None`
|
||||
means `re.fullmatch`. For performance reasons, the pattern is
|
||||
always precompiled using `re.compile`.
|
||||
|
||||
.. versionadded:: 19.2.0
|
||||
.. versionchanged:: 21.3.0 *regex* can be a pre-compiled pattern.
|
||||
"""
|
||||
valid_funcs = (re.fullmatch, None, re.search, re.match)
|
||||
if func not in valid_funcs:
|
||||
msg = "'func' must be one of {}.".format(
|
||||
", ".join(
|
||||
sorted((e and e.__name__) or "None" for e in set(valid_funcs))
|
||||
)
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
if isinstance(regex, Pattern):
|
||||
if flags:
|
||||
msg = "'flags' can only be used with a string pattern; pass flags to re.compile() instead"
|
||||
raise TypeError(msg)
|
||||
pattern = regex
|
||||
else:
|
||||
pattern = re.compile(regex, flags)
|
||||
|
||||
if func is re.match:
|
||||
match_func = pattern.match
|
||||
elif func is re.search:
|
||||
match_func = pattern.search
|
||||
else:
|
||||
match_func = pattern.fullmatch
|
||||
|
||||
return _MatchesReValidator(pattern, match_func)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _OptionalValidator:
|
||||
validator = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
if value is None:
|
||||
return
|
||||
|
||||
self.validator(inst, attr, value)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<optional validator for {self.validator!r} or None>"
|
||||
|
||||
|
||||
def optional(validator):
|
||||
"""
|
||||
A validator that makes an attribute optional. An optional attribute is one
|
||||
which can be set to `None` in addition to satisfying the requirements of
|
||||
the sub-validator.
|
||||
|
||||
Args:
|
||||
validator
|
||||
(typing.Callable | tuple[typing.Callable] | list[typing.Callable]):
|
||||
A validator (or validators) that is used for non-`None` values.
|
||||
|
||||
.. versionadded:: 15.1.0
|
||||
.. versionchanged:: 17.1.0 *validator* can be a list of validators.
|
||||
.. versionchanged:: 23.1.0 *validator* can also be a tuple of validators.
|
||||
"""
|
||||
if isinstance(validator, (list, tuple)):
|
||||
return _OptionalValidator(_AndValidator(validator))
|
||||
|
||||
return _OptionalValidator(validator)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _InValidator:
|
||||
options = attrib()
|
||||
_original_options = attrib(hash=False)
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
try:
|
||||
in_options = value in self.options
|
||||
except TypeError: # e.g. `1 in "abc"`
|
||||
in_options = False
|
||||
|
||||
if not in_options:
|
||||
msg = f"'{attr.name}' must be in {self._original_options!r} (got {value!r})"
|
||||
raise ValueError(
|
||||
msg,
|
||||
attr,
|
||||
self._original_options,
|
||||
value,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<in_ validator with options {self._original_options!r}>"
|
||||
|
||||
|
||||
def in_(options):
|
||||
"""
|
||||
A validator that raises a `ValueError` if the initializer is called with a
|
||||
value that does not belong in the *options* provided.
|
||||
|
||||
The check is performed using ``value in options``, so *options* has to
|
||||
support that operation.
|
||||
|
||||
To keep the validator hashable, dicts, lists, and sets are transparently
|
||||
transformed into a `tuple`.
|
||||
|
||||
Args:
|
||||
options: Allowed options.
|
||||
|
||||
Raises:
|
||||
ValueError:
|
||||
With a human readable error message, the attribute (of type
|
||||
`attrs.Attribute`), the expected options, and the value it got.
|
||||
|
||||
.. versionadded:: 17.1.0
|
||||
.. versionchanged:: 22.1.0
|
||||
The ValueError was incomplete until now and only contained the human
|
||||
readable error message. Now it contains all the information that has
|
||||
been promised since 17.1.0.
|
||||
.. versionchanged:: 24.1.0
|
||||
*options* that are a list, dict, or a set are now transformed into a
|
||||
tuple to keep the validator hashable.
|
||||
"""
|
||||
repr_options = options
|
||||
if isinstance(options, (list, dict, set)):
|
||||
options = tuple(options)
|
||||
|
||||
return _InValidator(options, repr_options)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=False, unsafe_hash=True)
|
||||
class _IsCallableValidator:
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if not callable(value):
|
||||
message = (
|
||||
"'{name}' must be callable "
|
||||
"(got {value!r} that is a {actual!r})."
|
||||
)
|
||||
raise NotCallableError(
|
||||
msg=message.format(
|
||||
name=attr.name, value=value, actual=value.__class__
|
||||
),
|
||||
value=value,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return "<is_callable validator>"
|
||||
|
||||
|
||||
def is_callable():
|
||||
"""
|
||||
A validator that raises a `attrs.exceptions.NotCallableError` if the
|
||||
initializer is called with a value for this particular attribute that is
|
||||
not callable.
|
||||
|
||||
.. versionadded:: 19.1.0
|
||||
|
||||
Raises:
|
||||
attrs.exceptions.NotCallableError:
|
||||
With a human readable error message containing the attribute
|
||||
(`attrs.Attribute`) name, and the value it got.
|
||||
"""
|
||||
return _IsCallableValidator()
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _DeepIterable:
|
||||
member_validator = attrib(validator=is_callable())
|
||||
iterable_validator = attrib(
|
||||
default=None, validator=optional(is_callable())
|
||||
)
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if self.iterable_validator is not None:
|
||||
self.iterable_validator(inst, attr, value)
|
||||
|
||||
for member in value:
|
||||
self.member_validator(inst, attr, member)
|
||||
|
||||
def __repr__(self):
|
||||
iterable_identifier = (
|
||||
""
|
||||
if self.iterable_validator is None
|
||||
else f" {self.iterable_validator!r}"
|
||||
)
|
||||
return (
|
||||
f"<deep_iterable validator for{iterable_identifier}"
|
||||
f" iterables of {self.member_validator!r}>"
|
||||
)
|
||||
|
||||
|
||||
def deep_iterable(member_validator, iterable_validator=None):
|
||||
"""
|
||||
A validator that performs deep validation of an iterable.
|
||||
|
||||
Args:
|
||||
member_validator: Validator(s) to apply to iterable members.
|
||||
|
||||
iterable_validator:
|
||||
Validator(s) to apply to iterable itself (optional).
|
||||
|
||||
Raises
|
||||
TypeError: if any sub-validators fail
|
||||
|
||||
.. versionadded:: 19.1.0
|
||||
|
||||
.. versionchanged:: 25.4.0
|
||||
*member_validator* and *iterable_validator* can now be a list or tuple
|
||||
of validators.
|
||||
"""
|
||||
if isinstance(member_validator, (list, tuple)):
|
||||
member_validator = and_(*member_validator)
|
||||
if isinstance(iterable_validator, (list, tuple)):
|
||||
iterable_validator = and_(*iterable_validator)
|
||||
return _DeepIterable(member_validator, iterable_validator)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _DeepMapping:
|
||||
key_validator = attrib(validator=optional(is_callable()))
|
||||
value_validator = attrib(validator=optional(is_callable()))
|
||||
mapping_validator = attrib(validator=optional(is_callable()))
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if self.mapping_validator is not None:
|
||||
self.mapping_validator(inst, attr, value)
|
||||
|
||||
for key in value:
|
||||
if self.key_validator is not None:
|
||||
self.key_validator(inst, attr, key)
|
||||
if self.value_validator is not None:
|
||||
self.value_validator(inst, attr, value[key])
|
||||
|
||||
def __repr__(self):
|
||||
return f"<deep_mapping validator for objects mapping {self.key_validator!r} to {self.value_validator!r}>"
|
||||
|
||||
|
||||
def deep_mapping(
|
||||
key_validator=None, value_validator=None, mapping_validator=None
|
||||
):
|
||||
"""
|
||||
A validator that performs deep validation of a dictionary.
|
||||
|
||||
All validators are optional, but at least one of *key_validator* or
|
||||
*value_validator* must be provided.
|
||||
|
||||
Args:
|
||||
key_validator: Validator(s) to apply to dictionary keys.
|
||||
|
||||
value_validator: Validator(s) to apply to dictionary values.
|
||||
|
||||
mapping_validator:
|
||||
Validator(s) to apply to top-level mapping attribute.
|
||||
|
||||
.. versionadded:: 19.1.0
|
||||
|
||||
.. versionchanged:: 25.4.0
|
||||
*key_validator* and *value_validator* are now optional, but at least one
|
||||
of them must be provided.
|
||||
|
||||
.. versionchanged:: 25.4.0
|
||||
*key_validator*, *value_validator*, and *mapping_validator* can now be a
|
||||
list or tuple of validators.
|
||||
|
||||
Raises:
|
||||
TypeError: If any sub-validator fails on validation.
|
||||
|
||||
ValueError:
|
||||
If neither *key_validator* nor *value_validator* is provided on
|
||||
instantiation.
|
||||
"""
|
||||
if key_validator is None and value_validator is None:
|
||||
msg = (
|
||||
"At least one of key_validator or value_validator must be provided"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
if isinstance(key_validator, (list, tuple)):
|
||||
key_validator = and_(*key_validator)
|
||||
if isinstance(value_validator, (list, tuple)):
|
||||
value_validator = and_(*value_validator)
|
||||
if isinstance(mapping_validator, (list, tuple)):
|
||||
mapping_validator = and_(*mapping_validator)
|
||||
|
||||
return _DeepMapping(key_validator, value_validator, mapping_validator)
|
||||
|
||||
|
||||
@attrs(repr=False, frozen=True, slots=True)
|
||||
class _NumberValidator:
|
||||
bound = attrib()
|
||||
compare_op = attrib()
|
||||
compare_func = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if not self.compare_func(value, self.bound):
|
||||
msg = f"'{attr.name}' must be {self.compare_op} {self.bound}: {value}"
|
||||
raise ValueError(msg)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Validator for x {self.compare_op} {self.bound}>"
|
||||
|
||||
|
||||
def lt(val):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called with a
|
||||
number larger or equal to *val*.
|
||||
|
||||
The validator uses `operator.lt` to compare the values.
|
||||
|
||||
Args:
|
||||
val: Exclusive upper bound for values.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _NumberValidator(val, "<", operator.lt)
|
||||
|
||||
|
||||
def le(val):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called with a
|
||||
number greater than *val*.
|
||||
|
||||
The validator uses `operator.le` to compare the values.
|
||||
|
||||
Args:
|
||||
val: Inclusive upper bound for values.
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _NumberValidator(val, "<=", operator.le)
|
||||
|
||||
|
||||
def ge(val):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called with a
|
||||
number smaller than *val*.
|
||||
|
||||
The validator uses `operator.ge` to compare the values.
|
||||
|
||||
Args:
|
||||
val: Inclusive lower bound for values
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _NumberValidator(val, ">=", operator.ge)
|
||||
|
||||
|
||||
def gt(val):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called with a
|
||||
number smaller or equal to *val*.
|
||||
|
||||
The validator uses `operator.gt` to compare the values.
|
||||
|
||||
Args:
|
||||
val: Exclusive lower bound for values
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _NumberValidator(val, ">", operator.gt)
|
||||
|
||||
|
||||
@attrs(repr=False, frozen=True, slots=True)
|
||||
class _MaxLengthValidator:
|
||||
max_length = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if len(value) > self.max_length:
|
||||
msg = f"Length of '{attr.name}' must be <= {self.max_length}: {len(value)}"
|
||||
raise ValueError(msg)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<max_len validator for {self.max_length}>"
|
||||
|
||||
|
||||
def max_len(length):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called
|
||||
with a string or iterable that is longer than *length*.
|
||||
|
||||
Args:
|
||||
length (int): Maximum length of the string or iterable
|
||||
|
||||
.. versionadded:: 21.3.0
|
||||
"""
|
||||
return _MaxLengthValidator(length)
|
||||
|
||||
|
||||
@attrs(repr=False, frozen=True, slots=True)
|
||||
class _MinLengthValidator:
|
||||
min_length = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if len(value) < self.min_length:
|
||||
msg = f"Length of '{attr.name}' must be >= {self.min_length}: {len(value)}"
|
||||
raise ValueError(msg)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<min_len validator for {self.min_length}>"
|
||||
|
||||
|
||||
def min_len(length):
|
||||
"""
|
||||
A validator that raises `ValueError` if the initializer is called
|
||||
with a string or iterable that is shorter than *length*.
|
||||
|
||||
Args:
|
||||
length (int): Minimum length of the string or iterable
|
||||
|
||||
.. versionadded:: 22.1.0
|
||||
"""
|
||||
return _MinLengthValidator(length)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _SubclassOfValidator:
|
||||
type = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
"""
|
||||
We use a callable class to be able to change the ``__repr__``.
|
||||
"""
|
||||
if not issubclass(value, self.type):
|
||||
msg = f"'{attr.name}' must be a subclass of {self.type!r} (got {value!r})."
|
||||
raise TypeError(
|
||||
msg,
|
||||
attr,
|
||||
self.type,
|
||||
value,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<subclass_of validator for type {self.type!r}>"
|
||||
|
||||
|
||||
def _subclass_of(type):
|
||||
"""
|
||||
A validator that raises a `TypeError` if the initializer is called with a
|
||||
wrong type for this particular attribute (checks are performed using
|
||||
`issubclass` therefore it's also valid to pass a tuple of types).
|
||||
|
||||
Args:
|
||||
type (type | tuple[type, ...]): The type(s) to check for.
|
||||
|
||||
Raises:
|
||||
TypeError:
|
||||
With a human readable error message, the attribute (of type
|
||||
`attrs.Attribute`), the expected type, and the value it got.
|
||||
"""
|
||||
return _SubclassOfValidator(type)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _NotValidator:
|
||||
validator = attrib()
|
||||
msg = attrib(
|
||||
converter=default_if_none(
|
||||
"not_ validator child '{validator!r}' "
|
||||
"did not raise a captured error"
|
||||
)
|
||||
)
|
||||
exc_types = attrib(
|
||||
validator=deep_iterable(
|
||||
member_validator=_subclass_of(Exception),
|
||||
iterable_validator=instance_of(tuple),
|
||||
),
|
||||
)
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
try:
|
||||
self.validator(inst, attr, value)
|
||||
except self.exc_types:
|
||||
pass # suppress error to invert validity
|
||||
else:
|
||||
raise ValueError(
|
||||
self.msg.format(
|
||||
validator=self.validator,
|
||||
exc_types=self.exc_types,
|
||||
),
|
||||
attr,
|
||||
self.validator,
|
||||
value,
|
||||
self.exc_types,
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<not_ validator wrapping {self.validator!r}, capturing {self.exc_types!r}>"
|
||||
|
||||
|
||||
def not_(validator, *, msg=None, exc_types=(ValueError, TypeError)):
|
||||
"""
|
||||
A validator that wraps and logically 'inverts' the validator passed to it.
|
||||
It will raise a `ValueError` if the provided validator *doesn't* raise a
|
||||
`ValueError` or `TypeError` (by default), and will suppress the exception
|
||||
if the provided validator *does*.
|
||||
|
||||
Intended to be used with existing validators to compose logic without
|
||||
needing to create inverted variants, for example, ``not_(in_(...))``.
|
||||
|
||||
Args:
|
||||
validator: A validator to be logically inverted.
|
||||
|
||||
msg (str):
|
||||
Message to raise if validator fails. Formatted with keys
|
||||
``exc_types`` and ``validator``.
|
||||
|
||||
exc_types (tuple[type, ...]):
|
||||
Exception type(s) to capture. Other types raised by child
|
||||
validators will not be intercepted and pass through.
|
||||
|
||||
Raises:
|
||||
ValueError:
|
||||
With a human readable error message, the attribute (of type
|
||||
`attrs.Attribute`), the validator that failed to raise an
|
||||
exception, the value it got, and the expected exception types.
|
||||
|
||||
.. versionadded:: 22.2.0
|
||||
"""
|
||||
try:
|
||||
exc_types = tuple(exc_types)
|
||||
except TypeError:
|
||||
exc_types = (exc_types,)
|
||||
return _NotValidator(validator, msg, exc_types)
|
||||
|
||||
|
||||
@attrs(repr=False, slots=True, unsafe_hash=True)
|
||||
class _OrValidator:
|
||||
validators = attrib()
|
||||
|
||||
def __call__(self, inst, attr, value):
|
||||
for v in self.validators:
|
||||
try:
|
||||
v(inst, attr, value)
|
||||
except Exception: # noqa: BLE001, PERF203, S112
|
||||
continue
|
||||
else:
|
||||
return
|
||||
|
||||
msg = f"None of {self.validators!r} satisfied for value {value!r}"
|
||||
raise ValueError(msg)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<or validator wrapping {self.validators!r}>"
|
||||
|
||||
|
||||
def or_(*validators):
|
||||
"""
|
||||
A validator that composes multiple validators into one.
|
||||
|
||||
When called on a value, it runs all wrapped validators until one of them is
|
||||
satisfied.
|
||||
|
||||
Args:
|
||||
validators (~collections.abc.Iterable[typing.Callable]):
|
||||
Arbitrary number of validators.
|
||||
|
||||
Raises:
|
||||
ValueError:
|
||||
If no validator is satisfied. Raised with a human-readable error
|
||||
message listing all the wrapped validators and the value that
|
||||
failed all of them.
|
||||
|
||||
.. versionadded:: 24.1.0
|
||||
"""
|
||||
vals = []
|
||||
for v in validators:
|
||||
vals.extend(v.validators if isinstance(v, _OrValidator) else [v])
|
||||
|
||||
return _OrValidator(tuple(vals))
|
||||
@@ -0,0 +1,140 @@
|
||||
from types import UnionType
|
||||
from typing import (
|
||||
Any,
|
||||
AnyStr,
|
||||
Callable,
|
||||
Container,
|
||||
ContextManager,
|
||||
Iterable,
|
||||
Mapping,
|
||||
Match,
|
||||
Pattern,
|
||||
TypeVar,
|
||||
overload,
|
||||
)
|
||||
|
||||
from attrs import _ValidatorType
|
||||
from attrs import _ValidatorArgType
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_T1 = TypeVar("_T1")
|
||||
_T2 = TypeVar("_T2")
|
||||
_T3 = TypeVar("_T3")
|
||||
_T4 = TypeVar("_T4")
|
||||
_T5 = TypeVar("_T5")
|
||||
_T6 = TypeVar("_T6")
|
||||
_I = TypeVar("_I", bound=Iterable)
|
||||
_K = TypeVar("_K")
|
||||
_V = TypeVar("_V")
|
||||
_M = TypeVar("_M", bound=Mapping)
|
||||
|
||||
def set_disabled(run: bool) -> None: ...
|
||||
def get_disabled() -> bool: ...
|
||||
def disabled() -> ContextManager[None]: ...
|
||||
|
||||
# To be more precise on instance_of use some overloads.
|
||||
# If there are more than 3 items in the tuple then we fall back to Any
|
||||
@overload
|
||||
def instance_of(type: type[_T]) -> _ValidatorType[_T]: ...
|
||||
@overload
|
||||
def instance_of(type: tuple[type[_T]]) -> _ValidatorType[_T]: ...
|
||||
@overload
|
||||
def instance_of(
|
||||
type: tuple[type[_T1], type[_T2]],
|
||||
) -> _ValidatorType[_T1 | _T2]: ...
|
||||
@overload
|
||||
def instance_of(
|
||||
type: tuple[type[_T1], type[_T2], type[_T3]],
|
||||
) -> _ValidatorType[_T1 | _T2 | _T3]: ...
|
||||
@overload
|
||||
def instance_of(type: tuple[type, ...]) -> _ValidatorType[Any]: ...
|
||||
@overload
|
||||
def instance_of(type: UnionType) -> _ValidatorType[Any]: ...
|
||||
def optional(
|
||||
validator: (
|
||||
_ValidatorType[_T]
|
||||
| list[_ValidatorType[_T]]
|
||||
| tuple[_ValidatorType[_T], ...]
|
||||
),
|
||||
) -> _ValidatorType[_T | None]: ...
|
||||
def in_(options: Container[_T]) -> _ValidatorType[_T]: ...
|
||||
def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ...
|
||||
def matches_re(
|
||||
regex: Pattern[AnyStr] | AnyStr,
|
||||
flags: int = ...,
|
||||
func: Callable[[AnyStr, AnyStr, int], Match[AnyStr] | None] | None = ...,
|
||||
) -> _ValidatorType[AnyStr]: ...
|
||||
def deep_iterable(
|
||||
member_validator: _ValidatorArgType[_T],
|
||||
iterable_validator: _ValidatorArgType[_I] | None = ...,
|
||||
) -> _ValidatorType[_I]: ...
|
||||
@overload
|
||||
def deep_mapping(
|
||||
key_validator: _ValidatorArgType[_K],
|
||||
value_validator: _ValidatorArgType[_V] | None = ...,
|
||||
mapping_validator: _ValidatorArgType[_M] | None = ...,
|
||||
) -> _ValidatorType[_M]: ...
|
||||
@overload
|
||||
def deep_mapping(
|
||||
key_validator: _ValidatorArgType[_K] | None = ...,
|
||||
value_validator: _ValidatorArgType[_V] = ...,
|
||||
mapping_validator: _ValidatorArgType[_M] | None = ...,
|
||||
) -> _ValidatorType[_M]: ...
|
||||
def is_callable() -> _ValidatorType[_T]: ...
|
||||
def lt(val: _T) -> _ValidatorType[_T]: ...
|
||||
def le(val: _T) -> _ValidatorType[_T]: ...
|
||||
def ge(val: _T) -> _ValidatorType[_T]: ...
|
||||
def gt(val: _T) -> _ValidatorType[_T]: ...
|
||||
def max_len(length: int) -> _ValidatorType[_T]: ...
|
||||
def min_len(length: int) -> _ValidatorType[_T]: ...
|
||||
def not_(
|
||||
validator: _ValidatorType[_T],
|
||||
*,
|
||||
msg: str | None = None,
|
||||
exc_types: type[Exception] | Iterable[type[Exception]] = ...,
|
||||
) -> _ValidatorType[_T]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[_T1],
|
||||
__v2: _ValidatorType[_T2],
|
||||
) -> _ValidatorType[_T1 | _T2]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[_T1],
|
||||
__v2: _ValidatorType[_T2],
|
||||
__v3: _ValidatorType[_T3],
|
||||
) -> _ValidatorType[_T1 | _T2 | _T3]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[_T1],
|
||||
__v2: _ValidatorType[_T2],
|
||||
__v3: _ValidatorType[_T3],
|
||||
__v4: _ValidatorType[_T4],
|
||||
) -> _ValidatorType[_T1 | _T2 | _T3 | _T4]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[_T1],
|
||||
__v2: _ValidatorType[_T2],
|
||||
__v3: _ValidatorType[_T3],
|
||||
__v4: _ValidatorType[_T4],
|
||||
__v5: _ValidatorType[_T5],
|
||||
) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[_T1],
|
||||
__v2: _ValidatorType[_T2],
|
||||
__v3: _ValidatorType[_T3],
|
||||
__v4: _ValidatorType[_T4],
|
||||
__v5: _ValidatorType[_T5],
|
||||
__v6: _ValidatorType[_T6],
|
||||
) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5 | _T6]: ...
|
||||
@overload
|
||||
def or_(
|
||||
__v1: _ValidatorType[Any],
|
||||
__v2: _ValidatorType[Any],
|
||||
__v3: _ValidatorType[Any],
|
||||
__v4: _ValidatorType[Any],
|
||||
__v5: _ValidatorType[Any],
|
||||
__v6: _ValidatorType[Any],
|
||||
*validators: _ValidatorType[Any],
|
||||
) -> _ValidatorType[Any]: ...
|
||||
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,199 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: attrs
|
||||
Version: 26.1.0
|
||||
Summary: Classes Without Boilerplate
|
||||
Project-URL: Documentation, https://www.attrs.org/
|
||||
Project-URL: Changelog, https://www.attrs.org/en/stable/changelog.html
|
||||
Project-URL: GitHub, https://github.com/python-attrs/attrs
|
||||
Project-URL: Funding, https://github.com/sponsors/hynek
|
||||
Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-attrs?utm_source=pypi-attrs&utm_medium=pypi
|
||||
Author-email: Hynek Schlawack <hs@ox.cx>
|
||||
License-Expression: MIT
|
||||
License-File: LICENSE
|
||||
Keywords: attribute,boilerplate,class
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Programming Language :: Python :: 3.13
|
||||
Classifier: Programming Language :: Python :: 3.14
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Classifier: Typing :: Typed
|
||||
Requires-Python: >=3.9
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.attrs.org/">
|
||||
<img src="https://raw.githubusercontent.com/python-attrs/attrs/main/docs/_static/attrs_logo.svg" width="35%" alt="attrs" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
|
||||
*attrs* is the Python package that will bring back the **joy** of **writing classes** by relieving you from the drudgery of implementing object protocols (aka [dunder methods](https://www.attrs.org/en/latest/glossary.html#term-dunder-methods)).
|
||||
Trusted by NASA for [Mars missions since 2020](https://github.com/readme/featured/nasa-ingenuity-helicopter)!
|
||||
|
||||
Its main goal is to help you to write **concise** and **correct** software without slowing down your code.
|
||||
|
||||
|
||||
## Sponsors
|
||||
|
||||
*attrs* would not be possible without our [amazing sponsors](https://github.com/sponsors/hynek).
|
||||
Especially those generously supporting us at the *The Organization* tier and higher:
|
||||
|
||||
<!-- sponsor-break-begin -->
|
||||
|
||||
<p align="center">
|
||||
|
||||
<!-- [[[cog
|
||||
import pathlib, tomllib
|
||||
|
||||
for sponsor in tomllib.loads(pathlib.Path("pyproject.toml").read_text())["tool"]["sponcon"]["sponsors"]:
|
||||
print(f'<a href="{sponsor["url"]}"><img title="{sponsor["title"]}" src="https://www.attrs.org/en/26.1.0/_static/sponsors/{sponsor["img"]}" width="190" /></a>')
|
||||
]]] -->
|
||||
<a href="https://www.variomedia.de/"><img title="Variomedia AG" src="https://www.attrs.org/en/26.1.0/_static/sponsors/Variomedia.svg" width="190" /></a>
|
||||
<a href="https://tidelift.com/?utm_source=lifter&utm_medium=referral&utm_campaign=hynek"><img title="Tidelift" src="https://www.attrs.org/en/26.1.0/_static/sponsors/Tidelift.svg" width="190" /></a>
|
||||
<a href="https://kraken.tech/"><img title="Kraken Tech" src="https://www.attrs.org/en/26.1.0/_static/sponsors/Kraken.svg" width="190" /></a>
|
||||
<a href="https://privacy-solutions.org/"><img title="Privacy Solutions" src="https://www.attrs.org/en/26.1.0/_static/sponsors/Privacy-Solutions.svg" width="190" /></a>
|
||||
<a href="https://filepreviews.io/"><img title="FilePreviews" src="https://www.attrs.org/en/26.1.0/_static/sponsors/FilePreviews.svg" width="190" /></a>
|
||||
<a href="https://www.testmuai.com/?utm_medium=sponsor&utm_source=structlog"><img title="TestMu AI" src="https://www.attrs.org/en/26.1.0/_static/sponsors/TestMu-AI.svg" width="190" /></a>
|
||||
<a href="https://polar.sh/"><img title="Polar" src="https://www.attrs.org/en/26.1.0/_static/sponsors/Polar.svg" width="190" /></a>
|
||||
<!-- [[[end]]] -->
|
||||
|
||||
</p>
|
||||
|
||||
<!-- sponsor-break-end -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Please consider <a href="https://github.com/sponsors/hynek">joining them</a> to help make <em>attrs</em>’s maintenance more sustainable!</strong>
|
||||
</p>
|
||||
|
||||
<!-- teaser-end -->
|
||||
|
||||
## Example
|
||||
|
||||
*attrs* gives you a class decorator and a way to declaratively define the attributes on that class:
|
||||
|
||||
<!-- code-begin -->
|
||||
|
||||
```pycon
|
||||
>>> from attrs import asdict, define, make_class, Factory
|
||||
|
||||
>>> @define
|
||||
... class SomeClass:
|
||||
... a_number: int = 42
|
||||
... list_of_numbers: list[int] = Factory(list)
|
||||
...
|
||||
... def hard_math(self, another_number):
|
||||
... return self.a_number + sum(self.list_of_numbers) * another_number
|
||||
|
||||
|
||||
>>> sc = SomeClass(1, [1, 2, 3])
|
||||
>>> sc
|
||||
SomeClass(a_number=1, list_of_numbers=[1, 2, 3])
|
||||
|
||||
>>> sc.hard_math(3)
|
||||
19
|
||||
>>> sc == SomeClass(1, [1, 2, 3])
|
||||
True
|
||||
>>> sc != SomeClass(2, [3, 2, 1])
|
||||
True
|
||||
|
||||
>>> asdict(sc)
|
||||
{'a_number': 1, 'list_of_numbers': [1, 2, 3]}
|
||||
|
||||
>>> SomeClass()
|
||||
SomeClass(a_number=42, list_of_numbers=[])
|
||||
|
||||
>>> C = make_class("C", ["a", "b"])
|
||||
>>> C("foo", "bar")
|
||||
C(a='foo', b='bar')
|
||||
```
|
||||
|
||||
After *declaring* your attributes, *attrs* gives you:
|
||||
|
||||
- a concise and explicit overview of the class's attributes,
|
||||
- a nice human-readable `__repr__`,
|
||||
- equality-checking methods,
|
||||
- an initializer,
|
||||
- and much more,
|
||||
|
||||
*without* writing dull boilerplate code again and again and *without* runtime performance penalties.
|
||||
|
||||
---
|
||||
|
||||
This example uses *attrs*'s modern APIs that have been introduced in version 20.1.0, and the *attrs* package import name that has been added in version 21.3.0.
|
||||
The classic APIs (`@attr.s`, `attr.ib`, plus their serious-business aliases) and the `attr` package import name will remain **indefinitely**.
|
||||
|
||||
Check out [*On The Core API Names*](https://www.attrs.org/en/latest/names.html) for an in-depth explanation!
|
||||
|
||||
|
||||
### Hate Type Annotations!?
|
||||
|
||||
No problem!
|
||||
Types are entirely **optional** with *attrs*.
|
||||
Simply assign `attrs.field()` to the attributes instead of annotating them with types:
|
||||
|
||||
```python
|
||||
from attrs import define, field
|
||||
|
||||
@define
|
||||
class SomeClass:
|
||||
a_number = field(default=42)
|
||||
list_of_numbers = field(factory=list)
|
||||
```
|
||||
|
||||
|
||||
## Data Classes
|
||||
|
||||
On the tin, *attrs* might remind you of `dataclasses` (and indeed, `dataclasses` [are a descendant](https://hynek.me/articles/import-attrs/) of *attrs*).
|
||||
In practice it does a lot more and is more flexible.
|
||||
For instance, it allows you to define [special handling of NumPy arrays for equality checks](https://www.attrs.org/en/stable/comparison.html#customization), allows more ways to [plug into the initialization process](https://www.attrs.org/en/stable/init.html#hooking-yourself-into-initialization), has a replacement for `__init_subclass__`, and allows for stepping through the generated methods using a debugger.
|
||||
|
||||
For more details, please refer to our [comparison page](https://www.attrs.org/en/stable/why.html#data-classes), but generally speaking, we are more likely to commit crimes against nature to make things work that one would expect to work, but that are quite complicated in practice.
|
||||
|
||||
|
||||
## Project Information
|
||||
|
||||
- [**Changelog**](https://www.attrs.org/en/stable/changelog.html)
|
||||
- [**Documentation**](https://www.attrs.org/)
|
||||
- [**PyPI**](https://pypi.org/project/attrs/)
|
||||
- [**Source Code**](https://github.com/python-attrs/attrs)
|
||||
- [**Contributing**](https://github.com/python-attrs/attrs/blob/main/.github/CONTRIBUTING.md)
|
||||
- [**Third-party Extensions**](https://github.com/python-attrs/attrs/wiki/Extensions-to-attrs)
|
||||
- **Get Help**: use the `python-attrs` tag on [Stack Overflow](https://stackoverflow.com/questions/tagged/python-attrs)
|
||||
|
||||
|
||||
### *attrs* for Enterprise
|
||||
|
||||
Available as part of the [Tidelift Subscription](https://tidelift.com/?utm_source=lifter&utm_medium=referral&utm_campaign=hynek).
|
||||
|
||||
The maintainers of *attrs* and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source packages you use to build your applications.
|
||||
Save time, reduce risk, and improve code health, while paying the maintainers of the exact packages you use.
|
||||
|
||||
## Release Information
|
||||
|
||||
### Backwards-incompatible Changes
|
||||
|
||||
- Field aliases are now resolved *before* calling `field_transformer`, so transformers receive fully populated `Attribute` objects with usable `alias` values instead of `None`.
|
||||
The new `Attribute.alias_is_default` flag indicates whether the alias was auto-generated (`True`) or explicitly set by the user (`False`).
|
||||
[#1509](https://github.com/python-attrs/attrs/issues/1509)
|
||||
|
||||
|
||||
### Changes
|
||||
|
||||
- Fix type annotations for `attrs.validators.optional()`, so it no longer rejects tuples with more than one validator.
|
||||
[#1496](https://github.com/python-attrs/attrs/issues/1496)
|
||||
- The `attrs.validators.disabled()` contextmanager can now be nested.
|
||||
[#1513](https://github.com/python-attrs/attrs/issues/1513)
|
||||
- Frozen classes can set `on_setattr=attrs.setters.NO_OP` in addition to `None`.
|
||||
[#1515](https://github.com/python-attrs/attrs/issues/1515)
|
||||
- It's now possible to pass *attrs* **instances** in addition to *attrs* **classes** to `attrs.fields()`.
|
||||
[#1529](https://github.com/python-attrs/attrs/issues/1529)
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
[Full changelog →](https://www.attrs.org/en/stable/changelog.html)
|
||||
@@ -0,0 +1,55 @@
|
||||
attr/__init__.py,sha256=fOYIvt1eGSqQre4uCS3sJWKZ0mwAuC8UD6qba5OS9_U,2057
|
||||
attr/__init__.pyi,sha256=pVGImAUVovq2_TYl_r_HIYnGlyOaoCuEhxo-EvsnnSc,11325
|
||||
attr/__pycache__/__init__.cpython-312.pyc,,
|
||||
attr/__pycache__/_cmp.cpython-312.pyc,,
|
||||
attr/__pycache__/_compat.cpython-312.pyc,,
|
||||
attr/__pycache__/_config.cpython-312.pyc,,
|
||||
attr/__pycache__/_funcs.cpython-312.pyc,,
|
||||
attr/__pycache__/_make.cpython-312.pyc,,
|
||||
attr/__pycache__/_next_gen.cpython-312.pyc,,
|
||||
attr/__pycache__/_version_info.cpython-312.pyc,,
|
||||
attr/__pycache__/converters.cpython-312.pyc,,
|
||||
attr/__pycache__/exceptions.cpython-312.pyc,,
|
||||
attr/__pycache__/filters.cpython-312.pyc,,
|
||||
attr/__pycache__/setters.cpython-312.pyc,,
|
||||
attr/__pycache__/validators.cpython-312.pyc,,
|
||||
attr/_cmp.py,sha256=3Nn1TjxllUYiX_nJoVnEkXoDk0hM1DYKj5DE7GZe4i0,4117
|
||||
attr/_cmp.pyi,sha256=U-_RU_UZOyPUEQzXE6RMYQQcjkZRY25wTH99sN0s7MM,368
|
||||
attr/_compat.py,sha256=x0g7iEUOnBVJC72zyFCgb1eKqyxS-7f2LGnNyZ_r95s,2829
|
||||
attr/_config.py,sha256=dGq3xR6fgZEF6UBt_L0T-eUHIB4i43kRmH0P28sJVw8,843
|
||||
attr/_funcs.py,sha256=Ix5IETTfz5F01F-12MF_CSFomIn2h8b67EVVz2gCtBE,16479
|
||||
attr/_make.py,sha256=H7OH2eWS5CnBzLUjNFE1WymfPrmF1r8fv2RPdt9MuYA,106129
|
||||
attr/_next_gen.py,sha256=BQtCUlzwg2gWHTYXBQvrEYBnzBUrDvO57u0Py6UCPhc,26274
|
||||
attr/_typing_compat.pyi,sha256=XDP54TUn-ZKhD62TOQebmzrwFyomhUCoGRpclb6alRA,469
|
||||
attr/_version_info.py,sha256=w4R-FYC3NK_kMkGUWJlYP4cVAlH9HRaC-um3fcjYkHM,2222
|
||||
attr/_version_info.pyi,sha256=x_M3L3WuB7r_ULXAWjx959udKQ4HLB8l-hsc1FDGNvk,209
|
||||
attr/converters.py,sha256=GlDeOzPeTFgeBBLbj9G57Ez5lAk68uhSALRYJ_exe84,3861
|
||||
attr/converters.pyi,sha256=orU2bff-VjQa2kMDyvnMQV73oJT2WRyQuw4ZR1ym1bE,643
|
||||
attr/exceptions.py,sha256=b4vMbnoQ3VpwWZhqrYi_ssXVCK8o2c4HQSS09cSUM9o,1990
|
||||
attr/exceptions.pyi,sha256=zZq8bCUnKAy9mDtBEw42ZhPhAUIHoTKedDQInJD883M,539
|
||||
attr/filters.py,sha256=ZBiKWLp3R0LfCZsq7X11pn9WX8NslS2wXM4jsnLOGc8,1795
|
||||
attr/filters.pyi,sha256=3J5BG-dTxltBk1_-RuNRUHrv2qu1v8v4aDNAQ7_mifA,208
|
||||
attr/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
attr/setters.py,sha256=5-dcT63GQK35ONEzSgfXCkbB7pPkaR-qv15mm4PVSzQ,1617
|
||||
attr/setters.pyi,sha256=NnVkaFU1BB4JB8E4JuXyrzTUgvtMpj8p3wBdJY7uix4,584
|
||||
attr/validators.py,sha256=m3QRzZTANr4f2C4eVdUoFg11NgXWak8Wat4qQTGhvcs,21553
|
||||
attr/validators.pyi,sha256=gM1ZmHaBckyYWI2EirpRNzqm3B19cw5Iq6B4Kno9YCM,4087
|
||||
attrs-26.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
attrs-26.1.0.dist-info/METADATA,sha256=TNQOaQ8jvzfLytNO_WdY4GLfHfB8hoM_fjzpW_H6OMw,8754
|
||||
attrs-26.1.0.dist-info/RECORD,,
|
||||
attrs-26.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
||||
attrs-26.1.0.dist-info/licenses/LICENSE,sha256=iCEVyV38KvHutnFPjsbVy8q_Znyv-HKfQkINpj9xTp8,1109
|
||||
attrs/__init__.py,sha256=RxaAZNwYiEh-fcvHLZNpQ_DWKni73M_jxEPEftiq1Zc,1183
|
||||
attrs/__init__.pyi,sha256=2gV79g9UxJppGSM48hAZJ6h_MHb70dZoJL31ZNJeZYI,9416
|
||||
attrs/__pycache__/__init__.cpython-312.pyc,,
|
||||
attrs/__pycache__/converters.cpython-312.pyc,,
|
||||
attrs/__pycache__/exceptions.cpython-312.pyc,,
|
||||
attrs/__pycache__/filters.cpython-312.pyc,,
|
||||
attrs/__pycache__/setters.cpython-312.pyc,,
|
||||
attrs/__pycache__/validators.cpython-312.pyc,,
|
||||
attrs/converters.py,sha256=8kQljrVwfSTRu8INwEk8SI0eGrzmWftsT7rM0EqyohM,76
|
||||
attrs/exceptions.py,sha256=ACCCmg19-vDFaDPY9vFl199SPXCQMN_bENs4DALjzms,76
|
||||
attrs/filters.py,sha256=VOUMZug9uEU6dUuA0dF1jInUK0PL3fLgP0VBS5d-CDE,73
|
||||
attrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
attrs/setters.py,sha256=eL1YidYQV3T2h9_SYIZSZR1FAcHGb1TuCTy0E0Lv2SU,73
|
||||
attrs/validators.py,sha256=xcy6wD5TtTkdCG1f4XWbocPSO0faBjk5IfVJfP6SUj0,76
|
||||
@@ -0,0 +1,4 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: hatchling 1.29.0
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
@@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Hynek Schlawack and the attrs contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr import (
|
||||
NOTHING,
|
||||
Attribute,
|
||||
AttrsInstance,
|
||||
Converter,
|
||||
Factory,
|
||||
NothingType,
|
||||
_make_getattr,
|
||||
assoc,
|
||||
cmp_using,
|
||||
define,
|
||||
evolve,
|
||||
field,
|
||||
fields,
|
||||
fields_dict,
|
||||
frozen,
|
||||
has,
|
||||
make_class,
|
||||
mutable,
|
||||
resolve_types,
|
||||
validate,
|
||||
)
|
||||
from attr._make import ClassProps
|
||||
from attr._next_gen import asdict, astuple, inspect
|
||||
|
||||
from . import converters, exceptions, filters, setters, validators
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NOTHING",
|
||||
"Attribute",
|
||||
"AttrsInstance",
|
||||
"ClassProps",
|
||||
"Converter",
|
||||
"Factory",
|
||||
"NothingType",
|
||||
"__author__",
|
||||
"__copyright__",
|
||||
"__description__",
|
||||
"__doc__",
|
||||
"__email__",
|
||||
"__license__",
|
||||
"__title__",
|
||||
"__url__",
|
||||
"__version__",
|
||||
"__version_info__",
|
||||
"asdict",
|
||||
"assoc",
|
||||
"astuple",
|
||||
"cmp_using",
|
||||
"converters",
|
||||
"define",
|
||||
"evolve",
|
||||
"exceptions",
|
||||
"field",
|
||||
"fields",
|
||||
"fields_dict",
|
||||
"filters",
|
||||
"frozen",
|
||||
"has",
|
||||
"inspect",
|
||||
"make_class",
|
||||
"mutable",
|
||||
"resolve_types",
|
||||
"setters",
|
||||
"validate",
|
||||
"validators",
|
||||
]
|
||||
|
||||
__getattr__ = _make_getattr(__name__)
|
||||
@@ -0,0 +1,314 @@
|
||||
import sys
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Mapping,
|
||||
Sequence,
|
||||
overload,
|
||||
TypeVar,
|
||||
)
|
||||
|
||||
# Because we need to type our own stuff, we have to make everything from
|
||||
# attr explicitly public too.
|
||||
from attr import __author__ as __author__
|
||||
from attr import __copyright__ as __copyright__
|
||||
from attr import __description__ as __description__
|
||||
from attr import __email__ as __email__
|
||||
from attr import __license__ as __license__
|
||||
from attr import __title__ as __title__
|
||||
from attr import __url__ as __url__
|
||||
from attr import __version__ as __version__
|
||||
from attr import __version_info__ as __version_info__
|
||||
from attr import assoc as assoc
|
||||
from attr import Attribute as Attribute
|
||||
from attr import AttrsInstance as AttrsInstance
|
||||
from attr import cmp_using as cmp_using
|
||||
from attr import converters as converters
|
||||
from attr import Converter as Converter
|
||||
from attr import evolve as evolve
|
||||
from attr import exceptions as exceptions
|
||||
from attr import Factory as Factory
|
||||
from attr import fields as fields
|
||||
from attr import fields_dict as fields_dict
|
||||
from attr import filters as filters
|
||||
from attr import has as has
|
||||
from attr import make_class as make_class
|
||||
from attr import NOTHING as NOTHING
|
||||
from attr import resolve_types as resolve_types
|
||||
from attr import setters as setters
|
||||
from attr import validate as validate
|
||||
from attr import validators as validators
|
||||
from attr import attrib, asdict as asdict, astuple as astuple
|
||||
from attr import NothingType as NothingType
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
from typing import dataclass_transform
|
||||
else:
|
||||
from typing_extensions import dataclass_transform
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_C = TypeVar("_C", bound=type)
|
||||
|
||||
_EqOrderType = bool | Callable[[Any], Any]
|
||||
_ValidatorType = Callable[[Any, "Attribute[_T]", _T], Any]
|
||||
_CallableConverterType = Callable[[Any], Any]
|
||||
_ConverterType = _CallableConverterType | Converter[Any, Any]
|
||||
_ReprType = Callable[[Any], str]
|
||||
_ReprArgType = bool | _ReprType
|
||||
_OnSetAttrType = Callable[[Any, "Attribute[Any]", Any], Any]
|
||||
_OnSetAttrArgType = _OnSetAttrType | list[_OnSetAttrType] | setters._NoOpType
|
||||
_FieldTransformer = Callable[
|
||||
[type, list["Attribute[Any]"]], list["Attribute[Any]"]
|
||||
]
|
||||
# FIXME: in reality, if multiple validators are passed they must be in a list
|
||||
# or tuple, but those are invariant and so would prevent subtypes of
|
||||
# _ValidatorType from working when passed in a list or tuple.
|
||||
_ValidatorArgType = _ValidatorType[_T] | Sequence[_ValidatorType[_T]]
|
||||
|
||||
@overload
|
||||
def field(
|
||||
*,
|
||||
default: None = ...,
|
||||
validator: None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
converter: None = ...,
|
||||
factory: None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: bool | None = ...,
|
||||
order: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
type: type | None = ...,
|
||||
) -> Any: ...
|
||||
|
||||
# This form catches an explicit None or no default and infers the type from the
|
||||
# other arguments.
|
||||
@overload
|
||||
def field(
|
||||
*,
|
||||
default: None = ...,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType, ...]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
type: type | None = ...,
|
||||
) -> _T: ...
|
||||
|
||||
# This form catches an explicit default argument.
|
||||
@overload
|
||||
def field(
|
||||
*,
|
||||
default: _T,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType, ...]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
type: type | None = ...,
|
||||
) -> _T: ...
|
||||
|
||||
# This form covers type=non-Type: e.g. forward references (str), Any
|
||||
@overload
|
||||
def field(
|
||||
*,
|
||||
default: _T | None = ...,
|
||||
validator: _ValidatorArgType[_T] | None = ...,
|
||||
repr: _ReprArgType = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
metadata: Mapping[Any, Any] | None = ...,
|
||||
converter: _ConverterType
|
||||
| list[_ConverterType]
|
||||
| tuple[_ConverterType, ...]
|
||||
| None = ...,
|
||||
factory: Callable[[], _T] | None = ...,
|
||||
kw_only: bool | None = ...,
|
||||
eq: _EqOrderType | None = ...,
|
||||
order: _EqOrderType | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
alias: str | None = ...,
|
||||
type: type | None = ...,
|
||||
) -> Any: ...
|
||||
@overload
|
||||
@dataclass_transform(field_specifiers=(attrib, field))
|
||||
def define(
|
||||
maybe_cls: _C,
|
||||
*,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: bool | None = ...,
|
||||
order: bool | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
) -> _C: ...
|
||||
@overload
|
||||
@dataclass_transform(field_specifiers=(attrib, field))
|
||||
def define(
|
||||
maybe_cls: None = ...,
|
||||
*,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: bool | None = ...,
|
||||
order: bool | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
) -> Callable[[_C], _C]: ...
|
||||
|
||||
mutable = define
|
||||
|
||||
@overload
|
||||
@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field))
|
||||
def frozen(
|
||||
maybe_cls: _C,
|
||||
*,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: bool | None = ...,
|
||||
order: bool | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
) -> _C: ...
|
||||
@overload
|
||||
@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field))
|
||||
def frozen(
|
||||
maybe_cls: None = ...,
|
||||
*,
|
||||
these: dict[str, Any] | None = ...,
|
||||
repr: bool = ...,
|
||||
unsafe_hash: bool | None = ...,
|
||||
hash: bool | None = ...,
|
||||
init: bool = ...,
|
||||
slots: bool = ...,
|
||||
frozen: bool = ...,
|
||||
weakref_slot: bool = ...,
|
||||
str: bool = ...,
|
||||
auto_attribs: bool = ...,
|
||||
kw_only: bool = ...,
|
||||
cache_hash: bool = ...,
|
||||
auto_exc: bool = ...,
|
||||
eq: bool | None = ...,
|
||||
order: bool | None = ...,
|
||||
auto_detect: bool = ...,
|
||||
getstate_setstate: bool | None = ...,
|
||||
on_setattr: _OnSetAttrArgType | None = ...,
|
||||
field_transformer: _FieldTransformer | None = ...,
|
||||
match_args: bool = ...,
|
||||
) -> Callable[[_C], _C]: ...
|
||||
|
||||
class ClassProps:
|
||||
# XXX: somehow when defining/using enums Mypy starts looking at our own
|
||||
# (untyped) code and causes tons of errors.
|
||||
Hashability: Any
|
||||
KeywordOnly: Any
|
||||
|
||||
is_exception: bool
|
||||
is_slotted: bool
|
||||
has_weakref_slot: bool
|
||||
is_frozen: bool
|
||||
# kw_only: ClassProps.KeywordOnly
|
||||
kw_only: Any
|
||||
collected_fields_by_mro: bool
|
||||
added_init: bool
|
||||
added_repr: bool
|
||||
added_eq: bool
|
||||
added_ordering: bool
|
||||
# hashability: ClassProps.Hashability
|
||||
hashability: Any
|
||||
added_match_args: bool
|
||||
added_str: bool
|
||||
added_pickling: bool
|
||||
on_setattr_hook: _OnSetAttrType | None
|
||||
field_transformer: Callable[[Attribute[Any]], Attribute[Any]] | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_exception: bool,
|
||||
is_slotted: bool,
|
||||
has_weakref_slot: bool,
|
||||
is_frozen: bool,
|
||||
# kw_only: ClassProps.KeywordOnly
|
||||
kw_only: Any,
|
||||
collected_fields_by_mro: bool,
|
||||
added_init: bool,
|
||||
added_repr: bool,
|
||||
added_eq: bool,
|
||||
added_ordering: bool,
|
||||
# hashability: ClassProps.Hashability
|
||||
hashability: Any,
|
||||
added_match_args: bool,
|
||||
added_str: bool,
|
||||
added_pickling: bool,
|
||||
on_setattr_hook: _OnSetAttrType,
|
||||
field_transformer: Callable[[Attribute[Any]], Attribute[Any]],
|
||||
) -> None: ...
|
||||
@property
|
||||
def is_hashable(self) -> bool: ...
|
||||
|
||||
def inspect(cls: type) -> ClassProps: ...
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr.converters import * # noqa: F403
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr.exceptions import * # noqa: F403
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr.filters import * # noqa: F403
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr.setters import * # noqa: F403
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from attr.validators import * # noqa: F403
|
||||
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,123 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: beautifulsoup4
|
||||
Version: 4.14.3
|
||||
Summary: Screen-scraping library
|
||||
Project-URL: Download, https://www.crummy.com/software/BeautifulSoup/bs4/download/
|
||||
Project-URL: Homepage, https://www.crummy.com/software/BeautifulSoup/bs4/
|
||||
Author-email: Leonard Richardson <leonardr@segfault.org>
|
||||
License: MIT License
|
||||
License-File: AUTHORS
|
||||
License-File: LICENSE
|
||||
Keywords: HTML,XML,parse,soup
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: MIT License
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||
Classifier: Topic :: Text Processing :: Markup :: HTML
|
||||
Classifier: Topic :: Text Processing :: Markup :: SGML
|
||||
Classifier: Topic :: Text Processing :: Markup :: XML
|
||||
Requires-Python: >=3.7.0
|
||||
Requires-Dist: soupsieve>=1.6.1
|
||||
Requires-Dist: typing-extensions>=4.0.0
|
||||
Provides-Extra: cchardet
|
||||
Requires-Dist: cchardet; extra == 'cchardet'
|
||||
Provides-Extra: chardet
|
||||
Requires-Dist: chardet; extra == 'chardet'
|
||||
Provides-Extra: charset-normalizer
|
||||
Requires-Dist: charset-normalizer; extra == 'charset-normalizer'
|
||||
Provides-Extra: html5lib
|
||||
Requires-Dist: html5lib; extra == 'html5lib'
|
||||
Provides-Extra: lxml
|
||||
Requires-Dist: lxml; extra == 'lxml'
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
Beautiful Soup is a library that makes it easy to scrape information
|
||||
from web pages. It sits atop an HTML or XML parser, providing Pythonic
|
||||
idioms for iterating, searching, and modifying the parse tree.
|
||||
|
||||
# Quick start
|
||||
|
||||
```
|
||||
>>> from bs4 import BeautifulSoup
|
||||
>>> soup = BeautifulSoup("<p>Some<b>bad<i>HTML")
|
||||
>>> print(soup.prettify())
|
||||
<html>
|
||||
<body>
|
||||
<p>
|
||||
Some
|
||||
<b>
|
||||
bad
|
||||
<i>
|
||||
HTML
|
||||
</i>
|
||||
</b>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
>>> soup.find(string="bad")
|
||||
'bad'
|
||||
>>> soup.i
|
||||
<i>HTML</i>
|
||||
#
|
||||
>>> soup = BeautifulSoup("<tag1>Some<tag2/>bad<tag3>XML", "xml")
|
||||
#
|
||||
>>> print(soup.prettify())
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<tag1>
|
||||
Some
|
||||
<tag2/>
|
||||
bad
|
||||
<tag3>
|
||||
XML
|
||||
</tag3>
|
||||
</tag1>
|
||||
```
|
||||
|
||||
To go beyond the basics, [comprehensive documentation is available](https://www.crummy.com/software/BeautifulSoup/bs4/doc/).
|
||||
|
||||
# Links
|
||||
|
||||
* [Homepage](https://www.crummy.com/software/BeautifulSoup/bs4/)
|
||||
* [Documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)
|
||||
* [Discussion group](https://groups.google.com/group/beautifulsoup/)
|
||||
* [Development](https://code.launchpad.net/beautifulsoup/)
|
||||
* [Bug tracker](https://bugs.launchpad.net/beautifulsoup/)
|
||||
* [Complete changelog](https://git.launchpad.net/beautifulsoup/tree/CHANGELOG)
|
||||
|
||||
# Note on Python 2 sunsetting
|
||||
|
||||
Beautiful Soup's support for Python 2 was discontinued on December 31,
|
||||
2020: one year after the sunset date for Python 2 itself. From this
|
||||
point onward, new Beautiful Soup development will exclusively target
|
||||
Python 3. The final release of Beautiful Soup 4 to support Python 2
|
||||
was 4.9.3.
|
||||
|
||||
# Supporting the project
|
||||
|
||||
If you use Beautiful Soup as part of your professional work, please consider a
|
||||
[Tidelift subscription](https://tidelift.com/subscription/pkg/pypi-beautifulsoup4?utm_source=pypi-beautifulsoup4&utm_medium=referral&utm_campaign=readme).
|
||||
This will support many of the free software projects your organization
|
||||
depends on, not just Beautiful Soup.
|
||||
|
||||
If you use Beautiful Soup for personal projects, the best way to say
|
||||
thank you is to read
|
||||
[Tool Safety](https://www.crummy.com/software/BeautifulSoup/zine/), a zine I
|
||||
wrote about what Beautiful Soup has taught me about software
|
||||
development.
|
||||
|
||||
# Building the documentation
|
||||
|
||||
The bs4/doc/ directory contains full documentation in Sphinx
|
||||
format. Run `make html` in that directory to create HTML
|
||||
documentation.
|
||||
|
||||
# Running the unit tests
|
||||
|
||||
Beautiful Soup supports unit test discovery using Pytest:
|
||||
|
||||
```
|
||||
$ pytest
|
||||
```
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
beautifulsoup4-4.14.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
beautifulsoup4-4.14.3.dist-info/METADATA,sha256=Ac93vA8Xp9FtgOcKXFM8ESfVdztimUfJ3WUpVlhKtsY,3812
|
||||
beautifulsoup4-4.14.3.dist-info/RECORD,,
|
||||
beautifulsoup4-4.14.3.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
beautifulsoup4-4.14.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
||||
beautifulsoup4-4.14.3.dist-info/licenses/AUTHORS,sha256=uYkjiRjh_aweRnF8tAW2PpJJeickE68NmJwd9siry28,2201
|
||||
beautifulsoup4-4.14.3.dist-info/licenses/LICENSE,sha256=VbTY1LHlvIbRDvrJG3TIe8t3UmsPW57a-LnNKtxzl7I,1441
|
||||
bs4/__init__.py,sha256=E7wiVp7oQK0JhdAYxpehZa8drv3W_sJv5oeTFiBfR5o,44386
|
||||
bs4/__pycache__/__init__.cpython-312.pyc,,
|
||||
bs4/__pycache__/_deprecation.cpython-312.pyc,,
|
||||
bs4/__pycache__/_typing.cpython-312.pyc,,
|
||||
bs4/__pycache__/_warnings.cpython-312.pyc,,
|
||||
bs4/__pycache__/css.cpython-312.pyc,,
|
||||
bs4/__pycache__/dammit.cpython-312.pyc,,
|
||||
bs4/__pycache__/diagnose.cpython-312.pyc,,
|
||||
bs4/__pycache__/element.cpython-312.pyc,,
|
||||
bs4/__pycache__/exceptions.cpython-312.pyc,,
|
||||
bs4/__pycache__/filter.cpython-312.pyc,,
|
||||
bs4/__pycache__/formatter.cpython-312.pyc,,
|
||||
bs4/_deprecation.py,sha256=niHJCk37APg8KEuFOa57ZXaxLdBmc_2V6uuaJqu7r30,2408
|
||||
bs4/_typing.py,sha256=zNcx7R1yCTK8WwtumP28hc7CJ3pMyZXj_VAeYaNXMZA,7549
|
||||
bs4/_warnings.py,sha256=ZuOETgcnEbZgw2N0nnNXn6wvtrn2ut7AF0d98bvkMFc,4711
|
||||
bs4/builder/__init__.py,sha256=Rl4qjOXvdyyyjayOFqbkgoUoo81IgoyKD-RwWeVK59g,31194
|
||||
bs4/builder/__pycache__/__init__.cpython-312.pyc,,
|
||||
bs4/builder/__pycache__/_html5lib.cpython-312.pyc,,
|
||||
bs4/builder/__pycache__/_htmlparser.cpython-312.pyc,,
|
||||
bs4/builder/__pycache__/_lxml.cpython-312.pyc,,
|
||||
bs4/builder/_html5lib.py,sha256=hL6xUk4_I2i5CMguFoYFlrI26cY4Dut7fOEQrUctHIM,23607
|
||||
bs4/builder/_htmlparser.py,sha256=CnULPQV2rm4vLojJABpQ7Xm9diddnEZx2Wcz_VTC1Mg,17445
|
||||
bs4/builder/_lxml.py,sha256=ks1e8boA_nOA2oomAhxeudccR6ThbEE-EllFqHRoPLA,18969
|
||||
bs4/css.py,sha256=_m_l_4SGWHnY620VJ21j_qQH1RX3p91sYVemgKxaLsM,12713
|
||||
bs4/dammit.py,sha256=ZJWa9K32X6N2imFHleqUq0ekf592weU1lvULN_WYWYk,57024
|
||||
bs4/diagnose.py,sha256=at98iuxyOrqec4V8iwkTIbNUqBCsq9Lr3fDAQx2129Y,7846
|
||||
bs4/element.py,sha256=oXmj7LG_2NpsDK90mq73q0PMK0FjFBIGSeTTJLVwwTc,120237
|
||||
bs4/exceptions.py,sha256=Q9FOadNe8QRvzDMaKSXe2Wtl8JK_oAZW7mbFZBVP_GE,951
|
||||
bs4/filter.py,sha256=rw8ZNhTDLEJVCEiSifou5tZR_3zBLeuvAyouY82qU_E,29201
|
||||
bs4/formatter.py,sha256=uBT0k6W8O5kJ9PCuJYjra97yoUqC-dlM9D_v-oRM0r8,10478
|
||||
bs4/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
@@ -0,0 +1,4 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: hatchling 1.27.0
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
@@ -0,0 +1,49 @@
|
||||
Behold, mortal, the origins of Beautiful Soup...
|
||||
================================================
|
||||
|
||||
Leonard Richardson is the primary maintainer.
|
||||
|
||||
Aaron DeVore, Isaac Muse and Chris Papademetrious have made
|
||||
significant contributions to the code base.
|
||||
|
||||
Mark Pilgrim provided the encoding detection code that forms the base
|
||||
of UnicodeDammit.
|
||||
|
||||
Thomas Kluyver and Ezio Melotti finished the work of getting Beautiful
|
||||
Soup 4 working under Python 3.
|
||||
|
||||
Simon Willison wrote soupselect, which was used to make Beautiful Soup
|
||||
support CSS selectors. Isaac Muse wrote SoupSieve, which made it
|
||||
possible to _remove_ the CSS selector code from Beautiful Soup.
|
||||
|
||||
Sam Ruby helped with a lot of edge cases.
|
||||
|
||||
Jonathan Ellis was awarded the prestigious Beau Potage D'Or for his
|
||||
work in solving the nestable tags conundrum.
|
||||
|
||||
An incomplete list of people have contributed patches to Beautiful
|
||||
Soup:
|
||||
|
||||
Istvan Albert, Andrew Lin, Anthony Baxter, Oliver Beattie, Andrew
|
||||
Boyko, Tony Chang, Francisco Canas, "Delong", Zephyr Fang, Fuzzy,
|
||||
Roman Gaufman, Yoni Gilad, Richie Hindle, Toshihiro Kamiya, Peteris
|
||||
Krumins, Kent Johnson, Marek Kapolka, Andreas Kostyrka, Roel Kramer,
|
||||
Ben Last, Robert Leftwich, Stefaan Lippens, "liquider", Staffan
|
||||
Malmgren, Ksenia Marasanova, JP Moins, Adam Monsen, John Nagle, "Jon",
|
||||
Ed Oskiewicz, Martijn Peters, Greg Phillips, Giles Radford, Stefano
|
||||
Revera, Arthur Rudolph, Marko Samastur, James Salter, Jouni Seppänen,
|
||||
Alexander Schmolck, Tim Shirley, Geoffrey Sneddon, Ville Skyttä,
|
||||
"Vikas", Jens Svalgaard, Andy Theyers, Eric Weiser, Glyn Webster, John
|
||||
Wiseman, Paul Wright, Danny Yoo
|
||||
|
||||
An incomplete list of people who made suggestions or found bugs or
|
||||
found ways to break Beautiful Soup:
|
||||
|
||||
Hanno Böck, Matteo Bertini, Chris Curvey, Simon Cusack, Bruce Eckel,
|
||||
Matt Ernst, Michael Foord, Tom Harris, Bill de hOra, Donald Howes,
|
||||
Matt Patterson, Scott Roberts, Steve Strassmann, Mike Williams,
|
||||
warchild at redho dot com, Sami Kuisma, Carlos Rocha, Bob Hutchison,
|
||||
Joren Mc, Michal Migurski, John Kleven, Tim Heaney, Tripp Lilley, Ed
|
||||
Summers, Dennis Sutch, Chris Smith, Aaron Swartz, Stuart
|
||||
Turner, Greg Edwards, Kevin J Kalupson, Nikos Kouremenos, Artur de
|
||||
Sousa Rocha, Yichun Wei, Per Vognsen
|
||||
@@ -0,0 +1,31 @@
|
||||
Beautiful Soup is made available under the MIT license:
|
||||
|
||||
Copyright (c) Leonard Richardson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
||||
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
Beautiful Soup incorporates code from the html5lib library, which is
|
||||
also made available under the MIT license. Copyright (c) James Graham
|
||||
and other contributors
|
||||
|
||||
Beautiful Soup has an optional dependency on the soupsieve library,
|
||||
which is also made available under the MIT license. Copyright (c)
|
||||
Isaac Muse
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,80 @@
|
||||
"""Helper functions for deprecation.
|
||||
|
||||
This interface is itself unstable and may change without warning. Do
|
||||
not use these functions yourself, even as a joke. The underscores are
|
||||
there for a reason. No support will be given.
|
||||
|
||||
In particular, most of this will go away without warning once
|
||||
Beautiful Soup drops support for Python 3.11, since Python 3.12
|
||||
defines a `@typing.deprecated()
|
||||
decorator. <https://peps.python.org/pep-0702/>`_
|
||||
"""
|
||||
|
||||
import functools
|
||||
import warnings
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
)
|
||||
|
||||
|
||||
def _deprecated_alias(old_name: str, new_name: str, version: str):
|
||||
"""Alias one attribute name to another for backward compatibility
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
|
||||
@property # type:ignore
|
||||
def alias(self) -> Any:
|
||||
":meta private:"
|
||||
warnings.warn(
|
||||
f"Access to deprecated property {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return getattr(self, new_name)
|
||||
|
||||
@alias.setter
|
||||
def alias(self, value: str) -> None:
|
||||
":meta private:"
|
||||
warnings.warn(
|
||||
f"Write to deprecated property {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return setattr(self, new_name, value)
|
||||
|
||||
return alias
|
||||
|
||||
|
||||
def _deprecated_function_alias(
|
||||
old_name: str, new_name: str, version: str
|
||||
) -> Callable[[Any], Any]:
|
||||
def alias(self, *args: Any, **kwargs: Any) -> Any:
|
||||
":meta private:"
|
||||
warnings.warn(
|
||||
f"Call to deprecated method {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return getattr(self, new_name)(*args, **kwargs)
|
||||
|
||||
return alias
|
||||
|
||||
|
||||
def _deprecated(replaced_by: str, version: str) -> Callable:
|
||||
def deprecate(func: Callable) -> Callable:
|
||||
@functools.wraps(func)
|
||||
def with_warning(*args: Any, **kwargs: Any) -> Any:
|
||||
":meta private:"
|
||||
warnings.warn(
|
||||
f"Call to deprecated method {func.__name__}. (Replaced by {replaced_by}) -- Deprecated since version {version}.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return with_warning
|
||||
|
||||
return deprecate
|
||||
@@ -0,0 +1,205 @@
|
||||
# Custom type aliases used throughout Beautiful Soup to improve readability.
|
||||
|
||||
# Notes on improvements to the type system in newer versions of Python
|
||||
# that can be used once Beautiful Soup drops support for older
|
||||
# versions:
|
||||
#
|
||||
# * ClassVar can be put on class variables now.
|
||||
# * In 3.10, x|y is an accepted shorthand for Union[x,y].
|
||||
# * In 3.10, TypeAlias gains capabilities that can be used to
|
||||
# improve the tree matching types (I don't remember what, exactly).
|
||||
# * In 3.9 it's possible to specialize the re.Match type,
|
||||
# e.g. re.Match[str]. In 3.8 there's a typing.re namespace for this,
|
||||
# but it's removed in 3.12, so to support the widest possible set of
|
||||
# versions I'm not using it.
|
||||
|
||||
from typing_extensions import (
|
||||
runtime_checkable,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
IO,
|
||||
Iterable,
|
||||
Mapping,
|
||||
Optional,
|
||||
Pattern,
|
||||
TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4.element import (
|
||||
AttributeValueList,
|
||||
NamespacedAttribute,
|
||||
NavigableString,
|
||||
PageElement,
|
||||
ResultSet,
|
||||
Tag,
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _RegularExpressionProtocol(Protocol):
|
||||
"""A protocol object which can accept either Python's built-in
|
||||
`re.Pattern` objects, or the similar ``Regex`` objects defined by the
|
||||
third-party ``regex`` package.
|
||||
"""
|
||||
|
||||
def search(
|
||||
self, string: str, pos: int = ..., endpos: int = ...
|
||||
) -> Optional[Any]: ...
|
||||
|
||||
@property
|
||||
def pattern(self) -> str: ...
|
||||
|
||||
|
||||
# Aliases for markup in various stages of processing.
|
||||
#
|
||||
#: The rawest form of markup: either a string, bytestring, or an open filehandle.
|
||||
_IncomingMarkup: TypeAlias = Union[str, bytes, IO[str], IO[bytes]]
|
||||
|
||||
#: Markup that is in memory but has (potentially) yet to be converted
|
||||
#: to Unicode.
|
||||
_RawMarkup: TypeAlias = Union[str, bytes]
|
||||
|
||||
# Aliases for character encodings
|
||||
#
|
||||
|
||||
#: A data encoding.
|
||||
_Encoding: TypeAlias = str
|
||||
|
||||
#: One or more data encodings.
|
||||
_Encodings: TypeAlias = Iterable[_Encoding]
|
||||
|
||||
# Aliases for XML namespaces
|
||||
#
|
||||
|
||||
#: The prefix for an XML namespace.
|
||||
_NamespacePrefix: TypeAlias = str
|
||||
|
||||
#: The URL of an XML namespace
|
||||
_NamespaceURL: TypeAlias = str
|
||||
|
||||
#: A mapping of prefixes to namespace URLs.
|
||||
_NamespaceMapping: TypeAlias = Dict[_NamespacePrefix, _NamespaceURL]
|
||||
|
||||
#: A mapping of namespace URLs to prefixes
|
||||
_InvertedNamespaceMapping: TypeAlias = Dict[_NamespaceURL, _NamespacePrefix]
|
||||
|
||||
# Aliases for the attribute values associated with HTML/XML tags.
|
||||
#
|
||||
|
||||
#: The value associated with an HTML or XML attribute. This is the
|
||||
#: relatively unprocessed value Beautiful Soup expects to come from a
|
||||
#: `TreeBuilder`.
|
||||
_RawAttributeValue: TypeAlias = str
|
||||
|
||||
#: A dictionary of names to `_RawAttributeValue` objects. This is how
|
||||
#: Beautiful Soup expects a `TreeBuilder` to represent a tag's
|
||||
#: attribute values.
|
||||
_RawAttributeValues: TypeAlias = (
|
||||
"Mapping[Union[str, NamespacedAttribute], _RawAttributeValue]"
|
||||
)
|
||||
|
||||
#: An attribute value in its final form, as stored in the
|
||||
# `Tag` class, after it has been processed and (in some cases)
|
||||
# split into a list of strings.
|
||||
_AttributeValue: TypeAlias = Union[str, "AttributeValueList"]
|
||||
|
||||
#: A dictionary of names to :py:data:`_AttributeValue` objects. This is what
|
||||
#: a tag's attributes look like after processing.
|
||||
_AttributeValues: TypeAlias = Dict[str, _AttributeValue]
|
||||
|
||||
#: The methods that deal with turning :py:data:`_RawAttributeValue` into
|
||||
#: :py:data:`_AttributeValue` may be called several times, even after the values
|
||||
#: are already processed (e.g. when cloning a tag), so they need to
|
||||
#: be able to acommodate both possibilities.
|
||||
_RawOrProcessedAttributeValues: TypeAlias = Union[_RawAttributeValues, _AttributeValues]
|
||||
|
||||
#: A number of tree manipulation methods can take either a `PageElement` or a
|
||||
#: normal Python string (which will be converted to a `NavigableString`).
|
||||
_InsertableElement: TypeAlias = Union["PageElement", str]
|
||||
|
||||
# Aliases to represent the many possibilities for matching bits of a
|
||||
# parse tree.
|
||||
#
|
||||
# This is very complicated because we're applying a formal type system
|
||||
# to some very DWIM code. The types we end up with will be the types
|
||||
# of the arguments to the SoupStrainer constructor and (more
|
||||
# familiarly to Beautiful Soup users) the find* methods.
|
||||
|
||||
#: A function that takes a PageElement and returns a yes-or-no answer.
|
||||
_PageElementMatchFunction: TypeAlias = Callable[["PageElement"], bool]
|
||||
|
||||
#: A function that takes the raw parsed ingredients of a markup tag
|
||||
#: and returns a yes-or-no answer.
|
||||
# Not necessary at the moment.
|
||||
# _AllowTagCreationFunction:TypeAlias = Callable[[Optional[str], str, Optional[_RawAttributeValues]], bool]
|
||||
|
||||
#: A function that takes the raw parsed ingredients of a markup string node
|
||||
#: and returns a yes-or-no answer.
|
||||
# Not necessary at the moment.
|
||||
# _AllowStringCreationFunction:TypeAlias = Callable[[Optional[str]], bool]
|
||||
|
||||
#: A function that takes a `Tag` and returns a yes-or-no answer.
|
||||
#: A `TagNameMatchRule` expects this kind of function, if you're
|
||||
#: going to pass it a function.
|
||||
_TagMatchFunction: TypeAlias = Callable[["Tag"], bool]
|
||||
|
||||
#: A function that takes a string (or None) and returns a yes-or-no
|
||||
#: answer. An `AttributeValueMatchRule` expects this kind of function, if
|
||||
#: you're going to pass it a function.
|
||||
_NullableStringMatchFunction: TypeAlias = Callable[[Optional[str]], bool]
|
||||
|
||||
#: A function that takes a string and returns a yes-or-no answer. A
|
||||
# `StringMatchRule` expects this kind of function, if you're going to
|
||||
# pass it a function.
|
||||
_StringMatchFunction: TypeAlias = Callable[[str], bool]
|
||||
|
||||
#: Either a tag name, an attribute value or a string can be matched
|
||||
#: against a string, bytestring, regular expression, or a boolean.
|
||||
_BaseStrainable: TypeAlias = Union[str, bytes, Pattern[str], bool]
|
||||
|
||||
#: A tag can be matched either with the `_BaseStrainable` options, or
|
||||
#: using a function that takes the `Tag` as its sole argument.
|
||||
_BaseStrainableElement: TypeAlias = Union[_BaseStrainable, _TagMatchFunction]
|
||||
|
||||
#: A tag's attribute value can be matched either with the
|
||||
#: `_BaseStrainable` options, or using a function that takes that
|
||||
#: value as its sole argument.
|
||||
_BaseStrainableAttribute: TypeAlias = Union[_BaseStrainable, _NullableStringMatchFunction]
|
||||
|
||||
#: A tag can be matched using either a single criterion or a list of
|
||||
#: criteria.
|
||||
_StrainableElement: TypeAlias = Union[
|
||||
_BaseStrainableElement, Iterable[_BaseStrainableElement]
|
||||
]
|
||||
|
||||
#: An attribute value can be matched using either a single criterion
|
||||
#: or a list of criteria.
|
||||
_StrainableAttribute: TypeAlias = Union[
|
||||
_BaseStrainableAttribute, Iterable[_BaseStrainableAttribute]
|
||||
]
|
||||
|
||||
#: An string can be matched using the same techniques as
|
||||
#: an attribute value.
|
||||
_StrainableString: TypeAlias = _StrainableAttribute
|
||||
|
||||
#: A dictionary may be used to match against multiple attribute vlaues at once.
|
||||
_StrainableAttributes: TypeAlias = Dict[str, _StrainableAttribute]
|
||||
|
||||
#: Many Beautiful soup methods return a PageElement or an ResultSet of
|
||||
#: PageElements. A PageElement is either a Tag or a NavigableString.
|
||||
#: These convenience aliases make it easier for IDE users to see which methods
|
||||
#: are available on the objects they're dealing with.
|
||||
_OneElement: TypeAlias = Union["PageElement", "Tag", "NavigableString"]
|
||||
_AtMostOneElement: TypeAlias = Optional[_OneElement]
|
||||
_AtMostOneTag: TypeAlias = Optional["Tag"]
|
||||
_AtMostOneNavigableString: TypeAlias = Optional["NavigableString"]
|
||||
_QueryResults: TypeAlias = "ResultSet[_OneElement]"
|
||||
_SomeTags: TypeAlias = "ResultSet[Tag]"
|
||||
_SomeNavigableStrings: TypeAlias = "ResultSet[NavigableString]"
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Define some custom warnings."""
|
||||
|
||||
|
||||
class GuessedAtParserWarning(UserWarning):
|
||||
"""The warning issued when BeautifulSoup has to guess what parser to
|
||||
use -- probably because no parser was specified in the constructor.
|
||||
"""
|
||||
|
||||
MESSAGE: str = """No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system ("%(parser)s"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.
|
||||
|
||||
The code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features="%(parser)s"' to the BeautifulSoup constructor.
|
||||
"""
|
||||
|
||||
|
||||
class UnusualUsageWarning(UserWarning):
|
||||
"""A superclass for warnings issued when Beautiful Soup sees
|
||||
something that is typically the result of a mistake in the calling
|
||||
code, but might be intentional on the part of the user. If it is
|
||||
in fact intentional, you can filter the individual warning class
|
||||
to get rid of the warning. If you don't like Beautiful Soup
|
||||
second-guessing what you are doing, you can filter the
|
||||
UnusualUsageWarningclass itself and get rid of these entirely.
|
||||
"""
|
||||
|
||||
|
||||
class MarkupResemblesLocatorWarning(UnusualUsageWarning):
|
||||
"""The warning issued when BeautifulSoup is given 'markup' that
|
||||
actually looks like a resource locator -- a URL or a path to a file
|
||||
on disk.
|
||||
"""
|
||||
|
||||
#: :meta private:
|
||||
GENERIC_MESSAGE: str = """
|
||||
|
||||
However, if you want to parse some data that happens to look like a %(what)s, then nothing has gone wrong: you are using Beautiful Soup correctly, and this warning is spurious and can be filtered. To make this warning go away, run this code before calling the BeautifulSoup constructor:
|
||||
|
||||
from bs4 import MarkupResemblesLocatorWarning
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
|
||||
"""
|
||||
|
||||
URL_MESSAGE: str = (
|
||||
"""The input passed in on this line looks more like a URL than HTML or XML.
|
||||
|
||||
If you meant to use Beautiful Soup to parse the web page found at a certain URL, then something has gone wrong. You should use an Python package like 'requests' to fetch the content behind the URL. Once you have the content as a string, you can feed that string into Beautiful Soup."""
|
||||
+ GENERIC_MESSAGE
|
||||
)
|
||||
|
||||
FILENAME_MESSAGE: str = (
|
||||
"""The input passed in on this line looks more like a filename than HTML or XML.
|
||||
|
||||
If you meant to use Beautiful Soup to parse the contents of a file on disk, then something has gone wrong. You should open the file first, using code like this:
|
||||
|
||||
filehandle = open(your filename)
|
||||
|
||||
You can then feed the open filehandle into Beautiful Soup instead of using the filename."""
|
||||
+ GENERIC_MESSAGE
|
||||
)
|
||||
|
||||
|
||||
class AttributeResemblesVariableWarning(UnusualUsageWarning, SyntaxWarning):
|
||||
"""The warning issued when Beautiful Soup suspects a provided
|
||||
attribute name may actually be the misspelled name of a Beautiful
|
||||
Soup variable. Generally speaking, this is only used in cases like
|
||||
"_class" where it's very unlikely the user would be referencing an
|
||||
XML attribute with that name.
|
||||
"""
|
||||
|
||||
MESSAGE: str = """%(original)r is an unusual attribute name and is a common misspelling for %(autocorrect)r.
|
||||
|
||||
If you meant %(autocorrect)r, change your code to use it, and this warning will go away.
|
||||
|
||||
If you really did mean to check the %(original)r attribute, this warning is spurious and can be filtered. To make it go away, run this code before creating your BeautifulSoup object:
|
||||
|
||||
from bs4 import AttributeResemblesVariableWarning
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", category=AttributeResemblesVariableWarning)
|
||||
"""
|
||||
|
||||
|
||||
class XMLParsedAsHTMLWarning(UnusualUsageWarning):
|
||||
"""The warning issued when an HTML parser is used to parse
|
||||
XML that is not (as far as we can tell) XHTML.
|
||||
"""
|
||||
|
||||
MESSAGE: str = """It looks like you're using an HTML parser to parse an XML document.
|
||||
|
||||
Assuming this really is an XML document, what you're doing might work, but you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the Python package 'lxml' installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor.
|
||||
|
||||
If you want or need to use an HTML parser on this document, you can make this warning go away by filtering it. To do that, run this code before calling the BeautifulSoup constructor:
|
||||
|
||||
from bs4 import XMLParsedAsHTMLWarning
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
|
||||
"""
|
||||
@@ -0,0 +1,848 @@
|
||||
from __future__ import annotations
|
||||
|
||||
# Use of this source code is governed by the MIT license.
|
||||
__license__ = "MIT"
|
||||
|
||||
from collections import defaultdict
|
||||
import re
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
cast,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Pattern,
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
import warnings
|
||||
import sys
|
||||
from bs4.element import (
|
||||
AttributeDict,
|
||||
AttributeValueList,
|
||||
CharsetMetaAttributeValue,
|
||||
ContentMetaAttributeValue,
|
||||
RubyParenthesisString,
|
||||
RubyTextString,
|
||||
Stylesheet,
|
||||
Script,
|
||||
TemplateString,
|
||||
nonwhitespace_re,
|
||||
)
|
||||
|
||||
# Exceptions were moved to their own module in 4.13. Import here for
|
||||
# backwards compatibility.
|
||||
from bs4.exceptions import ParserRejectedMarkup
|
||||
|
||||
from bs4._typing import (
|
||||
_AttributeValues,
|
||||
_RawAttributeValue,
|
||||
)
|
||||
|
||||
from bs4._warnings import XMLParsedAsHTMLWarning
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import (
|
||||
NavigableString,
|
||||
Tag,
|
||||
)
|
||||
from bs4._typing import (
|
||||
_AttributeValue,
|
||||
_Encoding,
|
||||
_Encodings,
|
||||
_RawOrProcessedAttributeValues,
|
||||
_RawMarkup,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"HTMLTreeBuilder",
|
||||
"SAXTreeBuilder",
|
||||
"TreeBuilder",
|
||||
"TreeBuilderRegistry",
|
||||
]
|
||||
|
||||
# Some useful features for a TreeBuilder to have.
|
||||
FAST = "fast"
|
||||
PERMISSIVE = "permissive"
|
||||
STRICT = "strict"
|
||||
XML = "xml"
|
||||
HTML = "html"
|
||||
HTML_5 = "html5"
|
||||
|
||||
__all__ = [
|
||||
"TreeBuilderRegistry",
|
||||
"TreeBuilder",
|
||||
"HTMLTreeBuilder",
|
||||
"DetectsXMLParsedAsHTML",
|
||||
|
||||
"ParserRejectedMarkup", # backwards compatibility only as of 4.13.0
|
||||
]
|
||||
|
||||
class TreeBuilderRegistry(object):
|
||||
"""A way of looking up TreeBuilder subclasses by their name or by desired
|
||||
features.
|
||||
"""
|
||||
|
||||
builders_for_feature: Dict[str, List[Type[TreeBuilder]]]
|
||||
builders: List[Type[TreeBuilder]]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.builders_for_feature = defaultdict(list)
|
||||
self.builders = []
|
||||
|
||||
def register(self, treebuilder_class: type[TreeBuilder]) -> None:
|
||||
"""Register a treebuilder based on its advertised features.
|
||||
|
||||
:param treebuilder_class: A subclass of `TreeBuilder`. its
|
||||
`TreeBuilder.features` attribute should list its features.
|
||||
"""
|
||||
for feature in treebuilder_class.features:
|
||||
self.builders_for_feature[feature].insert(0, treebuilder_class)
|
||||
self.builders.insert(0, treebuilder_class)
|
||||
|
||||
def lookup(self, *features: str) -> Optional[Type[TreeBuilder]]:
|
||||
"""Look up a TreeBuilder subclass with the desired features.
|
||||
|
||||
:param features: A list of features to look for. If none are
|
||||
provided, the most recently registered TreeBuilder subclass
|
||||
will be used.
|
||||
:return: A TreeBuilder subclass, or None if there's no
|
||||
registered subclass with all the requested features.
|
||||
"""
|
||||
if len(self.builders) == 0:
|
||||
# There are no builders at all.
|
||||
return None
|
||||
|
||||
if len(features) == 0:
|
||||
# They didn't ask for any features. Give them the most
|
||||
# recently registered builder.
|
||||
return self.builders[0]
|
||||
|
||||
# Go down the list of features in order, and eliminate any builders
|
||||
# that don't match every feature.
|
||||
feature_list = list(features)
|
||||
feature_list.reverse()
|
||||
candidates = None
|
||||
candidate_set = None
|
||||
while len(feature_list) > 0:
|
||||
feature = feature_list.pop()
|
||||
we_have_the_feature = self.builders_for_feature.get(feature, [])
|
||||
if len(we_have_the_feature) > 0:
|
||||
if candidates is None:
|
||||
candidates = we_have_the_feature
|
||||
candidate_set = set(candidates)
|
||||
elif candidate_set is not None:
|
||||
# Eliminate any candidates that don't have this feature.
|
||||
candidate_set = candidate_set.intersection(set(we_have_the_feature))
|
||||
|
||||
# The only valid candidates are the ones in candidate_set.
|
||||
# Go through the original list of candidates and pick the first one
|
||||
# that's in candidate_set.
|
||||
if candidate_set is None or candidates is None:
|
||||
return None
|
||||
for candidate in candidates:
|
||||
if candidate in candidate_set:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
#: The `BeautifulSoup` constructor will take a list of features
|
||||
#: and use it to look up `TreeBuilder` classes in this registry.
|
||||
builder_registry: TreeBuilderRegistry = TreeBuilderRegistry()
|
||||
|
||||
|
||||
class TreeBuilder(object):
|
||||
"""Turn a textual document into a Beautiful Soup object tree.
|
||||
|
||||
This is an abstract superclass which smooths out the behavior of
|
||||
different parser libraries into a single, unified interface.
|
||||
|
||||
:param multi_valued_attributes: If this is set to None, the
|
||||
TreeBuilder will not turn any values for attributes like
|
||||
'class' into lists. Setting this to a dictionary will
|
||||
customize this behavior; look at :py:attr:`bs4.builder.HTMLTreeBuilder.DEFAULT_CDATA_LIST_ATTRIBUTES`
|
||||
for an example.
|
||||
|
||||
Internally, these are called "CDATA list attributes", but that
|
||||
probably doesn't make sense to an end-user, so the argument name
|
||||
is ``multi_valued_attributes``.
|
||||
|
||||
:param preserve_whitespace_tags: A set of tags to treat
|
||||
the way <pre> tags are treated in HTML. Tags in this set
|
||||
are immune from pretty-printing; their contents will always be
|
||||
output as-is.
|
||||
|
||||
:param string_containers: A dictionary mapping tag names to
|
||||
the classes that should be instantiated to contain the textual
|
||||
contents of those tags. The default is to use NavigableString
|
||||
for every tag, no matter what the name. You can override the
|
||||
default by changing :py:attr:`DEFAULT_STRING_CONTAINERS`.
|
||||
|
||||
:param store_line_numbers: If the parser keeps track of the line
|
||||
numbers and positions of the original markup, that information
|
||||
will, by default, be stored in each corresponding
|
||||
:py:class:`bs4.element.Tag` object. You can turn this off by
|
||||
passing store_line_numbers=False; then Tag.sourcepos and
|
||||
Tag.sourceline will always be None. If the parser you're using
|
||||
doesn't keep track of this information, then store_line_numbers
|
||||
is irrelevant.
|
||||
|
||||
:param attribute_dict_class: The value of a multi-valued attribute
|
||||
(such as HTML's 'class') willl be stored in an instance of this
|
||||
class. The default is Beautiful Soup's built-in
|
||||
`AttributeValueList`, which is a normal Python list, and you
|
||||
will probably never need to change it.
|
||||
"""
|
||||
|
||||
USE_DEFAULT: Any = object() #: :meta private:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
multi_valued_attributes: Dict[str, Set[str]] = USE_DEFAULT,
|
||||
preserve_whitespace_tags: Set[str] = USE_DEFAULT,
|
||||
store_line_numbers: bool = USE_DEFAULT,
|
||||
string_containers: Dict[str, Type[NavigableString]] = USE_DEFAULT,
|
||||
empty_element_tags: Set[str] = USE_DEFAULT,
|
||||
attribute_dict_class: Type[AttributeDict] = AttributeDict,
|
||||
attribute_value_list_class: Type[AttributeValueList] = AttributeValueList,
|
||||
):
|
||||
self.soup = None
|
||||
if multi_valued_attributes is self.USE_DEFAULT:
|
||||
multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
|
||||
self.cdata_list_attributes = multi_valued_attributes
|
||||
if preserve_whitespace_tags is self.USE_DEFAULT:
|
||||
preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
|
||||
self.preserve_whitespace_tags = preserve_whitespace_tags
|
||||
if empty_element_tags is self.USE_DEFAULT:
|
||||
self.empty_element_tags = self.DEFAULT_EMPTY_ELEMENT_TAGS
|
||||
else:
|
||||
self.empty_element_tags = empty_element_tags
|
||||
# TODO: store_line_numbers is probably irrelevant now that
|
||||
# the behavior of sourceline and sourcepos has been made consistent
|
||||
# everywhere.
|
||||
if store_line_numbers == self.USE_DEFAULT:
|
||||
store_line_numbers = self.TRACKS_LINE_NUMBERS
|
||||
self.store_line_numbers = store_line_numbers
|
||||
if string_containers == self.USE_DEFAULT:
|
||||
string_containers = self.DEFAULT_STRING_CONTAINERS
|
||||
self.string_containers = string_containers
|
||||
self.attribute_dict_class = attribute_dict_class
|
||||
self.attribute_value_list_class = attribute_value_list_class
|
||||
|
||||
NAME: str = "[Unknown tree builder]"
|
||||
ALTERNATE_NAMES: Iterable[str] = []
|
||||
features: Iterable[str] = []
|
||||
|
||||
is_xml: bool = False
|
||||
picklable: bool = False
|
||||
|
||||
soup: Optional[BeautifulSoup] #: :meta private:
|
||||
|
||||
#: A tag will be considered an empty-element
|
||||
#: tag when and only when it has no contents.
|
||||
empty_element_tags: Optional[Set[str]] = None #: :meta private:
|
||||
cdata_list_attributes: Dict[str, Set[str]] #: :meta private:
|
||||
preserve_whitespace_tags: Set[str] #: :meta private:
|
||||
string_containers: Dict[str, Type[NavigableString]] #: :meta private:
|
||||
tracks_line_numbers: bool #: :meta private:
|
||||
|
||||
#: A value for these tag/attribute combinations is a space- or
|
||||
#: comma-separated list of CDATA, rather than a single CDATA.
|
||||
DEFAULT_CDATA_LIST_ATTRIBUTES: Dict[str, Set[str]] = defaultdict(set)
|
||||
|
||||
#: Whitespace should be preserved inside these tags.
|
||||
DEFAULT_PRESERVE_WHITESPACE_TAGS: Set[str] = set()
|
||||
|
||||
#: The textual contents of tags with these names should be
|
||||
#: instantiated with some class other than `bs4.element.NavigableString`.
|
||||
DEFAULT_STRING_CONTAINERS: Dict[str, Type[bs4.element.NavigableString]] = {} # type:ignore
|
||||
|
||||
#: By default, tags are treated as empty-element tags if they have
|
||||
#: no contents--that is, using XML rules. HTMLTreeBuilder
|
||||
#: defines a different set of DEFAULT_EMPTY_ELEMENT_TAGS based on the
|
||||
#: HTML 4 and HTML5 standards.
|
||||
DEFAULT_EMPTY_ELEMENT_TAGS: Optional[Set[str]] = None
|
||||
|
||||
#: Most parsers don't keep track of line numbers.
|
||||
TRACKS_LINE_NUMBERS: bool = False
|
||||
|
||||
def initialize_soup(self, soup: BeautifulSoup) -> None:
|
||||
"""The BeautifulSoup object has been initialized and is now
|
||||
being associated with the TreeBuilder.
|
||||
|
||||
:param soup: A BeautifulSoup object.
|
||||
"""
|
||||
self.soup = soup
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Do any work necessary to reset the underlying parser
|
||||
for a new document.
|
||||
|
||||
By default, this does nothing.
|
||||
"""
|
||||
pass
|
||||
|
||||
def can_be_empty_element(self, tag_name: str) -> bool:
|
||||
"""Might a tag with this name be an empty-element tag?
|
||||
|
||||
The final markup may or may not actually present this tag as
|
||||
self-closing.
|
||||
|
||||
For instance: an HTMLBuilder does not consider a <p> tag to be
|
||||
an empty-element tag (it's not in
|
||||
HTMLBuilder.empty_element_tags). This means an empty <p> tag
|
||||
will be presented as "<p></p>", not "<p/>" or "<p>".
|
||||
|
||||
The default implementation has no opinion about which tags are
|
||||
empty-element tags, so a tag will be presented as an
|
||||
empty-element tag if and only if it has no children.
|
||||
"<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
|
||||
be left alone.
|
||||
|
||||
:param tag_name: The name of a markup tag.
|
||||
"""
|
||||
if self.empty_element_tags is None:
|
||||
return True
|
||||
return tag_name in self.empty_element_tags
|
||||
|
||||
def feed(self, markup: _RawMarkup) -> None:
|
||||
"""Run incoming markup through some parsing process."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def prepare_markup(
|
||||
self,
|
||||
markup: _RawMarkup,
|
||||
user_specified_encoding: Optional[_Encoding] = None,
|
||||
document_declared_encoding: Optional[_Encoding] = None,
|
||||
exclude_encodings: Optional[_Encodings] = None,
|
||||
) -> Iterable[Tuple[_RawMarkup, Optional[_Encoding], Optional[_Encoding], bool]]:
|
||||
"""Run any preliminary steps necessary to make incoming markup
|
||||
acceptable to the parser.
|
||||
|
||||
:param markup: The markup that's about to be parsed.
|
||||
:param user_specified_encoding: The user asked to try this encoding
|
||||
to convert the markup into a Unicode string.
|
||||
:param document_declared_encoding: The markup itself claims to be
|
||||
in this encoding. NOTE: This argument is not used by the
|
||||
calling code and can probably be removed.
|
||||
:param exclude_encodings: The user asked *not* to try any of
|
||||
these encodings.
|
||||
|
||||
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
|
||||
has undergone character replacement)
|
||||
|
||||
Each 4-tuple represents a strategy that the parser can try
|
||||
to convert the document to Unicode and parse it. Each
|
||||
strategy will be tried in turn.
|
||||
|
||||
By default, the only strategy is to parse the markup
|
||||
as-is. See `LXMLTreeBuilderForXML` and
|
||||
`HTMLParserTreeBuilder` for implementations that take into
|
||||
account the quirks of particular parsers.
|
||||
|
||||
:meta private:
|
||||
|
||||
"""
|
||||
yield markup, None, None, False
|
||||
|
||||
def test_fragment_to_document(self, fragment: str) -> str:
|
||||
"""Wrap an HTML fragment to make it look like a document.
|
||||
|
||||
Different parsers do this differently. For instance, lxml
|
||||
introduces an empty <head> tag, and html5lib
|
||||
doesn't. Abstracting this away lets us write simple tests
|
||||
which run HTML fragments through the parser and compare the
|
||||
results against other HTML fragments.
|
||||
|
||||
This method should not be used outside of unit tests.
|
||||
|
||||
:param fragment: A fragment of HTML.
|
||||
:return: A full HTML document.
|
||||
:meta private:
|
||||
"""
|
||||
return fragment
|
||||
|
||||
def set_up_substitutions(self, tag: Tag) -> bool:
|
||||
"""Set up any substitutions that will need to be performed on
|
||||
a `Tag` when it's output as a string.
|
||||
|
||||
By default, this does nothing. See `HTMLTreeBuilder` for a
|
||||
case where this is used.
|
||||
|
||||
:return: Whether or not a substitution was performed.
|
||||
:meta private:
|
||||
"""
|
||||
return False
|
||||
|
||||
def _replace_cdata_list_attribute_values(
|
||||
self, tag_name: str, attrs: _RawOrProcessedAttributeValues
|
||||
) -> _AttributeValues:
|
||||
"""When an attribute value is associated with a tag that can
|
||||
have multiple values for that attribute, convert the string
|
||||
value to a list of strings.
|
||||
|
||||
Basically, replaces class="foo bar" with class=["foo", "bar"]
|
||||
|
||||
NOTE: This method modifies its input in place.
|
||||
|
||||
:param tag_name: The name of a tag.
|
||||
:param attrs: A dictionary containing the tag's attributes.
|
||||
Any appropriate attribute values will be modified in place.
|
||||
:return: The modified dictionary that was originally passed in.
|
||||
"""
|
||||
|
||||
# First, cast the attrs dict to _AttributeValues. This might
|
||||
# not be accurate yet, but it will be by the time this method
|
||||
# returns.
|
||||
modified_attrs = cast(_AttributeValues, attrs)
|
||||
if not modified_attrs or not self.cdata_list_attributes:
|
||||
# Nothing to do.
|
||||
return modified_attrs
|
||||
|
||||
# There is at least a possibility that we need to modify one of
|
||||
# the attribute values.
|
||||
universal: Set[str] = self.cdata_list_attributes.get("*", set())
|
||||
tag_specific = self.cdata_list_attributes.get(tag_name.lower(), None)
|
||||
for attr in list(modified_attrs.keys()):
|
||||
modified_value: _AttributeValue
|
||||
if attr in universal or (tag_specific and attr in tag_specific):
|
||||
# We have a "class"-type attribute whose string
|
||||
# value is a whitespace-separated list of
|
||||
# values. Split it into a list.
|
||||
original_value: _AttributeValue = modified_attrs[attr]
|
||||
if isinstance(original_value, _RawAttributeValue):
|
||||
# This is a _RawAttributeValue (a string) that
|
||||
# needs to be split and converted to a
|
||||
# AttributeValueList so it can be an
|
||||
# _AttributeValue.
|
||||
modified_value = self.attribute_value_list_class(
|
||||
nonwhitespace_re.findall(original_value)
|
||||
)
|
||||
else:
|
||||
# html5lib calls setAttributes twice for the
|
||||
# same tag when rearranging the parse tree. On
|
||||
# the second call the attribute value here is
|
||||
# already a list. This can also happen when a
|
||||
# Tag object is cloned. If this happens, leave
|
||||
# the value alone rather than trying to split
|
||||
# it again.
|
||||
modified_value = original_value
|
||||
modified_attrs[attr] = modified_value
|
||||
return modified_attrs
|
||||
|
||||
|
||||
class SAXTreeBuilder(TreeBuilder):
|
||||
"""A Beautiful Soup treebuilder that listens for SAX events.
|
||||
|
||||
This is not currently used for anything, and it will be removed
|
||||
soon. It was a good idea, but it wasn't properly integrated into the
|
||||
rest of Beautiful Soup, so there have been long stretches where it
|
||||
hasn't worked properly.
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
warnings.warn(
|
||||
"The SAXTreeBuilder class was deprecated in 4.13.0 and will be removed soon thereafter. It is completely untested and probably doesn't work; do not use it.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
super(SAXTreeBuilder, self).__init__(*args, **kwargs)
|
||||
|
||||
def feed(self, markup: _RawMarkup) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
def startElement(self, name: str, attrs: Dict[str, str]) -> None:
|
||||
attrs = AttributeDict((key[1], value) for key, value in list(attrs.items()))
|
||||
# print("Start %s, %r" % (name, attrs))
|
||||
assert self.soup is not None
|
||||
self.soup.handle_starttag(name, None, None, attrs)
|
||||
|
||||
def endElement(self, name: str) -> None:
|
||||
# print("End %s" % name)
|
||||
assert self.soup is not None
|
||||
self.soup.handle_endtag(name)
|
||||
|
||||
def startElementNS(
|
||||
self, nsTuple: Tuple[str, str], nodeName: str, attrs: Dict[str, str]
|
||||
) -> None:
|
||||
# Throw away (ns, nodeName) for now.
|
||||
self.startElement(nodeName, attrs)
|
||||
|
||||
def endElementNS(self, nsTuple: Tuple[str, str], nodeName: str) -> None:
|
||||
# Throw away (ns, nodeName) for now.
|
||||
self.endElement(nodeName)
|
||||
# handler.endElementNS((ns, node.nodeName), node.nodeName)
|
||||
|
||||
def startPrefixMapping(self, prefix: str, nodeValue: str) -> None:
|
||||
# Ignore the prefix for now.
|
||||
pass
|
||||
|
||||
def endPrefixMapping(self, prefix: str) -> None:
|
||||
# Ignore the prefix for now.
|
||||
# handler.endPrefixMapping(prefix)
|
||||
pass
|
||||
|
||||
def characters(self, content: str) -> None:
|
||||
assert self.soup is not None
|
||||
self.soup.handle_data(content)
|
||||
|
||||
def startDocument(self) -> None:
|
||||
pass
|
||||
|
||||
def endDocument(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class HTMLTreeBuilder(TreeBuilder):
|
||||
"""This TreeBuilder knows facts about HTML, such as which tags are treated
|
||||
specially by the HTML standard.
|
||||
"""
|
||||
|
||||
#: Some HTML tags are defined as having no contents. Beautiful Soup
|
||||
#: treats these specially.
|
||||
DEFAULT_EMPTY_ELEMENT_TAGS: Optional[Set[str]] = set(
|
||||
[
|
||||
# These are from HTML5.
|
||||
"area",
|
||||
"base",
|
||||
"br",
|
||||
"col",
|
||||
"embed",
|
||||
"hr",
|
||||
"img",
|
||||
"input",
|
||||
"keygen",
|
||||
"link",
|
||||
"menuitem",
|
||||
"meta",
|
||||
"param",
|
||||
"source",
|
||||
"track",
|
||||
"wbr",
|
||||
# These are from earlier versions of HTML and are removed in HTML5.
|
||||
"basefont",
|
||||
"bgsound",
|
||||
"command",
|
||||
"frame",
|
||||
"image",
|
||||
"isindex",
|
||||
"nextid",
|
||||
"spacer",
|
||||
]
|
||||
)
|
||||
|
||||
#: The HTML standard defines these tags as block-level elements. Beautiful
|
||||
#: Soup does not treat these elements differently from other elements,
|
||||
#: but it may do so eventually, and this information is available if
|
||||
#: you need to use it.
|
||||
DEFAULT_BLOCK_ELEMENTS: Set[str] = set(
|
||||
[
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"canvas",
|
||||
"dd",
|
||||
"div",
|
||||
"dl",
|
||||
"dt",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"li",
|
||||
"main",
|
||||
"nav",
|
||||
"noscript",
|
||||
"ol",
|
||||
"output",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"tfoot",
|
||||
"ul",
|
||||
"video",
|
||||
]
|
||||
)
|
||||
|
||||
#: These HTML tags need special treatment so they can be
|
||||
#: represented by a string class other than `bs4.element.NavigableString`.
|
||||
#:
|
||||
#: For some of these tags, it's because the HTML standard defines
|
||||
#: an unusual content model for them. I made this list by going
|
||||
#: through the HTML spec
|
||||
#: (https://html.spec.whatwg.org/#metadata-content) and looking for
|
||||
#: "metadata content" elements that can contain strings.
|
||||
#:
|
||||
#: The Ruby tags (<rt> and <rp>) are here despite being normal
|
||||
#: "phrasing content" tags, because the content they contain is
|
||||
#: qualitatively different from other text in the document, and it
|
||||
#: can be useful to be able to distinguish it.
|
||||
#:
|
||||
#: TODO: Arguably <noscript> could go here but it seems
|
||||
#: qualitatively different from the other tags.
|
||||
DEFAULT_STRING_CONTAINERS: Dict[str, Type[bs4.element.NavigableString]] = { # type:ignore
|
||||
"rt": RubyTextString,
|
||||
"rp": RubyParenthesisString,
|
||||
"style": Stylesheet,
|
||||
"script": Script,
|
||||
"template": TemplateString,
|
||||
}
|
||||
|
||||
#: The HTML standard defines these attributes as containing a
|
||||
#: space-separated list of values, not a single value. That is,
|
||||
#: class="foo bar" means that the 'class' attribute has two values,
|
||||
#: 'foo' and 'bar', not the single value 'foo bar'. When we
|
||||
#: encounter one of these attributes, we will parse its value into
|
||||
#: a list of values if possible. Upon output, the list will be
|
||||
#: converted back into a string.
|
||||
DEFAULT_CDATA_LIST_ATTRIBUTES: Dict[str, Set[str]] = {
|
||||
"*": {"class", "accesskey", "dropzone"},
|
||||
"a": {"rel", "rev"},
|
||||
"link": {"rel", "rev"},
|
||||
"td": {"headers"},
|
||||
"th": {"headers"},
|
||||
"form": {"accept-charset"},
|
||||
"object": {"archive"},
|
||||
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
|
||||
"area": {"rel"},
|
||||
"icon": {"sizes"},
|
||||
"iframe": {"sandbox"},
|
||||
"output": {"for"},
|
||||
}
|
||||
|
||||
#: By default, whitespace inside these HTML tags will be
|
||||
#: preserved rather than being collapsed.
|
||||
DEFAULT_PRESERVE_WHITESPACE_TAGS: set[str] = set(["pre", "textarea"])
|
||||
|
||||
def set_up_substitutions(self, tag: Tag) -> bool:
|
||||
"""Replace the declared encoding in a <meta> tag with a placeholder,
|
||||
to be substituted when the tag is output to a string.
|
||||
|
||||
An HTML document may come in to Beautiful Soup as one
|
||||
encoding, but exit in a different encoding, and the <meta> tag
|
||||
needs to be changed to reflect this.
|
||||
|
||||
:return: Whether or not a substitution was performed.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
# We are only interested in <meta> tags
|
||||
if tag.name != "meta":
|
||||
return False
|
||||
|
||||
# TODO: This cast will fail in the (very unlikely) scenario
|
||||
# that the programmer who instantiates the TreeBuilder
|
||||
# specifies meta['content'] or meta['charset'] as
|
||||
# cdata_list_attributes.
|
||||
content: Optional[str] = cast(Optional[str], tag.get("content"))
|
||||
charset: Optional[str] = cast(Optional[str], tag.get("charset"))
|
||||
|
||||
# But we can accommodate meta['http-equiv'] being made a
|
||||
# cdata_list_attribute (again, very unlikely) without much
|
||||
# trouble.
|
||||
http_equiv: List[str] = tag.get_attribute_list("http-equiv")
|
||||
|
||||
# We are interested in <meta> tags that say what encoding the
|
||||
# document was originally in. This means HTML 5-style <meta>
|
||||
# tags that provide the "charset" attribute. It also means
|
||||
# HTML 4-style <meta> tags that provide the "content"
|
||||
# attribute and have "http-equiv" set to "content-type".
|
||||
#
|
||||
# In both cases we will replace the value of the appropriate
|
||||
# attribute with a standin object that can take on any
|
||||
# encoding.
|
||||
substituted = False
|
||||
if charset is not None:
|
||||
# HTML 5 style:
|
||||
# <meta charset="utf8">
|
||||
tag["charset"] = CharsetMetaAttributeValue(charset)
|
||||
substituted = True
|
||||
|
||||
elif content is not None and any(
|
||||
x.lower() == "content-type" for x in http_equiv
|
||||
):
|
||||
# HTML 4 style:
|
||||
# <meta http-equiv="content-type" content="text/html; charset=utf8">
|
||||
tag["content"] = ContentMetaAttributeValue(content)
|
||||
substituted = True
|
||||
|
||||
return substituted
|
||||
|
||||
|
||||
class DetectsXMLParsedAsHTML(object):
|
||||
"""A mixin class for any class (a TreeBuilder, or some class used by a
|
||||
TreeBuilder) that's in a position to detect whether an XML
|
||||
document is being incorrectly parsed as HTML, and issue an
|
||||
appropriate warning.
|
||||
|
||||
This requires being able to observe an incoming processing
|
||||
instruction that might be an XML declaration, and also able to
|
||||
observe tags as they're opened. If you can't do that for a given
|
||||
`TreeBuilder`, there's a less reliable implementation based on
|
||||
examining the raw markup.
|
||||
"""
|
||||
|
||||
#: Regular expression for seeing if string markup has an <html> tag.
|
||||
LOOKS_LIKE_HTML: Pattern[str] = re.compile("<[^ +]html", re.I)
|
||||
|
||||
#: Regular expression for seeing if byte markup has an <html> tag.
|
||||
LOOKS_LIKE_HTML_B: Pattern[bytes] = re.compile(b"<[^ +]html", re.I)
|
||||
|
||||
#: The start of an XML document string.
|
||||
XML_PREFIX: str = "<?xml"
|
||||
|
||||
#: The start of an XML document bytestring.
|
||||
XML_PREFIX_B: bytes = b"<?xml"
|
||||
|
||||
# This is typed as str, not `ProcessingInstruction`, because this
|
||||
# check may be run before any Beautiful Soup objects are created.
|
||||
_first_processing_instruction: Optional[str] #: :meta private:
|
||||
_root_tag_name: Optional[str] #: :meta private:
|
||||
|
||||
@classmethod
|
||||
def warn_if_markup_looks_like_xml(
|
||||
cls, markup: Optional[_RawMarkup], stacklevel: int = 3
|
||||
) -> bool:
|
||||
"""Perform a check on some markup to see if it looks like XML
|
||||
that's not XHTML. If so, issue a warning.
|
||||
|
||||
This is much less reliable than doing the check while parsing,
|
||||
but some of the tree builders can't do that.
|
||||
|
||||
:param stacklevel: The stacklevel of the code calling this\
|
||||
function.
|
||||
|
||||
:return: True if the markup looks like non-XHTML XML, False
|
||||
otherwise.
|
||||
"""
|
||||
if markup is None:
|
||||
return False
|
||||
markup = markup[:500]
|
||||
if isinstance(markup, bytes):
|
||||
markup_b: bytes = markup
|
||||
looks_like_xml = markup_b.startswith(
|
||||
cls.XML_PREFIX_B
|
||||
) and not cls.LOOKS_LIKE_HTML_B.search(markup)
|
||||
else:
|
||||
markup_s: str = markup
|
||||
looks_like_xml = markup_s.startswith(
|
||||
cls.XML_PREFIX
|
||||
) and not cls.LOOKS_LIKE_HTML.search(markup)
|
||||
|
||||
if looks_like_xml:
|
||||
cls._warn(stacklevel=stacklevel + 2)
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _warn(cls, stacklevel: int = 5) -> None:
|
||||
"""Issue a warning about XML being parsed as HTML."""
|
||||
warnings.warn(
|
||||
XMLParsedAsHTMLWarning.MESSAGE,
|
||||
XMLParsedAsHTMLWarning,
|
||||
stacklevel=stacklevel,
|
||||
)
|
||||
|
||||
def _initialize_xml_detector(self) -> None:
|
||||
"""Call this method before parsing a document."""
|
||||
self._first_processing_instruction = None
|
||||
self._root_tag_name = None
|
||||
|
||||
def _document_might_be_xml(self, processing_instruction: str) -> None:
|
||||
"""Call this method when encountering an XML declaration, or a
|
||||
"processing instruction" that might be an XML declaration.
|
||||
|
||||
This helps Beautiful Soup detect potential issues later, if
|
||||
the XML document turns out to be a non-XHTML document that's
|
||||
being parsed as XML.
|
||||
"""
|
||||
if (
|
||||
self._first_processing_instruction is not None
|
||||
or self._root_tag_name is not None
|
||||
):
|
||||
# The document has already started. Don't bother checking
|
||||
# anymore.
|
||||
return
|
||||
|
||||
self._first_processing_instruction = processing_instruction
|
||||
|
||||
# We won't know until we encounter the first tag whether or
|
||||
# not this is actually a problem.
|
||||
|
||||
def _root_tag_encountered(self, name: str) -> None:
|
||||
"""Call this when you encounter the document's root tag.
|
||||
|
||||
This is where we actually check whether an XML document is
|
||||
being incorrectly parsed as HTML, and issue the warning.
|
||||
"""
|
||||
if self._root_tag_name is not None:
|
||||
# This method was incorrectly called multiple times. Do
|
||||
# nothing.
|
||||
return
|
||||
|
||||
self._root_tag_name = name
|
||||
|
||||
if (
|
||||
name != "html"
|
||||
and self._first_processing_instruction is not None
|
||||
and self._first_processing_instruction.lower().startswith("xml ")
|
||||
):
|
||||
# We encountered an XML declaration and then a tag other
|
||||
# than 'html'. This is a reliable indicator that a
|
||||
# non-XHTML document is being parsed as XML.
|
||||
self._warn(stacklevel=10)
|
||||
|
||||
|
||||
def register_treebuilders_from(module: ModuleType) -> None:
|
||||
"""Copy TreeBuilders from the given module into this module."""
|
||||
this_module = sys.modules[__name__]
|
||||
for name in module.__all__:
|
||||
obj = getattr(module, name)
|
||||
|
||||
if issubclass(obj, TreeBuilder):
|
||||
setattr(this_module, name, obj)
|
||||
this_module.__all__.append(name)
|
||||
# Register the builder while we're at it.
|
||||
this_module.builder_registry.register(obj)
|
||||
|
||||
|
||||
# Builders are registered in reverse order of priority, so that custom
|
||||
# builder registrations will take precedence. In general, we want lxml
|
||||
# to take precedence over html5lib, because it's faster. And we only
|
||||
# want to use HTMLParser as a last resort.
|
||||
from . import _htmlparser # noqa: E402
|
||||
|
||||
register_treebuilders_from(_htmlparser)
|
||||
try:
|
||||
from . import _html5lib
|
||||
|
||||
register_treebuilders_from(_html5lib)
|
||||
except ImportError:
|
||||
# They don't have html5lib installed.
|
||||
pass
|
||||
try:
|
||||
from . import _lxml
|
||||
|
||||
register_treebuilders_from(_lxml)
|
||||
except ImportError:
|
||||
# They don't have lxml installed.
|
||||
pass
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,611 @@
|
||||
# Use of this source code is governed by the MIT license.
|
||||
__license__ = "MIT"
|
||||
|
||||
__all__ = [
|
||||
"HTML5TreeBuilder",
|
||||
]
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
cast,
|
||||
Dict,
|
||||
Iterable,
|
||||
Optional,
|
||||
Sequence,
|
||||
TYPE_CHECKING,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
from typing_extensions import TypeAlias
|
||||
from bs4._typing import (
|
||||
_AttributeValue,
|
||||
_AttributeValues,
|
||||
_Encoding,
|
||||
_Encodings,
|
||||
_NamespaceURL,
|
||||
_RawMarkup,
|
||||
)
|
||||
|
||||
import warnings
|
||||
from bs4.builder import (
|
||||
DetectsXMLParsedAsHTML,
|
||||
PERMISSIVE,
|
||||
HTML,
|
||||
HTML_5,
|
||||
HTMLTreeBuilder,
|
||||
)
|
||||
from bs4.element import (
|
||||
NamespacedAttribute,
|
||||
PageElement,
|
||||
nonwhitespace_re,
|
||||
)
|
||||
import html5lib
|
||||
from html5lib.constants import (
|
||||
namespaces,
|
||||
)
|
||||
from bs4.element import (
|
||||
Comment,
|
||||
Doctype,
|
||||
NavigableString,
|
||||
Tag,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from html5lib.treebuilders import base as treebuilder_base
|
||||
|
||||
|
||||
class HTML5TreeBuilder(HTMLTreeBuilder):
|
||||
"""Use `html5lib <https://github.com/html5lib/html5lib-python>`_ to
|
||||
build a tree.
|
||||
|
||||
Note that `HTML5TreeBuilder` does not support some common HTML
|
||||
`TreeBuilder` features. Some of these features could theoretically
|
||||
be implemented, but at the very least it's quite difficult,
|
||||
because html5lib moves the parse tree around as it's being built.
|
||||
|
||||
Specifically:
|
||||
|
||||
* This `TreeBuilder` doesn't use different subclasses of
|
||||
`NavigableString` (e.g. `Script`) based on the name of the tag
|
||||
in which the string was found.
|
||||
* You can't use a `SoupStrainer` to parse only part of a document.
|
||||
"""
|
||||
|
||||
NAME: str = "html5lib"
|
||||
|
||||
features: Iterable[str] = [NAME, PERMISSIVE, HTML_5, HTML]
|
||||
|
||||
#: html5lib can tell us which line number and position in the
|
||||
#: original file is the source of an element.
|
||||
TRACKS_LINE_NUMBERS: bool = True
|
||||
|
||||
underlying_builder: "TreeBuilderForHtml5lib" #: :meta private:
|
||||
user_specified_encoding: Optional[_Encoding]
|
||||
|
||||
def prepare_markup(
|
||||
self,
|
||||
markup: _RawMarkup,
|
||||
user_specified_encoding: Optional[_Encoding] = None,
|
||||
document_declared_encoding: Optional[_Encoding] = None,
|
||||
exclude_encodings: Optional[_Encodings] = None,
|
||||
) -> Iterable[Tuple[_RawMarkup, Optional[_Encoding], Optional[_Encoding], bool]]:
|
||||
# Store the user-specified encoding for use later on.
|
||||
self.user_specified_encoding = user_specified_encoding
|
||||
|
||||
# document_declared_encoding and exclude_encodings aren't used
|
||||
# ATM because the html5lib TreeBuilder doesn't use
|
||||
# UnicodeDammit.
|
||||
for variable, name in (
|
||||
(document_declared_encoding, "document_declared_encoding"),
|
||||
(exclude_encodings, "exclude_encodings"),
|
||||
):
|
||||
if variable:
|
||||
warnings.warn(
|
||||
f"You provided a value for {name}, but the html5lib tree builder doesn't support {name}.",
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
# html5lib only parses HTML, so if it's given XML that's worth
|
||||
# noting.
|
||||
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup, stacklevel=3)
|
||||
|
||||
yield (markup, None, None, False)
|
||||
|
||||
# These methods are defined by Beautiful Soup.
|
||||
def feed(self, markup: _RawMarkup) -> None:
|
||||
"""Run some incoming markup through some parsing process,
|
||||
populating the `BeautifulSoup` object in `HTML5TreeBuilder.soup`.
|
||||
"""
|
||||
if self.soup is not None and self.soup.parse_only is not None:
|
||||
warnings.warn(
|
||||
"You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.",
|
||||
stacklevel=4,
|
||||
)
|
||||
|
||||
# self.underlying_builder is probably None now, but it'll be set
|
||||
# when html5lib calls self.create_treebuilder().
|
||||
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
|
||||
assert self.underlying_builder is not None
|
||||
self.underlying_builder.parser = parser
|
||||
extra_kwargs = dict()
|
||||
if not isinstance(markup, str):
|
||||
# kwargs, specifically override_encoding, will eventually
|
||||
# be passed in to html5lib's
|
||||
# HTMLBinaryInputStream.__init__.
|
||||
extra_kwargs["override_encoding"] = self.user_specified_encoding
|
||||
|
||||
doc = parser.parse(markup, **extra_kwargs) # type:ignore
|
||||
|
||||
# Set the character encoding detected by the tokenizer.
|
||||
if isinstance(markup, str):
|
||||
# We need to special-case this because html5lib sets
|
||||
# charEncoding to UTF-8 if it gets Unicode input.
|
||||
doc.original_encoding = None
|
||||
else:
|
||||
original_encoding = parser.tokenizer.stream.charEncoding[0] # type:ignore
|
||||
# The encoding is an html5lib Encoding object. We want to
|
||||
# use a string for compatibility with other tree builders.
|
||||
original_encoding = original_encoding.name
|
||||
doc.original_encoding = original_encoding
|
||||
self.underlying_builder.parser = None
|
||||
|
||||
def create_treebuilder(
|
||||
self, namespaceHTMLElements: bool
|
||||
) -> "TreeBuilderForHtml5lib":
|
||||
"""Called by html5lib to instantiate the kind of class it
|
||||
calls a 'TreeBuilder'.
|
||||
|
||||
:param namespaceHTMLElements: Whether or not to namespace HTML elements.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
self.underlying_builder = TreeBuilderForHtml5lib(
|
||||
namespaceHTMLElements, self.soup, store_line_numbers=self.store_line_numbers
|
||||
)
|
||||
return self.underlying_builder
|
||||
|
||||
def test_fragment_to_document(self, fragment: str) -> str:
|
||||
"""See `TreeBuilder`."""
|
||||
return "<html><head></head><body>%s</body></html>" % fragment
|
||||
|
||||
|
||||
class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
|
||||
soup: "BeautifulSoup" #: :meta private:
|
||||
parser: Optional[html5lib.HTMLParser] #: :meta private:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
namespaceHTMLElements: bool,
|
||||
soup: Optional["BeautifulSoup"] = None,
|
||||
store_line_numbers: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
if soup:
|
||||
self.soup = soup
|
||||
else:
|
||||
warnings.warn(
|
||||
"The optionality of the 'soup' argument to the TreeBuilderForHtml5lib constructor is deprecated as of Beautiful Soup 4.13.0: 'soup' is now required. If you can't pass in a BeautifulSoup object here, or you get this warning and it seems mysterious to you, please contact the Beautiful Soup developer team for possible un-deprecation.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# TODO: Why is the parser 'html.parser' here? Using
|
||||
# html5lib doesn't cause an infinite loop and is more
|
||||
# accurate. Best to get rid of this entire section, I think.
|
||||
self.soup = BeautifulSoup(
|
||||
"", "html.parser", store_line_numbers=store_line_numbers, **kwargs
|
||||
)
|
||||
# TODO: What are **kwargs exactly? Should they be passed in
|
||||
# here in addition to/instead of being passed to the BeautifulSoup
|
||||
# constructor?
|
||||
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
|
||||
|
||||
# This will be set later to a real html5lib HTMLParser object,
|
||||
# which we can use to track the current line number.
|
||||
self.parser = None
|
||||
self.store_line_numbers = store_line_numbers
|
||||
|
||||
def documentClass(self) -> "Element":
|
||||
self.soup.reset()
|
||||
return Element(self.soup, self.soup, None)
|
||||
|
||||
def insertDoctype(self, token: Dict[str, Any]) -> None:
|
||||
name: str = cast(str, token["name"])
|
||||
publicId: Optional[str] = cast(Optional[str], token["publicId"])
|
||||
systemId: Optional[str] = cast(Optional[str], token["systemId"])
|
||||
|
||||
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
|
||||
self.soup.object_was_parsed(doctype)
|
||||
|
||||
def elementClass(self, name: str, namespace: str) -> "Element":
|
||||
sourceline: Optional[int] = None
|
||||
sourcepos: Optional[int] = None
|
||||
if self.parser is not None and self.store_line_numbers:
|
||||
# This represents the point immediately after the end of the
|
||||
# tag. We don't know when the tag started, but we do know
|
||||
# where it ended -- the character just before this one.
|
||||
sourceline, sourcepos = self.parser.tokenizer.stream.position() # type:ignore
|
||||
assert sourcepos is not None
|
||||
sourcepos = sourcepos - 1
|
||||
tag = self.soup.new_tag(
|
||||
name, namespace, sourceline=sourceline, sourcepos=sourcepos
|
||||
)
|
||||
|
||||
return Element(tag, self.soup, namespace)
|
||||
|
||||
def commentClass(self, data: str) -> "TextNode":
|
||||
return TextNode(Comment(data), self.soup)
|
||||
|
||||
def fragmentClass(self) -> "Element":
|
||||
"""This is only used by html5lib HTMLParser.parseFragment(),
|
||||
which is never used by Beautiful Soup, only by the html5lib
|
||||
unit tests. Since we don't currently hook into those tests,
|
||||
the implementation is left blank.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def getFragment(self) -> "Element":
|
||||
"""This is only used by the html5lib unit tests. Since we
|
||||
don't currently hook into those tests, the implementation is
|
||||
left blank.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def appendChild(self, node: "Element") -> None:
|
||||
# TODO: This code is not covered by the BS4 tests, and
|
||||
# apparently not triggered by the html5lib test suite either.
|
||||
# But it doesn't seem test-specific and there are calls to it
|
||||
# (or a method with the same name) all over html5lib, so I'm
|
||||
# leaving the implementation in place rather than replacing it
|
||||
# with NotImplementedError()
|
||||
self.soup.append(node.element)
|
||||
|
||||
def getDocument(self) -> "BeautifulSoup":
|
||||
return self.soup
|
||||
|
||||
def testSerializer(self, node: "Element") -> None:
|
||||
"""This is only used by the html5lib unit tests. Since we
|
||||
don't currently hook into those tests, the implementation is
|
||||
left blank.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class AttrList(object):
|
||||
"""Represents a Tag's attributes in a way compatible with html5lib."""
|
||||
|
||||
element: Tag
|
||||
attrs: _AttributeValues
|
||||
|
||||
def __init__(self, element: Tag):
|
||||
self.element = element
|
||||
self.attrs = dict(self.element.attrs)
|
||||
|
||||
def __iter__(self) -> Iterable[Tuple[str, _AttributeValue]]:
|
||||
return list(self.attrs.items()).__iter__()
|
||||
|
||||
def __setitem__(self, name: str, value: _AttributeValue) -> None:
|
||||
# If this attribute is a multi-valued attribute for this element,
|
||||
# turn its value into a list.
|
||||
list_attr = self.element.cdata_list_attributes or {}
|
||||
if name in list_attr.get("*", []) or (
|
||||
self.element.name in list_attr
|
||||
and name in list_attr.get(self.element.name, [])
|
||||
):
|
||||
# A node that is being cloned may have already undergone
|
||||
# this procedure. Check for this and skip it.
|
||||
if not isinstance(value, list):
|
||||
assert isinstance(value, str)
|
||||
value = self.element.attribute_value_list_class(
|
||||
nonwhitespace_re.findall(value)
|
||||
)
|
||||
self.element[name] = value
|
||||
|
||||
def items(self) -> Iterable[Tuple[str, _AttributeValue]]:
|
||||
return list(self.attrs.items())
|
||||
|
||||
def keys(self) -> Iterable[str]:
|
||||
return list(self.attrs.keys())
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.attrs)
|
||||
|
||||
def __getitem__(self, name: str) -> _AttributeValue:
|
||||
return self.attrs[name]
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
return name in list(self.attrs.keys())
|
||||
|
||||
|
||||
class BeautifulSoupNode(treebuilder_base.Node):
|
||||
# A node can correspond to _either_ a Tag _or_ a NavigableString.
|
||||
tag: Optional[Tag]
|
||||
string: Optional[NavigableString]
|
||||
soup: "BeautifulSoup"
|
||||
namespace: Optional[_NamespaceURL]
|
||||
|
||||
@property
|
||||
def element(self) -> PageElement:
|
||||
assert self.tag is not None or self.string is not None
|
||||
if self.tag is not None:
|
||||
return self.tag
|
||||
else:
|
||||
assert self.string is not None
|
||||
return self.string
|
||||
|
||||
@property
|
||||
def nodeType(self) -> int:
|
||||
"""Return the html5lib constant corresponding to the type of
|
||||
the underlying DOM object.
|
||||
|
||||
NOTE: This property is only accessed by the html5lib test
|
||||
suite, not by Beautiful Soup proper.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
# TODO-TYPING: typeshed stubs are incorrect about this;
|
||||
# cloneNode returns a new Node, not None.
|
||||
def cloneNode(self) -> treebuilder_base.Node: # type:ignore
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class Element(BeautifulSoupNode):
|
||||
namespace: Optional[_NamespaceURL]
|
||||
|
||||
def __init__(
|
||||
self, element: Tag, soup: "BeautifulSoup", namespace: Optional[_NamespaceURL]
|
||||
):
|
||||
self.tag = element
|
||||
self.string = None
|
||||
self.soup = soup
|
||||
self.namespace = namespace
|
||||
treebuilder_base.Node.__init__(self, element.name)
|
||||
|
||||
def appendChild(self, node: "BeautifulSoupNode") -> None:
|
||||
string_child: Optional[NavigableString] = None
|
||||
child: PageElement
|
||||
if type(node.string) is NavigableString:
|
||||
# We check for NavigableString *only* because we want to avoid
|
||||
# joining PreformattedStrings, such as Comments, with nearby strings.
|
||||
string_child = child = node.string
|
||||
else:
|
||||
child = node.element
|
||||
node.parent = self
|
||||
|
||||
if (
|
||||
child is not None
|
||||
and child.parent is not None
|
||||
and not isinstance(child, str)
|
||||
):
|
||||
node.element.extract()
|
||||
|
||||
if (
|
||||
string_child is not None
|
||||
and self.tag is not None and self.tag.contents
|
||||
and type(self.tag.contents[-1]) is NavigableString
|
||||
):
|
||||
# We are appending a string onto another string.
|
||||
# TODO This has O(n^2) performance, for input like
|
||||
# "a</a>a</a>a</a>..."
|
||||
old_element = self.tag.contents[-1]
|
||||
new_element = self.soup.new_string(old_element + string_child)
|
||||
old_element.replace_with(new_element)
|
||||
self.soup._most_recent_element = new_element
|
||||
else:
|
||||
if isinstance(node, str):
|
||||
# Create a brand new NavigableString from this string.
|
||||
child = self.soup.new_string(node)
|
||||
|
||||
# Tell Beautiful Soup to act as if it parsed this element
|
||||
# immediately after the parent's last descendant. (Or
|
||||
# immediately after the parent, if it has no children.)
|
||||
if self.tag is not None and self.tag.contents:
|
||||
most_recent_element = self.tag._last_descendant(False)
|
||||
elif self.element.next_element is not None:
|
||||
# Something from further ahead in the parse tree is
|
||||
# being inserted into this earlier element. This is
|
||||
# very annoying because it means an expensive search
|
||||
# for the last element in the tree.
|
||||
most_recent_element = self.soup._last_descendant()
|
||||
else:
|
||||
most_recent_element = self.element
|
||||
|
||||
self.soup.object_was_parsed(
|
||||
child, parent=self.tag, most_recent_element=most_recent_element
|
||||
)
|
||||
|
||||
def getAttributes(self) -> AttrList:
|
||||
assert self.tag is not None
|
||||
return AttrList(self.tag)
|
||||
|
||||
# An HTML5lib attribute name may either be a single string,
|
||||
# or a tuple (namespace, name).
|
||||
_Html5libAttributeName: TypeAlias = Union[str, Tuple[str, str]]
|
||||
# Now we can define the type this method accepts as a dictionary
|
||||
# mapping those attribute names to single string values.
|
||||
_Html5libAttributes: TypeAlias = Dict[_Html5libAttributeName, str]
|
||||
|
||||
def setAttributes(self, attributes: Optional[_Html5libAttributes]) -> None:
|
||||
assert self.tag is not None
|
||||
if attributes is not None and len(attributes) > 0:
|
||||
# Replace any namespaced attributes with
|
||||
# NamespacedAttribute objects.
|
||||
for name, value in list(attributes.items()):
|
||||
if isinstance(name, tuple):
|
||||
new_name = NamespacedAttribute(*name)
|
||||
del attributes[name]
|
||||
attributes[new_name] = value
|
||||
|
||||
# We can now cast attributes to the type of Dict
|
||||
# used by Beautiful Soup.
|
||||
normalized_attributes = cast(_AttributeValues, attributes)
|
||||
|
||||
# Values for tags like 'class' came in as single strings;
|
||||
# replace them with lists of strings as appropriate.
|
||||
self.soup.builder._replace_cdata_list_attribute_values(
|
||||
self.name, normalized_attributes
|
||||
)
|
||||
|
||||
# Then set the attributes on the Tag associated with this
|
||||
# BeautifulSoupNode.
|
||||
for name, value_or_values in list(normalized_attributes.items()):
|
||||
self.tag[name] = value_or_values
|
||||
|
||||
# The attributes may contain variables that need substitution.
|
||||
# Call set_up_substitutions manually.
|
||||
#
|
||||
# The Tag constructor called this method when the Tag was created,
|
||||
# but we just set/changed the attributes, so call it again.
|
||||
self.soup.builder.set_up_substitutions(self.tag)
|
||||
|
||||
attributes = property(getAttributes, setAttributes)
|
||||
|
||||
def insertText(
|
||||
self, data: str, insertBefore: Optional["BeautifulSoupNode"] = None
|
||||
) -> None:
|
||||
text = TextNode(self.soup.new_string(data), self.soup)
|
||||
if insertBefore:
|
||||
self.insertBefore(text, insertBefore)
|
||||
else:
|
||||
self.appendChild(text)
|
||||
|
||||
def insertBefore(
|
||||
self, node: "BeautifulSoupNode", refNode: "BeautifulSoupNode"
|
||||
) -> None:
|
||||
assert self.tag is not None
|
||||
index = self.tag.index(refNode.element)
|
||||
if (
|
||||
type(node.element) is NavigableString
|
||||
and self.tag.contents
|
||||
and type(self.tag.contents[index - 1]) is NavigableString
|
||||
):
|
||||
# (See comments in appendChild)
|
||||
old_node = self.tag.contents[index - 1]
|
||||
assert type(old_node) is NavigableString
|
||||
new_str = self.soup.new_string(old_node + node.element)
|
||||
old_node.replace_with(new_str)
|
||||
else:
|
||||
self.tag.insert(index, node.element)
|
||||
node.parent = self
|
||||
|
||||
def removeChild(self, node: "Element") -> None:
|
||||
node.element.extract()
|
||||
|
||||
def reparentChildren(self, newParent: "Element") -> None:
|
||||
"""Move all of this tag's children into another tag."""
|
||||
# print("MOVE", self.element.contents)
|
||||
# print("FROM", self.element)
|
||||
# print("TO", new_parent.element)
|
||||
|
||||
element = self.tag
|
||||
assert element is not None
|
||||
new_parent_element = newParent.tag
|
||||
assert new_parent_element is not None
|
||||
# Determine what this tag's next_element will be once all the children
|
||||
# are removed.
|
||||
final_next_element = element.next_sibling
|
||||
|
||||
new_parents_last_descendant = new_parent_element._last_descendant(False, False)
|
||||
if len(new_parent_element.contents) > 0:
|
||||
# The new parent already contains children. We will be
|
||||
# appending this tag's children to the end.
|
||||
|
||||
# We can make this assertion since we know new_parent has
|
||||
# children.
|
||||
assert new_parents_last_descendant is not None
|
||||
new_parents_last_child = new_parent_element.contents[-1]
|
||||
new_parents_last_descendant_next_element = (
|
||||
new_parents_last_descendant.next_element
|
||||
)
|
||||
else:
|
||||
# The new parent contains no children.
|
||||
new_parents_last_child = None
|
||||
new_parents_last_descendant_next_element = new_parent_element.next_element
|
||||
|
||||
to_append = element.contents
|
||||
if len(to_append) > 0:
|
||||
# Set the first child's previous_element and previous_sibling
|
||||
# to elements within the new parent
|
||||
first_child = to_append[0]
|
||||
if new_parents_last_descendant is not None:
|
||||
first_child.previous_element = new_parents_last_descendant
|
||||
else:
|
||||
first_child.previous_element = new_parent_element
|
||||
first_child.previous_sibling = new_parents_last_child
|
||||
if new_parents_last_descendant is not None:
|
||||
new_parents_last_descendant.next_element = first_child
|
||||
else:
|
||||
new_parent_element.next_element = first_child
|
||||
if new_parents_last_child is not None:
|
||||
new_parents_last_child.next_sibling = first_child
|
||||
|
||||
# Find the very last element being moved. It is now the
|
||||
# parent's last descendant. It has no .next_sibling and
|
||||
# its .next_element is whatever the previous last
|
||||
# descendant had.
|
||||
last_childs_last_descendant = to_append[-1]._last_descendant(
|
||||
is_initialized=False, accept_self=True
|
||||
)
|
||||
|
||||
# Since we passed accept_self=True into _last_descendant,
|
||||
# there's no possibility that the result is None.
|
||||
assert last_childs_last_descendant is not None
|
||||
last_childs_last_descendant.next_element = (
|
||||
new_parents_last_descendant_next_element
|
||||
)
|
||||
if new_parents_last_descendant_next_element is not None:
|
||||
# TODO-COVERAGE: This code has no test coverage and
|
||||
# I'm not sure how to get html5lib to go through this
|
||||
# path, but it's just the other side of the previous
|
||||
# line.
|
||||
new_parents_last_descendant_next_element.previous_element = (
|
||||
last_childs_last_descendant
|
||||
)
|
||||
last_childs_last_descendant.next_sibling = None
|
||||
|
||||
for child in to_append:
|
||||
child.parent = new_parent_element
|
||||
new_parent_element.contents.append(child)
|
||||
|
||||
# Now that this element has no children, change its .next_element.
|
||||
element.contents = []
|
||||
element.next_element = final_next_element
|
||||
|
||||
# print("DONE WITH MOVE")
|
||||
# print("FROM", self.element)
|
||||
# print("TO", new_parent_element)
|
||||
|
||||
# TODO-TYPING: typeshed stubs are incorrect about this;
|
||||
# hasContent returns a boolean, not None.
|
||||
def hasContent(self) -> bool: # type:ignore
|
||||
return self.tag is None or len(self.tag.contents) > 0
|
||||
|
||||
# TODO-TYPING: typeshed stubs are incorrect about this;
|
||||
# cloneNode returns a new Node, not None.
|
||||
def cloneNode(self) -> treebuilder_base.Node: # type:ignore
|
||||
assert self.tag is not None
|
||||
tag = self.soup.new_tag(self.tag.name, self.namespace)
|
||||
node = Element(tag, self.soup, self.namespace)
|
||||
for key, value in self.attributes:
|
||||
node.attributes[key] = value
|
||||
return node
|
||||
|
||||
def getNameTuple(self) -> Tuple[Optional[_NamespaceURL], str]:
|
||||
if self.namespace is None:
|
||||
return namespaces["html"], self.name
|
||||
else:
|
||||
return self.namespace, self.name
|
||||
|
||||
nameTuple = property(getNameTuple)
|
||||
|
||||
|
||||
class TextNode(BeautifulSoupNode):
|
||||
|
||||
def __init__(self, element: NavigableString, soup: "BeautifulSoup"):
|
||||
treebuilder_base.Node.__init__(self, None)
|
||||
self.tag = None
|
||||
self.string = element
|
||||
self.soup = soup
|
||||
@@ -0,0 +1,462 @@
|
||||
# encoding: utf-8
|
||||
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
|
||||
from __future__ import annotations
|
||||
|
||||
# Use of this source code is governed by the MIT license.
|
||||
__license__ = "MIT"
|
||||
|
||||
__all__ = [
|
||||
"HTMLParserTreeBuilder",
|
||||
]
|
||||
|
||||
from html.parser import HTMLParser
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
cast,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
from bs4.element import (
|
||||
AttributeDict,
|
||||
CData,
|
||||
Comment,
|
||||
Declaration,
|
||||
Doctype,
|
||||
ProcessingInstruction,
|
||||
)
|
||||
from bs4.dammit import EntitySubstitution, UnicodeDammit
|
||||
|
||||
from bs4.builder import (
|
||||
DetectsXMLParsedAsHTML,
|
||||
HTML,
|
||||
HTMLTreeBuilder,
|
||||
STRICT,
|
||||
)
|
||||
|
||||
from bs4.exceptions import ParserRejectedMarkup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import NavigableString
|
||||
from bs4._typing import (
|
||||
_Encoding,
|
||||
_Encodings,
|
||||
_RawMarkup,
|
||||
)
|
||||
|
||||
HTMLPARSER = "html.parser"
|
||||
|
||||
_DuplicateAttributeHandler = Callable[[Dict[str, str], str, str], None]
|
||||
|
||||
|
||||
class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
|
||||
#: Constant to handle duplicate attributes by ignoring later values
|
||||
#: and keeping the earlier ones.
|
||||
REPLACE: str = "replace"
|
||||
|
||||
#: Constant to handle duplicate attributes by replacing earlier values
|
||||
#: with later ones.
|
||||
IGNORE: str = "ignore"
|
||||
|
||||
"""A subclass of the Python standard library's HTMLParser class, which
|
||||
listens for HTMLParser events and translates them into calls
|
||||
to Beautiful Soup's tree construction API.
|
||||
|
||||
:param on_duplicate_attribute: A strategy for what to do if a
|
||||
tag includes the same attribute more than once. Accepted
|
||||
values are: REPLACE (replace earlier values with later
|
||||
ones, the default), IGNORE (keep the earliest value
|
||||
encountered), or a callable. A callable must take three
|
||||
arguments: the dictionary of attributes already processed,
|
||||
the name of the duplicate attribute, and the most recent value
|
||||
encountered.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
soup: BeautifulSoup,
|
||||
*args: Any,
|
||||
on_duplicate_attribute: Union[str, _DuplicateAttributeHandler] = REPLACE,
|
||||
**kwargs: Any,
|
||||
):
|
||||
self.soup = soup
|
||||
self.on_duplicate_attribute = on_duplicate_attribute
|
||||
self.attribute_dict_class = soup.builder.attribute_dict_class
|
||||
HTMLParser.__init__(self, *args, **kwargs)
|
||||
|
||||
# Keep a list of empty-element tags that were encountered
|
||||
# without an explicit closing tag. If we encounter a closing tag
|
||||
# of this type, we'll associate it with one of those entries.
|
||||
#
|
||||
# This isn't a stack because we don't care about the
|
||||
# order. It's a list of closing tags we've already handled and
|
||||
# will ignore, assuming they ever show up.
|
||||
self.already_closed_empty_element = []
|
||||
|
||||
self._initialize_xml_detector()
|
||||
|
||||
on_duplicate_attribute: Union[str, _DuplicateAttributeHandler]
|
||||
already_closed_empty_element: List[str]
|
||||
soup: BeautifulSoup
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
# NOTE: This method is required so long as Python 3.9 is
|
||||
# supported. The corresponding code is removed from HTMLParser
|
||||
# in 3.5, but not removed from ParserBase until 3.10.
|
||||
# https://github.com/python/cpython/issues/76025
|
||||
#
|
||||
# The original implementation turned the error into a warning,
|
||||
# but in every case I discovered, this made HTMLParser
|
||||
# immediately crash with an error message that was less
|
||||
# helpful than the warning. The new implementation makes it
|
||||
# more clear that html.parser just can't parse this
|
||||
# markup. The 3.10 implementation does the same, though it
|
||||
# raises AssertionError rather than calling a method. (We
|
||||
# catch this error and wrap it in a ParserRejectedMarkup.)
|
||||
raise ParserRejectedMarkup(message)
|
||||
|
||||
def handle_startendtag(
|
||||
self, tag: str, attrs: List[Tuple[str, Optional[str]]]
|
||||
) -> None:
|
||||
"""Handle an incoming empty-element tag.
|
||||
|
||||
html.parser only calls this method when the markup looks like
|
||||
<tag/>.
|
||||
"""
|
||||
# `handle_empty_element` tells handle_starttag not to close the tag
|
||||
# just because its name matches a known empty-element tag. We
|
||||
# know that this is an empty-element tag, and we want to call
|
||||
# handle_endtag ourselves.
|
||||
self.handle_starttag(tag, attrs, handle_empty_element=False)
|
||||
self.handle_endtag(tag)
|
||||
|
||||
def handle_starttag(
|
||||
self,
|
||||
tag: str,
|
||||
attrs: List[Tuple[str, Optional[str]]],
|
||||
handle_empty_element: bool = True,
|
||||
) -> None:
|
||||
"""Handle an opening tag, e.g. '<tag>'
|
||||
|
||||
:param handle_empty_element: True if this tag is known to be
|
||||
an empty-element tag (i.e. there is not expected to be any
|
||||
closing tag).
|
||||
"""
|
||||
# TODO: handle namespaces here?
|
||||
attr_dict: AttributeDict = self.attribute_dict_class()
|
||||
for key, value in attrs:
|
||||
# Change None attribute values to the empty string
|
||||
# for consistency with the other tree builders.
|
||||
if value is None:
|
||||
value = ""
|
||||
if key in attr_dict:
|
||||
# A single attribute shows up multiple times in this
|
||||
# tag. How to handle it depends on the
|
||||
# on_duplicate_attribute setting.
|
||||
on_dupe = self.on_duplicate_attribute
|
||||
if on_dupe == self.IGNORE:
|
||||
pass
|
||||
elif on_dupe in (None, self.REPLACE):
|
||||
attr_dict[key] = value
|
||||
else:
|
||||
on_dupe = cast(_DuplicateAttributeHandler, on_dupe)
|
||||
on_dupe(attr_dict, key, value)
|
||||
else:
|
||||
attr_dict[key] = value
|
||||
# print("START", tag)
|
||||
sourceline: Optional[int]
|
||||
sourcepos: Optional[int]
|
||||
if self.soup.builder.store_line_numbers:
|
||||
sourceline, sourcepos = self.getpos()
|
||||
else:
|
||||
sourceline = sourcepos = None
|
||||
tagObj = self.soup.handle_starttag(
|
||||
tag, None, None, attr_dict, sourceline=sourceline, sourcepos=sourcepos
|
||||
)
|
||||
if tagObj is not None and tagObj.is_empty_element and handle_empty_element:
|
||||
# Unlike other parsers, html.parser doesn't send separate end tag
|
||||
# events for empty-element tags. (It's handled in
|
||||
# handle_startendtag, but only if the original markup looked like
|
||||
# <tag/>.)
|
||||
#
|
||||
# So we need to call handle_endtag() ourselves. Since we
|
||||
# know the start event is identical to the end event, we
|
||||
# don't want handle_endtag() to cross off any previous end
|
||||
# events for tags of this name.
|
||||
self.handle_endtag(tag, check_already_closed=False)
|
||||
|
||||
# But we might encounter an explicit closing tag for this tag
|
||||
# later on. If so, we want to ignore it.
|
||||
self.already_closed_empty_element.append(tag)
|
||||
|
||||
if self._root_tag_name is None:
|
||||
self._root_tag_encountered(tag)
|
||||
|
||||
def handle_endtag(self, tag: str, check_already_closed: bool = True) -> None:
|
||||
"""Handle a closing tag, e.g. '</tag>'
|
||||
|
||||
:param tag: A tag name.
|
||||
:param check_already_closed: True if this tag is expected to
|
||||
be the closing portion of an empty-element tag,
|
||||
e.g. '<tag></tag>'.
|
||||
"""
|
||||
# print("END", tag)
|
||||
if check_already_closed and tag in self.already_closed_empty_element:
|
||||
# This is a redundant end tag for an empty-element tag.
|
||||
# We've already called handle_endtag() for it, so just
|
||||
# check it off the list.
|
||||
# print("ALREADY CLOSED", tag)
|
||||
self.already_closed_empty_element.remove(tag)
|
||||
else:
|
||||
self.soup.handle_endtag(tag)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
"""Handle some textual data that shows up between tags."""
|
||||
self.soup.handle_data(data)
|
||||
|
||||
def handle_charref(self, name: str) -> None:
|
||||
"""Handle a numeric character reference by converting it to the
|
||||
corresponding Unicode character and treating it as textual
|
||||
data.
|
||||
|
||||
:param name: Character number, possibly in hexadecimal.
|
||||
"""
|
||||
# TODO: This was originally a workaround for a bug in
|
||||
# HTMLParser. (http://bugs.python.org/issue13633) The bug has
|
||||
# been fixed, but removing this code still makes some
|
||||
# Beautiful Soup tests fail. This needs investigation.
|
||||
real_name:int
|
||||
if name.startswith("x"):
|
||||
real_name = int(name.lstrip("x"), 16)
|
||||
elif name.startswith("X"):
|
||||
real_name = int(name.lstrip("X"), 16)
|
||||
else:
|
||||
real_name = int(name)
|
||||
|
||||
data, replacement_added = UnicodeDammit.numeric_character_reference(real_name)
|
||||
if replacement_added:
|
||||
self.soup.contains_replacement_characters = True
|
||||
self.handle_data(data)
|
||||
|
||||
def handle_entityref(self, name: str) -> None:
|
||||
"""Handle a named entity reference by converting it to the
|
||||
corresponding Unicode character(s) and treating it as textual
|
||||
data.
|
||||
|
||||
:param name: Name of the entity reference.
|
||||
"""
|
||||
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
|
||||
if character is not None:
|
||||
data = character
|
||||
else:
|
||||
# If this were XML, it would be ambiguous whether "&foo"
|
||||
# was an character entity reference with a missing
|
||||
# semicolon or the literal string "&foo". Since this is
|
||||
# HTML, we have a complete list of all character entity references,
|
||||
# and this one wasn't found, so assume it's the literal string "&foo".
|
||||
data = "&%s" % name
|
||||
self.handle_data(data)
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
"""Handle an HTML comment.
|
||||
|
||||
:param data: The text of the comment.
|
||||
"""
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(Comment)
|
||||
|
||||
def handle_decl(self, decl: str) -> None:
|
||||
"""Handle a DOCTYPE declaration.
|
||||
|
||||
:param data: The text of the declaration.
|
||||
"""
|
||||
self.soup.endData()
|
||||
decl = decl[len("DOCTYPE ") :]
|
||||
self.soup.handle_data(decl)
|
||||
self.soup.endData(Doctype)
|
||||
|
||||
def unknown_decl(self, data: str) -> None:
|
||||
"""Handle a declaration of unknown type -- probably a CDATA block.
|
||||
|
||||
:param data: The text of the declaration.
|
||||
"""
|
||||
cls: Type[NavigableString]
|
||||
if data.upper().startswith("CDATA["):
|
||||
cls = CData
|
||||
data = data[len("CDATA[") :]
|
||||
else:
|
||||
cls = Declaration
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(cls)
|
||||
|
||||
def handle_pi(self, data: str) -> None:
|
||||
"""Handle a processing instruction.
|
||||
|
||||
:param data: The text of the instruction.
|
||||
"""
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(data)
|
||||
self._document_might_be_xml(data)
|
||||
self.soup.endData(ProcessingInstruction)
|
||||
|
||||
|
||||
class HTMLParserTreeBuilder(HTMLTreeBuilder):
|
||||
"""A Beautiful soup `bs4.builder.TreeBuilder` that uses the
|
||||
:py:class:`html.parser.HTMLParser` parser, found in the Python
|
||||
standard library.
|
||||
|
||||
"""
|
||||
|
||||
is_xml: bool = False
|
||||
picklable: bool = True
|
||||
NAME: str = HTMLPARSER
|
||||
features: Iterable[str] = [NAME, HTML, STRICT]
|
||||
parser_args: Tuple[Iterable[Any], Dict[str, Any]]
|
||||
|
||||
#: The html.parser knows which line number and position in the
|
||||
#: original file is the source of an element.
|
||||
TRACKS_LINE_NUMBERS: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parser_args: Optional[Iterable[Any]] = None,
|
||||
parser_kwargs: Optional[Dict[str, Any]] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""Constructor.
|
||||
|
||||
:param parser_args: Positional arguments to pass into
|
||||
the BeautifulSoupHTMLParser constructor, once it's
|
||||
invoked.
|
||||
:param parser_kwargs: Keyword arguments to pass into
|
||||
the BeautifulSoupHTMLParser constructor, once it's
|
||||
invoked.
|
||||
:param kwargs: Keyword arguments for the superclass constructor.
|
||||
"""
|
||||
# Some keyword arguments will be pulled out of kwargs and placed
|
||||
# into parser_kwargs.
|
||||
extra_parser_kwargs = dict()
|
||||
for arg in ("on_duplicate_attribute",):
|
||||
if arg in kwargs:
|
||||
value = kwargs.pop(arg)
|
||||
extra_parser_kwargs[arg] = value
|
||||
super(HTMLParserTreeBuilder, self).__init__(**kwargs)
|
||||
parser_args = parser_args or []
|
||||
parser_kwargs = parser_kwargs or {}
|
||||
parser_kwargs.update(extra_parser_kwargs)
|
||||
parser_kwargs["convert_charrefs"] = False
|
||||
self.parser_args = (parser_args, parser_kwargs)
|
||||
|
||||
def prepare_markup(
|
||||
self,
|
||||
markup: _RawMarkup,
|
||||
user_specified_encoding: Optional[_Encoding] = None,
|
||||
document_declared_encoding: Optional[_Encoding] = None,
|
||||
exclude_encodings: Optional[_Encodings] = None,
|
||||
) -> Iterable[Tuple[str, Optional[_Encoding], Optional[_Encoding], bool]]:
|
||||
"""Run any preliminary steps necessary to make incoming markup
|
||||
acceptable to the parser.
|
||||
|
||||
:param markup: Some markup -- probably a bytestring.
|
||||
:param user_specified_encoding: The user asked to try this encoding.
|
||||
:param document_declared_encoding: The markup itself claims to be
|
||||
in this encoding.
|
||||
:param exclude_encodings: The user asked _not_ to try any of
|
||||
these encodings.
|
||||
|
||||
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
|
||||
has undergone character replacement)
|
||||
|
||||
Each 4-tuple represents a strategy for parsing the document.
|
||||
This TreeBuilder uses Unicode, Dammit to convert the markup
|
||||
into Unicode, so the ``markup`` element of the tuple will
|
||||
always be a string.
|
||||
"""
|
||||
if isinstance(markup, str):
|
||||
# Parse Unicode as-is.
|
||||
yield (markup, None, None, False)
|
||||
return
|
||||
|
||||
# Ask UnicodeDammit to sniff the most likely encoding.
|
||||
|
||||
known_definite_encodings: List[_Encoding] = []
|
||||
if user_specified_encoding:
|
||||
# This was provided by the end-user; treat it as a known
|
||||
# definite encoding per the algorithm laid out in the
|
||||
# HTML5 spec. (See the EncodingDetector class for
|
||||
# details.)
|
||||
known_definite_encodings.append(user_specified_encoding)
|
||||
|
||||
user_encodings: List[_Encoding] = []
|
||||
if document_declared_encoding:
|
||||
# This was found in the document; treat it as a slightly
|
||||
# lower-priority user encoding.
|
||||
user_encodings.append(document_declared_encoding)
|
||||
|
||||
dammit = UnicodeDammit(
|
||||
markup,
|
||||
known_definite_encodings=known_definite_encodings,
|
||||
user_encodings=user_encodings,
|
||||
is_html=True,
|
||||
exclude_encodings=exclude_encodings,
|
||||
)
|
||||
|
||||
if dammit.unicode_markup is None:
|
||||
# In every case I've seen, Unicode, Dammit is able to
|
||||
# convert the markup into Unicode, even if it needs to use
|
||||
# REPLACEMENT CHARACTER. But there is a code path that
|
||||
# could result in unicode_markup being None, and
|
||||
# HTMLParser can only parse Unicode, so here we handle
|
||||
# that code path.
|
||||
raise ParserRejectedMarkup(
|
||||
"Could not convert input to Unicode, and html.parser will not accept bytestrings."
|
||||
)
|
||||
else:
|
||||
yield (
|
||||
dammit.unicode_markup,
|
||||
dammit.original_encoding,
|
||||
dammit.declared_html_encoding,
|
||||
dammit.contains_replacement_characters,
|
||||
)
|
||||
|
||||
def feed(self, markup: _RawMarkup, _parser_class:type[BeautifulSoupHTMLParser] =BeautifulSoupHTMLParser) -> None:
|
||||
"""
|
||||
:param markup: The markup to feed into the parser.
|
||||
:param _parser_class: An HTMLParser subclass to use. This is only intended for use in unit tests.
|
||||
"""
|
||||
args, kwargs = self.parser_args
|
||||
|
||||
# HTMLParser.feed will only handle str, but
|
||||
# BeautifulSoup.markup is allowed to be _RawMarkup, because
|
||||
# it's set by the yield value of
|
||||
# TreeBuilder.prepare_markup. Fortunately,
|
||||
# HTMLParserTreeBuilder.prepare_markup always yields a str
|
||||
# (UnicodeDammit.unicode_markup).
|
||||
assert isinstance(markup, str)
|
||||
|
||||
# We know BeautifulSoup calls TreeBuilder.initialize_soup
|
||||
# before calling feed(), so we can assume self.soup
|
||||
# is set.
|
||||
assert self.soup is not None
|
||||
parser = _parser_class(self.soup, *args, **kwargs)
|
||||
|
||||
try:
|
||||
parser.feed(markup)
|
||||
parser.close()
|
||||
except AssertionError as e:
|
||||
# html.parser raises AssertionError in rare cases to
|
||||
# indicate a fatal problem with the markup, especially
|
||||
# when there's an error in the doctype declaration.
|
||||
raise ParserRejectedMarkup(e)
|
||||
parser.already_closed_empty_element = []
|
||||
@@ -0,0 +1,501 @@
|
||||
# encoding: utf-8
|
||||
from __future__ import annotations
|
||||
|
||||
# Use of this source code is governed by the MIT license.
|
||||
__license__ = "MIT"
|
||||
|
||||
__all__ = [
|
||||
"LXMLTreeBuilderForXML",
|
||||
"LXMLTreeBuilder",
|
||||
]
|
||||
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
|
||||
from io import BytesIO
|
||||
from io import StringIO
|
||||
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
from lxml import etree # type:ignore
|
||||
from bs4.element import (
|
||||
AttributeDict,
|
||||
XMLAttributeDict,
|
||||
Comment,
|
||||
Doctype,
|
||||
NamespacedAttribute,
|
||||
ProcessingInstruction,
|
||||
XMLProcessingInstruction,
|
||||
)
|
||||
from bs4.builder import (
|
||||
DetectsXMLParsedAsHTML,
|
||||
FAST,
|
||||
HTML,
|
||||
HTMLTreeBuilder,
|
||||
PERMISSIVE,
|
||||
TreeBuilder,
|
||||
XML,
|
||||
)
|
||||
from bs4.dammit import EncodingDetector
|
||||
from bs4.exceptions import ParserRejectedMarkup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4._typing import (
|
||||
_Encoding,
|
||||
_Encodings,
|
||||
_NamespacePrefix,
|
||||
_NamespaceURL,
|
||||
_NamespaceMapping,
|
||||
_InvertedNamespaceMapping,
|
||||
_RawMarkup,
|
||||
)
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
LXML: str = "lxml"
|
||||
|
||||
|
||||
def _invert(d: dict[Any, Any]) -> dict[Any, Any]:
|
||||
"Invert a dictionary."
|
||||
return dict((v, k) for k, v in list(d.items()))
|
||||
|
||||
|
||||
_LXMLParser: TypeAlias = Union[etree.XMLParser, etree.HTMLParser]
|
||||
_ParserOrParserClass: TypeAlias = Union[
|
||||
_LXMLParser, Type[etree.XMLParser], Type[etree.HTMLParser]
|
||||
]
|
||||
|
||||
|
||||
class LXMLTreeBuilderForXML(TreeBuilder):
|
||||
DEFAULT_PARSER_CLASS: Type[etree.XMLParser] = etree.XMLParser
|
||||
|
||||
is_xml: bool = True
|
||||
|
||||
#: Set this to true (probably by passing huge_tree=True into the :
|
||||
#: BeautifulSoup constructor) to enable the lxml feature "disable security
|
||||
#: restrictions and support very deep trees and very long text
|
||||
#: content".
|
||||
huge_tree: bool
|
||||
|
||||
processing_instruction_class: Type[ProcessingInstruction]
|
||||
|
||||
NAME: str = "lxml-xml"
|
||||
ALTERNATE_NAMES: Iterable[str] = ["xml"]
|
||||
|
||||
# Well, it's permissive by XML parser standards.
|
||||
features: Iterable[str] = [NAME, LXML, XML, FAST, PERMISSIVE]
|
||||
|
||||
CHUNK_SIZE: int = 512
|
||||
|
||||
# This namespace mapping is specified in the XML Namespace
|
||||
# standard.
|
||||
DEFAULT_NSMAPS: _NamespaceMapping = dict(xml="http://www.w3.org/XML/1998/namespace")
|
||||
|
||||
DEFAULT_NSMAPS_INVERTED: _InvertedNamespaceMapping = _invert(DEFAULT_NSMAPS)
|
||||
|
||||
nsmaps: List[Optional[_InvertedNamespaceMapping]]
|
||||
empty_element_tags: Optional[Set[str]]
|
||||
parser: Any
|
||||
_default_parser: Optional[etree.XMLParser]
|
||||
|
||||
# NOTE: If we parsed Element objects and looked at .sourceline,
|
||||
# we'd be able to see the line numbers from the original document.
|
||||
# But instead we build an XMLParser or HTMLParser object to serve
|
||||
# as the target of parse messages, and those messages don't include
|
||||
# line numbers.
|
||||
# See: https://bugs.launchpad.net/lxml/+bug/1846906
|
||||
|
||||
def initialize_soup(self, soup: BeautifulSoup) -> None:
|
||||
"""Let the BeautifulSoup object know about the standard namespace
|
||||
mapping.
|
||||
|
||||
:param soup: A `BeautifulSoup`.
|
||||
"""
|
||||
# Beyond this point, self.soup is set, so we can assume (and
|
||||
# assert) it's not None whenever necessary.
|
||||
super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
|
||||
self._register_namespaces(self.DEFAULT_NSMAPS)
|
||||
|
||||
def _register_namespaces(self, mapping: Dict[str, str]) -> None:
|
||||
"""Let the BeautifulSoup object know about namespaces encountered
|
||||
while parsing the document.
|
||||
|
||||
This might be useful later on when creating CSS selectors.
|
||||
|
||||
This will track (almost) all namespaces, even ones that were
|
||||
only in scope for part of the document. If two namespaces have
|
||||
the same prefix, only the first one encountered will be
|
||||
tracked. Un-prefixed namespaces are not tracked.
|
||||
|
||||
:param mapping: A dictionary mapping namespace prefixes to URIs.
|
||||
"""
|
||||
assert self.soup is not None
|
||||
for key, value in list(mapping.items()):
|
||||
# This is 'if key' and not 'if key is not None' because we
|
||||
# don't track un-prefixed namespaces. Soupselect will
|
||||
# treat an un-prefixed namespace as the default, which
|
||||
# causes confusion in some cases.
|
||||
if key and key not in self.soup._namespaces:
|
||||
# Let the BeautifulSoup object know about a new namespace.
|
||||
# If there are multiple namespaces defined with the same
|
||||
# prefix, the first one in the document takes precedence.
|
||||
self.soup._namespaces[key] = value
|
||||
|
||||
def default_parser(self, encoding: Optional[_Encoding]) -> _ParserOrParserClass:
|
||||
"""Find the default parser for the given encoding.
|
||||
|
||||
:return: Either a parser object or a class, which
|
||||
will be instantiated with default arguments.
|
||||
"""
|
||||
if self._default_parser is not None:
|
||||
return self._default_parser
|
||||
return self.DEFAULT_PARSER_CLASS(target=self, recover=True, huge_tree=self.huge_tree, encoding=encoding)
|
||||
|
||||
def parser_for(self, encoding: Optional[_Encoding]) -> _LXMLParser:
|
||||
"""Instantiate an appropriate parser for the given encoding.
|
||||
|
||||
:param encoding: A string.
|
||||
:return: A parser object such as an `etree.XMLParser`.
|
||||
"""
|
||||
# Use the default parser.
|
||||
parser = self.default_parser(encoding)
|
||||
|
||||
if callable(parser):
|
||||
# Instantiate the parser with default arguments
|
||||
parser = parser(target=self, recover=True, huge_tree=self.huge_tree, encoding=encoding)
|
||||
return parser
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parser: Optional[etree.XMLParser] = None,
|
||||
empty_element_tags: Optional[Set[str]] = None,
|
||||
huge_tree: bool = False,
|
||||
**kwargs: Any,
|
||||
):
|
||||
# TODO: Issue a warning if parser is present but not a
|
||||
# callable, since that means there's no way to create new
|
||||
# parsers for different encodings.
|
||||
self._default_parser = parser
|
||||
self.soup = None
|
||||
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
|
||||
self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
|
||||
if self.is_xml:
|
||||
self.processing_instruction_class = XMLProcessingInstruction
|
||||
else:
|
||||
self.processing_instruction_class = ProcessingInstruction
|
||||
|
||||
if "attribute_dict_class" not in kwargs:
|
||||
kwargs["attribute_dict_class"] = XMLAttributeDict
|
||||
self.huge_tree = huge_tree
|
||||
|
||||
super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
|
||||
|
||||
def _getNsTag(self, tag: str) -> Tuple[Optional[str], str]:
|
||||
# Split the namespace URL out of a fully-qualified lxml tag
|
||||
# name. Copied from lxml's src/lxml/sax.py.
|
||||
if tag[0] == "{" and "}" in tag:
|
||||
namespace, name = tag[1:].split("}", 1)
|
||||
return (namespace, name)
|
||||
return (None, tag)
|
||||
|
||||
def prepare_markup(
|
||||
self,
|
||||
markup: _RawMarkup,
|
||||
user_specified_encoding: Optional[_Encoding] = None,
|
||||
document_declared_encoding: Optional[_Encoding] = None,
|
||||
exclude_encodings: Optional[_Encodings] = None,
|
||||
) -> Iterable[
|
||||
Tuple[Union[str, bytes], Optional[_Encoding], Optional[_Encoding], bool]
|
||||
]:
|
||||
"""Run any preliminary steps necessary to make incoming markup
|
||||
acceptable to the parser.
|
||||
|
||||
lxml really wants to get a bytestring and convert it to
|
||||
Unicode itself. So instead of using UnicodeDammit to convert
|
||||
the bytestring to Unicode using different encodings, this
|
||||
implementation uses EncodingDetector to iterate over the
|
||||
encodings, and tell lxml to try to parse the document as each
|
||||
one in turn.
|
||||
|
||||
:param markup: Some markup -- hopefully a bytestring.
|
||||
:param user_specified_encoding: The user asked to try this encoding.
|
||||
:param document_declared_encoding: The markup itself claims to be
|
||||
in this encoding.
|
||||
:param exclude_encodings: The user asked _not_ to try any of
|
||||
these encodings.
|
||||
|
||||
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
|
||||
has undergone character replacement)
|
||||
|
||||
Each 4-tuple represents a strategy for converting the
|
||||
document to Unicode and parsing it. Each strategy will be tried
|
||||
in turn.
|
||||
"""
|
||||
if not self.is_xml:
|
||||
# We're in HTML mode, so if we're given XML, that's worth
|
||||
# noting.
|
||||
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup, stacklevel=3)
|
||||
|
||||
if isinstance(markup, str):
|
||||
# We were given Unicode. Maybe lxml can parse Unicode on
|
||||
# this system?
|
||||
|
||||
# TODO: This is a workaround for
|
||||
# https://bugs.launchpad.net/lxml/+bug/1948551.
|
||||
# We can remove it once the upstream issue is fixed.
|
||||
if len(markup) > 0 and markup[0] == "\N{BYTE ORDER MARK}":
|
||||
markup = markup[1:]
|
||||
yield markup, None, document_declared_encoding, False
|
||||
|
||||
if isinstance(markup, str):
|
||||
# No, apparently not. Convert the Unicode to UTF-8 and
|
||||
# tell lxml to parse it as UTF-8.
|
||||
yield (markup.encode("utf8"), "utf8", document_declared_encoding, False)
|
||||
|
||||
# Since the document was Unicode in the first place, there
|
||||
# is no need to try any more strategies; we know this will
|
||||
# work.
|
||||
return
|
||||
|
||||
known_definite_encodings: List[_Encoding] = []
|
||||
if user_specified_encoding:
|
||||
# This was provided by the end-user; treat it as a known
|
||||
# definite encoding per the algorithm laid out in the
|
||||
# HTML5 spec. (See the EncodingDetector class for
|
||||
# details.)
|
||||
known_definite_encodings.append(user_specified_encoding)
|
||||
|
||||
user_encodings: List[_Encoding] = []
|
||||
if document_declared_encoding:
|
||||
# This was found in the document; treat it as a slightly
|
||||
# lower-priority user encoding.
|
||||
user_encodings.append(document_declared_encoding)
|
||||
|
||||
detector = EncodingDetector(
|
||||
markup,
|
||||
known_definite_encodings=known_definite_encodings,
|
||||
user_encodings=user_encodings,
|
||||
is_html=not self.is_xml,
|
||||
exclude_encodings=exclude_encodings,
|
||||
)
|
||||
for encoding in detector.encodings:
|
||||
yield (detector.markup, encoding, document_declared_encoding, False)
|
||||
|
||||
def feed(self, markup: _RawMarkup) -> None:
|
||||
io: Union[BytesIO, StringIO]
|
||||
if isinstance(markup, bytes):
|
||||
io = BytesIO(markup)
|
||||
elif isinstance(markup, str):
|
||||
io = StringIO(markup)
|
||||
|
||||
# initialize_soup is called before feed, so we know this
|
||||
# is not None.
|
||||
assert self.soup is not None
|
||||
|
||||
# Call feed() at least once, even if the markup is empty,
|
||||
# or the parser won't be initialized.
|
||||
data = io.read(self.CHUNK_SIZE)
|
||||
try:
|
||||
self.parser = self.parser_for(self.soup.original_encoding)
|
||||
self.parser.feed(data)
|
||||
while len(data) != 0:
|
||||
# Now call feed() on the rest of the data, chunk by chunk.
|
||||
data = io.read(self.CHUNK_SIZE)
|
||||
if len(data) != 0:
|
||||
self.parser.feed(data)
|
||||
self.parser.close()
|
||||
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
|
||||
raise ParserRejectedMarkup(e)
|
||||
|
||||
def close(self) -> None:
|
||||
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
|
||||
|
||||
def start(
|
||||
self,
|
||||
tag: str | bytes,
|
||||
attrib: Dict[str | bytes, str | bytes],
|
||||
nsmap: _NamespaceMapping = {},
|
||||
) -> None:
|
||||
# This is called by lxml code as a result of calling
|
||||
# BeautifulSoup.feed(), and we know self.soup is set by the time feed()
|
||||
# is called.
|
||||
assert self.soup is not None
|
||||
assert isinstance(tag, str)
|
||||
|
||||
# We need to recreate the attribute dict for three
|
||||
# reasons. First, for type checking, so we can assert there
|
||||
# are no bytestrings in the keys or values. Second, because we
|
||||
# need a mutable dict--lxml might send us an immutable
|
||||
# dictproxy. Third, so we can handle namespaced attribute
|
||||
# names by converting the keys to NamespacedAttributes.
|
||||
new_attrib: Dict[Union[str, NamespacedAttribute], str] = (
|
||||
self.attribute_dict_class()
|
||||
)
|
||||
for k, v in attrib.items():
|
||||
assert isinstance(k, str)
|
||||
assert isinstance(v, str)
|
||||
new_attrib[k] = v
|
||||
|
||||
nsprefix: Optional[_NamespacePrefix] = None
|
||||
namespace: Optional[_NamespaceURL] = None
|
||||
# Invert each namespace map as it comes in.
|
||||
if len(nsmap) == 0 and len(self.nsmaps) > 1:
|
||||
# There are no new namespaces for this tag, but
|
||||
# non-default namespaces are in play, so we need a
|
||||
# separate tag stack to know when they end.
|
||||
self.nsmaps.append(None)
|
||||
elif len(nsmap) > 0:
|
||||
# A new namespace mapping has come into play.
|
||||
|
||||
# First, Let the BeautifulSoup object know about it.
|
||||
self._register_namespaces(nsmap)
|
||||
|
||||
# Then, add it to our running list of inverted namespace
|
||||
# mappings.
|
||||
self.nsmaps.append(_invert(nsmap))
|
||||
|
||||
# The currently active namespace prefixes have
|
||||
# changed. Calculate the new mapping so it can be stored
|
||||
# with all Tag objects created while these prefixes are in
|
||||
# scope.
|
||||
current_mapping = dict(self.active_namespace_prefixes[-1])
|
||||
current_mapping.update(nsmap)
|
||||
|
||||
# We should not track un-prefixed namespaces as we can only hold one
|
||||
# and it will be recognized as the default namespace by soupsieve,
|
||||
# which may be confusing in some situations.
|
||||
if "" in current_mapping:
|
||||
del current_mapping[""]
|
||||
self.active_namespace_prefixes.append(current_mapping)
|
||||
|
||||
# Also treat the namespace mapping as a set of attributes on the
|
||||
# tag, so we can recreate it later.
|
||||
for prefix, namespace in list(nsmap.items()):
|
||||
attribute = NamespacedAttribute(
|
||||
"xmlns", prefix, "http://www.w3.org/2000/xmlns/"
|
||||
)
|
||||
new_attrib[attribute] = namespace
|
||||
|
||||
# Namespaces are in play. Find any attributes that came in
|
||||
# from lxml with namespaces attached to their names, and
|
||||
# turn then into NamespacedAttribute objects.
|
||||
final_attrib: AttributeDict = self.attribute_dict_class()
|
||||
for attr, value in list(new_attrib.items()):
|
||||
namespace, attr = self._getNsTag(attr)
|
||||
if namespace is None:
|
||||
final_attrib[attr] = value
|
||||
else:
|
||||
nsprefix = self._prefix_for_namespace(namespace)
|
||||
attr = NamespacedAttribute(nsprefix, attr, namespace)
|
||||
final_attrib[attr] = value
|
||||
|
||||
namespace, tag = self._getNsTag(tag)
|
||||
nsprefix = self._prefix_for_namespace(namespace)
|
||||
self.soup.handle_starttag(
|
||||
tag,
|
||||
namespace,
|
||||
nsprefix,
|
||||
final_attrib,
|
||||
namespaces=self.active_namespace_prefixes[-1],
|
||||
)
|
||||
|
||||
def _prefix_for_namespace(
|
||||
self, namespace: Optional[_NamespaceURL]
|
||||
) -> Optional[_NamespacePrefix]:
|
||||
"""Find the currently active prefix for the given namespace."""
|
||||
if namespace is None:
|
||||
return None
|
||||
for inverted_nsmap in reversed(self.nsmaps):
|
||||
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||
return inverted_nsmap[namespace]
|
||||
return None
|
||||
|
||||
def end(self, tag: str | bytes) -> None:
|
||||
assert self.soup is not None
|
||||
assert isinstance(tag, str)
|
||||
self.soup.endData()
|
||||
namespace, tag = self._getNsTag(tag)
|
||||
nsprefix = None
|
||||
if namespace is not None:
|
||||
for inverted_nsmap in reversed(self.nsmaps):
|
||||
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||
nsprefix = inverted_nsmap[namespace]
|
||||
break
|
||||
self.soup.handle_endtag(tag, nsprefix)
|
||||
if len(self.nsmaps) > 1:
|
||||
# This tag, or one of its parents, introduced a namespace
|
||||
# mapping, so pop it off the stack.
|
||||
out_of_scope_nsmap = self.nsmaps.pop()
|
||||
|
||||
if out_of_scope_nsmap is not None:
|
||||
# This tag introduced a namespace mapping which is no
|
||||
# longer in scope. Recalculate the currently active
|
||||
# namespace prefixes.
|
||||
self.active_namespace_prefixes.pop()
|
||||
|
||||
def pi(self, target: str, data: str) -> None:
|
||||
assert self.soup is not None
|
||||
self.soup.endData()
|
||||
data = target + " " + data
|
||||
self.soup.handle_data(data)
|
||||
self.soup.endData(self.processing_instruction_class)
|
||||
|
||||
def data(self, data: str | bytes) -> None:
|
||||
assert self.soup is not None
|
||||
assert isinstance(data, str)
|
||||
self.soup.handle_data(data)
|
||||
|
||||
def doctype(self, name: str, pubid: str, system: str) -> None:
|
||||
assert self.soup is not None
|
||||
self.soup.endData()
|
||||
doctype_string = Doctype._string_for_name_and_ids(name, pubid, system)
|
||||
self.soup.handle_data(doctype_string)
|
||||
self.soup.endData(containerClass=Doctype)
|
||||
|
||||
def comment(self, text: str | bytes) -> None:
|
||||
"Handle comments as Comment objects."
|
||||
assert self.soup is not None
|
||||
assert isinstance(text, str)
|
||||
self.soup.endData()
|
||||
self.soup.handle_data(text)
|
||||
self.soup.endData(Comment)
|
||||
|
||||
def test_fragment_to_document(self, fragment: str) -> str:
|
||||
"""See `TreeBuilder`."""
|
||||
return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
|
||||
|
||||
|
||||
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
|
||||
NAME: str = LXML
|
||||
ALTERNATE_NAMES: Iterable[str] = ["lxml-html"]
|
||||
|
||||
features: Iterable[str] = list(ALTERNATE_NAMES) + [NAME, HTML, FAST, PERMISSIVE]
|
||||
is_xml: bool = False
|
||||
|
||||
def default_parser(self, encoding: Optional[_Encoding]) -> _ParserOrParserClass:
|
||||
return etree.HTMLParser
|
||||
|
||||
def feed(self, markup: _RawMarkup) -> None:
|
||||
# We know self.soup is set by the time feed() is called.
|
||||
assert self.soup is not None
|
||||
encoding = self.soup.original_encoding
|
||||
try:
|
||||
self.parser = self.parser_for(encoding)
|
||||
self.parser.feed(markup)
|
||||
self.parser.close()
|
||||
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
|
||||
raise ParserRejectedMarkup(e)
|
||||
|
||||
def test_fragment_to_document(self, fragment: str) -> str:
|
||||
"""See `TreeBuilder`."""
|
||||
return "<html><body>%s</body></html>" % fragment
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Integration code for CSS selectors using `Soup Sieve <https://facelessuser.github.io/soupsieve/>`_ (pypi: ``soupsieve``).
|
||||
|
||||
Acquire a `CSS` object through the `element.Tag.css` attribute of
|
||||
the starting point of your CSS selector, or (if you want to run a
|
||||
selector against the entire document) of the `BeautifulSoup` object
|
||||
itself.
|
||||
|
||||
The main advantage of doing this instead of using ``soupsieve``
|
||||
functions is that you don't need to keep passing the `element.Tag` to be
|
||||
selected against, since the `CSS` object is permanently scoped to that
|
||||
`element.Tag`.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
cast,
|
||||
Iterable,
|
||||
Iterator,
|
||||
MutableSequence,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
import warnings
|
||||
from bs4._typing import _NamespaceMapping
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from soupsieve import SoupSieve
|
||||
from bs4 import element
|
||||
from bs4.element import ResultSet, Tag
|
||||
|
||||
soupsieve: Optional[ModuleType]
|
||||
try:
|
||||
import soupsieve
|
||||
except ImportError:
|
||||
soupsieve = None
|
||||
warnings.warn(
|
||||
"The soupsieve package is not installed. CSS selectors cannot be used."
|
||||
)
|
||||
|
||||
|
||||
class CSS(object):
|
||||
"""A proxy object against the ``soupsieve`` library, to simplify its
|
||||
CSS selector API.
|
||||
|
||||
You don't need to instantiate this class yourself; instead, use
|
||||
`element.Tag.css`.
|
||||
|
||||
:param tag: All CSS selectors run by this object will use this as
|
||||
their starting point.
|
||||
|
||||
:param api: An optional drop-in replacement for the ``soupsieve`` module,
|
||||
intended for use in unit tests.
|
||||
"""
|
||||
|
||||
def __init__(self, tag: element.Tag, api: Optional[ModuleType] = None):
|
||||
if api is None:
|
||||
api = soupsieve
|
||||
if api is None:
|
||||
raise NotImplementedError(
|
||||
"Cannot execute CSS selectors because the soupsieve package is not installed."
|
||||
)
|
||||
self.api = api
|
||||
self.tag = tag
|
||||
|
||||
def escape(self, ident: str) -> str:
|
||||
"""Escape a CSS identifier.
|
||||
|
||||
This is a simple wrapper around `soupsieve.escape() <https://facelessuser.github.io/soupsieve/api/#soupsieveescape>`_. See the
|
||||
documentation for that function for more information.
|
||||
"""
|
||||
if soupsieve is None:
|
||||
raise NotImplementedError(
|
||||
"Cannot escape CSS identifiers because the soupsieve package is not installed."
|
||||
)
|
||||
return cast(str, self.api.escape(ident))
|
||||
|
||||
def _ns(
|
||||
self, ns: Optional[_NamespaceMapping], select: str
|
||||
) -> Optional[_NamespaceMapping]:
|
||||
"""Normalize a dictionary of namespaces."""
|
||||
if not isinstance(select, self.api.SoupSieve) and ns is None:
|
||||
# If the selector is a precompiled pattern, it already has
|
||||
# a namespace context compiled in, which cannot be
|
||||
# replaced.
|
||||
ns = self.tag._namespaces
|
||||
return ns
|
||||
|
||||
def _rs(self, results: MutableSequence[Tag]) -> ResultSet[Tag]:
|
||||
"""Normalize a list of results to a py:class:`ResultSet`.
|
||||
|
||||
A py:class:`ResultSet` is more consistent with the rest of
|
||||
Beautiful Soup's API, and :py:meth:`ResultSet.__getattr__` has
|
||||
a helpful error message if you try to treat a list of results
|
||||
as a single result (a common mistake).
|
||||
"""
|
||||
# Import here to avoid circular import
|
||||
from bs4 import ResultSet
|
||||
|
||||
return ResultSet(None, results)
|
||||
|
||||
def compile(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> SoupSieve:
|
||||
"""Pre-compile a selector and return the compiled object.
|
||||
|
||||
:param selector: A CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will use the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.compile() <https://facelessuser.github.io/soupsieve/api/#soupsievecompile>`_ method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into Soup Sieve's
|
||||
`soupsieve.compile() <https://facelessuser.github.io/soupsieve/api/#soupsievecompile>`_ method.
|
||||
|
||||
:return: A precompiled selector object.
|
||||
:rtype: soupsieve.SoupSieve
|
||||
"""
|
||||
return self.api.compile(select, self._ns(namespaces, select), flags, **kwargs)
|
||||
|
||||
def select_one(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> element.Tag | None:
|
||||
"""Perform a CSS selection operation on the current Tag and return the
|
||||
first result, if any.
|
||||
|
||||
This uses the Soup Sieve library. For more information, see
|
||||
that library's documentation for the `soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
|
||||
|
||||
:param selector: A CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will use the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into Soup Sieve's
|
||||
`soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
|
||||
"""
|
||||
return self.api.select_one(
|
||||
select, self.tag, self._ns(namespaces, select), flags, **kwargs
|
||||
)
|
||||
|
||||
def select(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
limit: int = 0,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> ResultSet[element.Tag]:
|
||||
"""Perform a CSS selection operation on the current `element.Tag`.
|
||||
|
||||
This uses the Soup Sieve library. For more information, see
|
||||
that library's documentation for the `soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
|
||||
|
||||
:param selector: A CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will pass in the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param limit: After finding this number of results, stop looking.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into Soup Sieve's
|
||||
`soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
|
||||
"""
|
||||
if limit is None:
|
||||
limit = 0
|
||||
|
||||
return self._rs(
|
||||
self.api.select(
|
||||
select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
|
||||
)
|
||||
)
|
||||
|
||||
def iselect(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
limit: int = 0,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[element.Tag]:
|
||||
"""Perform a CSS selection operation on the current `element.Tag`.
|
||||
|
||||
This uses the Soup Sieve library. For more information, see
|
||||
that library's documentation for the `soupsieve.iselect()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_
|
||||
method. It is the same as select(), but it returns a generator
|
||||
instead of a list.
|
||||
|
||||
:param selector: A string containing a CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will pass in the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param limit: After finding this number of results, stop looking.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.iselect() <https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_ method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into Soup Sieve's
|
||||
`soupsieve.iselect() <https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_ method.
|
||||
"""
|
||||
return self.api.iselect(
|
||||
select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
|
||||
)
|
||||
|
||||
def closest(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> Optional[element.Tag]:
|
||||
"""Find the `element.Tag` closest to this one that matches the given selector.
|
||||
|
||||
This uses the Soup Sieve library. For more information, see
|
||||
that library's documentation for the `soupsieve.closest()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_
|
||||
method.
|
||||
|
||||
:param selector: A string containing a CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will pass in the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.closest() <https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_ method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into Soup Sieve's
|
||||
`soupsieve.closest() <https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_ method.
|
||||
|
||||
"""
|
||||
return self.api.closest(
|
||||
select, self.tag, self._ns(namespaces, select), flags, **kwargs
|
||||
)
|
||||
|
||||
def match(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> bool:
|
||||
"""Check whether or not this `element.Tag` matches the given CSS selector.
|
||||
|
||||
This uses the Soup Sieve library. For more information, see
|
||||
that library's documentation for the `soupsieve.match()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
|
||||
method.
|
||||
|
||||
:param: a CSS selector.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will pass in the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.match()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
|
||||
method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into SoupSieve's
|
||||
`soupsieve.match()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
|
||||
method.
|
||||
"""
|
||||
return cast(
|
||||
bool,
|
||||
self.api.match(
|
||||
select, self.tag, self._ns(namespaces, select), flags, **kwargs
|
||||
),
|
||||
)
|
||||
|
||||
def filter(
|
||||
self,
|
||||
select: str,
|
||||
namespaces: Optional[_NamespaceMapping] = None,
|
||||
flags: int = 0,
|
||||
**kwargs: Any,
|
||||
) -> ResultSet[element.Tag]:
|
||||
"""Filter this `element.Tag`'s direct children based on the given CSS selector.
|
||||
|
||||
This uses the Soup Sieve library. It works the same way as
|
||||
passing a `element.Tag` into that library's `soupsieve.filter()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
|
||||
method. For more information, see the documentation for
|
||||
`soupsieve.filter()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_.
|
||||
|
||||
:param namespaces: A dictionary mapping namespace prefixes
|
||||
used in the CSS selector to namespace URIs. By default,
|
||||
Beautiful Soup will pass in the prefixes it encountered while
|
||||
parsing the document.
|
||||
|
||||
:param flags: Flags to be passed into Soup Sieve's
|
||||
`soupsieve.filter()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
|
||||
method.
|
||||
|
||||
:param kwargs: Keyword arguments to be passed into SoupSieve's
|
||||
`soupsieve.filter()
|
||||
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
|
||||
method.
|
||||
"""
|
||||
return self._rs(
|
||||
self.api.filter(
|
||||
select, self.tag, self._ns(namespaces, select), flags, **kwargs
|
||||
)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,268 @@
|
||||
"""Diagnostic functions, mainly for use when doing tech support."""
|
||||
|
||||
# Use of this source code is governed by the MIT license.
|
||||
__license__ = "MIT"
|
||||
|
||||
import cProfile
|
||||
from io import BytesIO
|
||||
from html.parser import HTMLParser
|
||||
import bs4
|
||||
from bs4 import BeautifulSoup, __version__
|
||||
from bs4.builder import builder_registry
|
||||
from typing import (
|
||||
Any,
|
||||
IO,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bs4._typing import _IncomingMarkup
|
||||
|
||||
import pstats
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
import sys
|
||||
|
||||
|
||||
def diagnose(data: "_IncomingMarkup") -> None:
|
||||
"""Diagnostic suite for isolating common problems.
|
||||
|
||||
:param data: Some markup that needs to be explained.
|
||||
:return: None; diagnostics are printed to standard output.
|
||||
"""
|
||||
print(("Diagnostic running on Beautiful Soup %s" % __version__))
|
||||
print(("Python version %s" % sys.version))
|
||||
|
||||
basic_parsers = ["html.parser", "html5lib", "lxml"]
|
||||
for name in basic_parsers:
|
||||
for builder in builder_registry.builders:
|
||||
if name in builder.features:
|
||||
break
|
||||
else:
|
||||
basic_parsers.remove(name)
|
||||
print(
|
||||
("I noticed that %s is not installed. Installing it may help." % name)
|
||||
)
|
||||
|
||||
if "lxml" in basic_parsers:
|
||||
basic_parsers.append("lxml-xml")
|
||||
try:
|
||||
from lxml import etree # type:ignore
|
||||
|
||||
print(("Found lxml version %s" % ".".join(map(str, etree.LXML_VERSION))))
|
||||
except ImportError:
|
||||
print("lxml is not installed or couldn't be imported.")
|
||||
|
||||
if "html5lib" in basic_parsers:
|
||||
try:
|
||||
import html5lib
|
||||
|
||||
print(("Found html5lib version %s" % html5lib.__version__))
|
||||
except ImportError:
|
||||
print("html5lib is not installed or couldn't be imported.")
|
||||
|
||||
if hasattr(data, "read"):
|
||||
data = data.read()
|
||||
|
||||
for parser in basic_parsers:
|
||||
print(("Trying to parse your markup with %s" % parser))
|
||||
success = False
|
||||
try:
|
||||
soup = BeautifulSoup(data, features=parser)
|
||||
success = True
|
||||
except Exception:
|
||||
print(("%s could not parse the markup." % parser))
|
||||
traceback.print_exc()
|
||||
if success:
|
||||
print(("Here's what %s did with the markup:" % parser))
|
||||
print((soup.prettify()))
|
||||
|
||||
print(("-" * 80))
|
||||
|
||||
|
||||
def lxml_trace(data: "_IncomingMarkup", html: bool = True, **kwargs: Any) -> None:
|
||||
"""Print out the lxml events that occur during parsing.
|
||||
|
||||
This lets you see how lxml parses a document when no Beautiful
|
||||
Soup code is running. You can use this to determine whether
|
||||
an lxml-specific problem is in Beautiful Soup's lxml tree builders
|
||||
or in lxml itself.
|
||||
|
||||
:param data: Some markup.
|
||||
:param html: If True, markup will be parsed with lxml's HTML parser.
|
||||
if False, lxml's XML parser will be used.
|
||||
"""
|
||||
from lxml import etree
|
||||
|
||||
recover = kwargs.pop("recover", True)
|
||||
if isinstance(data, str):
|
||||
data = data.encode("utf8")
|
||||
if not isinstance(data, IO):
|
||||
reader = BytesIO(data)
|
||||
for event, element in etree.iterparse(reader, html=html, recover=recover, **kwargs):
|
||||
print(("%s, %4s, %s" % (event, element.tag, element.text)))
|
||||
|
||||
|
||||
class AnnouncingParser(HTMLParser):
|
||||
"""Subclass of HTMLParser that announces parse events, without doing
|
||||
anything else.
|
||||
|
||||
You can use this to get a picture of how html.parser sees a given
|
||||
document. The easiest way to do this is to call `htmlparser_trace`.
|
||||
"""
|
||||
|
||||
def _p(self, s: str) -> None:
|
||||
print(s)
|
||||
|
||||
def handle_starttag(
|
||||
self,
|
||||
name: str,
|
||||
attrs: List[Tuple[str, Optional[str]]],
|
||||
handle_empty_element: bool = True,
|
||||
) -> None:
|
||||
self._p(f"{name} {attrs} START")
|
||||
|
||||
def handle_endtag(self, name: str, check_already_closed: bool = True) -> None:
|
||||
self._p("%s END" % name)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self._p("%s DATA" % data)
|
||||
|
||||
def handle_charref(self, name: str) -> None:
|
||||
self._p("%s CHARREF" % name)
|
||||
|
||||
def handle_entityref(self, name: str) -> None:
|
||||
self._p("%s ENTITYREF" % name)
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
self._p("%s COMMENT" % data)
|
||||
|
||||
def handle_decl(self, data: str) -> None:
|
||||
self._p("%s DECL" % data)
|
||||
|
||||
def unknown_decl(self, data: str) -> None:
|
||||
self._p("%s UNKNOWN-DECL" % data)
|
||||
|
||||
def handle_pi(self, data: str) -> None:
|
||||
self._p("%s PI" % data)
|
||||
|
||||
|
||||
def htmlparser_trace(data: str) -> None:
|
||||
"""Print out the HTMLParser events that occur during parsing.
|
||||
|
||||
This lets you see how HTMLParser parses a document when no
|
||||
Beautiful Soup code is running.
|
||||
|
||||
:param data: Some markup.
|
||||
"""
|
||||
parser = AnnouncingParser()
|
||||
parser.feed(data)
|
||||
|
||||
|
||||
_vowels: str = "aeiou"
|
||||
_consonants: str = "bcdfghjklmnpqrstvwxyz"
|
||||
|
||||
|
||||
def rword(length: int = 5) -> str:
|
||||
"""Generate a random word-like string.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
s = ""
|
||||
for i in range(length):
|
||||
if i % 2 == 0:
|
||||
t = _consonants
|
||||
else:
|
||||
t = _vowels
|
||||
s += random.choice(t)
|
||||
return s
|
||||
|
||||
|
||||
def rsentence(length: int = 4) -> str:
|
||||
"""Generate a random sentence-like string.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
return " ".join(rword(random.randint(4, 9)) for i in range(length))
|
||||
|
||||
|
||||
def rdoc(num_elements: int = 1000) -> str:
|
||||
"""Randomly generate an invalid HTML document.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
tag_names = ["p", "div", "span", "i", "b", "script", "table"]
|
||||
elements = []
|
||||
for i in range(num_elements):
|
||||
choice = random.randint(0, 3)
|
||||
if choice == 0:
|
||||
# New tag.
|
||||
tag_name = random.choice(tag_names)
|
||||
elements.append("<%s>" % tag_name)
|
||||
elif choice == 1:
|
||||
elements.append(rsentence(random.randint(1, 4)))
|
||||
elif choice == 2:
|
||||
# Close a tag.
|
||||
tag_name = random.choice(tag_names)
|
||||
elements.append("</%s>" % tag_name)
|
||||
return "<html>" + "\n".join(elements) + "</html>"
|
||||
|
||||
|
||||
def benchmark_parsers(num_elements: int = 100000) -> None:
|
||||
"""Very basic head-to-head performance benchmark."""
|
||||
print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
|
||||
data = rdoc(num_elements)
|
||||
print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
|
||||
|
||||
for parser_name in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
|
||||
success = False
|
||||
try:
|
||||
a = time.time()
|
||||
BeautifulSoup(data, parser_name)
|
||||
b = time.time()
|
||||
success = True
|
||||
except Exception:
|
||||
print(("%s could not parse the markup." % parser_name))
|
||||
traceback.print_exc()
|
||||
if success:
|
||||
print(("BS4+%s parsed the markup in %.2fs." % (parser_name, b - a)))
|
||||
|
||||
from lxml import etree
|
||||
|
||||
a = time.time()
|
||||
etree.HTML(data)
|
||||
b = time.time()
|
||||
print(("Raw lxml parsed the markup in %.2fs." % (b - a)))
|
||||
|
||||
import html5lib
|
||||
|
||||
parser = html5lib.HTMLParser()
|
||||
a = time.time()
|
||||
parser.parse(data)
|
||||
b = time.time()
|
||||
print(("Raw html5lib parsed the markup in %.2fs." % (b - a)))
|
||||
|
||||
|
||||
def profile(num_elements: int = 100000, parser: str = "lxml") -> None:
|
||||
"""Use Python's profiler on a randomly generated document."""
|
||||
filehandle = tempfile.NamedTemporaryFile()
|
||||
filename = filehandle.name
|
||||
|
||||
data = rdoc(num_elements)
|
||||
vars = dict(bs4=bs4, data=data, parser=parser)
|
||||
cProfile.runctx("bs4.BeautifulSoup(data, parser)", vars, vars, filename)
|
||||
|
||||
stats = pstats.Stats(filename)
|
||||
# stats.strip_dirs()
|
||||
stats.sort_stats("cumulative")
|
||||
stats.print_stats("_html5lib|bs4", 50)
|
||||
|
||||
|
||||
# If this file is run as a script, standard input is diagnosed.
|
||||
if __name__ == "__main__":
|
||||
diagnose(sys.stdin.read())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
"""Exceptions defined by Beautiful Soup itself."""
|
||||
|
||||
from typing import Union
|
||||
|
||||
|
||||
class StopParsing(Exception):
|
||||
"""Exception raised by a TreeBuilder if it's unable to continue parsing."""
|
||||
|
||||
|
||||
class FeatureNotFound(ValueError):
|
||||
"""Exception raised by the BeautifulSoup constructor if no parser with the
|
||||
requested features is found.
|
||||
"""
|
||||
|
||||
|
||||
class ParserRejectedMarkup(Exception):
|
||||
"""An Exception to be raised when the underlying parser simply
|
||||
refuses to parse the given markup.
|
||||
"""
|
||||
|
||||
def __init__(self, message_or_exception: Union[str, Exception]):
|
||||
"""Explain why the parser rejected the given markup, either
|
||||
with a textual explanation or another exception.
|
||||
"""
|
||||
if isinstance(message_or_exception, Exception):
|
||||
e = message_or_exception
|
||||
message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
|
||||
super(ParserRejectedMarkup, self).__init__(message_or_exception)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user