Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F120694033
poolmanager.py
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Sun, Jul 6, 09:15
Size
9 KB
Mime Type
text/x-python
Expires
Tue, Jul 8, 09:15 (1 d, 23 h)
Engine
blob
Format
Raw Data
Handle
27204540
Attached To
R3852 EMS for Smart-Building
poolmanager.py
View Options
import
logging
try
:
# Python 3
from
urllib.parse
import
urljoin
except
ImportError
:
from
urlparse
import
urljoin
from
._collections
import
RecentlyUsedContainer
from
.connectionpool
import
HTTPConnectionPool
,
HTTPSConnectionPool
from
.connectionpool
import
port_by_scheme
from
.exceptions
import
LocationValueError
,
MaxRetryError
,
ProxySchemeUnknown
from
.request
import
RequestMethods
from
.util.url
import
parse_url
from
.util.retry
import
Retry
__all__
=
[
'PoolManager'
,
'ProxyManager'
,
'proxy_from_url'
]
pool_classes_by_scheme
=
{
'http'
:
HTTPConnectionPool
,
'https'
:
HTTPSConnectionPool
,
}
log
=
logging
.
getLogger
(
__name__
)
SSL_KEYWORDS
=
(
'key_file'
,
'cert_file'
,
'cert_reqs'
,
'ca_certs'
,
'ssl_version'
)
class
PoolManager
(
RequestMethods
):
"""
Allows for arbitrary requests while transparently keeping track of
necessary connection pools for you.
:param num_pools:
Number of connection pools to cache before discarding the least
recently used pool.
:param headers:
Headers to include with all requests, unless other headers are given
explicitly.
:param \**connection_pool_kw:
Additional parameters are used to create fresh
:class:`urllib3.connectionpool.ConnectionPool` instances.
Example::
>>> manager = PoolManager(num_pools=2)
>>> r = manager.request('GET', 'http://google.com/')
>>> r = manager.request('GET', 'http://google.com/mail')
>>> r = manager.request('GET', 'http://yahoo.com/')
>>> len(manager.pools)
2
"""
proxy
=
None
def
__init__
(
self
,
num_pools
=
10
,
headers
=
None
,
**
connection_pool_kw
):
RequestMethods
.
__init__
(
self
,
headers
)
self
.
connection_pool_kw
=
connection_pool_kw
self
.
pools
=
RecentlyUsedContainer
(
num_pools
,
dispose_func
=
lambda
p
:
p
.
close
())
def
__enter__
(
self
):
return
self
def
__exit__
(
self
,
exc_type
,
exc_val
,
exc_tb
):
self
.
clear
()
# Return False to re-raise any potential exceptions
return
False
def
_new_pool
(
self
,
scheme
,
host
,
port
):
"""
Create a new :class:`ConnectionPool` based on host, port and scheme.
This method is used to actually create the connection pools handed out
by :meth:`connection_from_url` and companion methods. It is intended
to be overridden for customization.
"""
pool_cls
=
pool_classes_by_scheme
[
scheme
]
kwargs
=
self
.
connection_pool_kw
if
scheme
==
'http'
:
kwargs
=
self
.
connection_pool_kw
.
copy
()
for
kw
in
SSL_KEYWORDS
:
kwargs
.
pop
(
kw
,
None
)
return
pool_cls
(
host
,
port
,
**
kwargs
)
def
clear
(
self
):
"""
Empty our store of pools and direct them all to close.
This will not affect in-flight connections, but they will not be
re-used after completion.
"""
self
.
pools
.
clear
()
def
connection_from_host
(
self
,
host
,
port
=
None
,
scheme
=
'http'
):
"""
Get a :class:`ConnectionPool` based on the host, port, and scheme.
If ``port`` isn't given, it will be derived from the ``scheme`` using
``urllib3.connectionpool.port_by_scheme``.
"""
if
not
host
:
raise
LocationValueError
(
"No host specified."
)
scheme
=
scheme
or
'http'
port
=
port
or
port_by_scheme
.
get
(
scheme
,
80
)
pool_key
=
(
scheme
,
host
,
port
)
with
self
.
pools
.
lock
:
# If the scheme, host, or port doesn't match existing open
# connections, open a new ConnectionPool.
pool
=
self
.
pools
.
get
(
pool_key
)
if
pool
:
return
pool
# Make a fresh ConnectionPool of the desired type
pool
=
self
.
_new_pool
(
scheme
,
host
,
port
)
self
.
pools
[
pool_key
]
=
pool
return
pool
def
connection_from_url
(
self
,
url
):
"""
Similar to :func:`urllib3.connectionpool.connection_from_url` but
doesn't pass any additional parameters to the
:class:`urllib3.connectionpool.ConnectionPool` constructor.
Additional parameters are taken from the :class:`.PoolManager`
constructor.
"""
u
=
parse_url
(
url
)
return
self
.
connection_from_host
(
u
.
host
,
port
=
u
.
port
,
scheme
=
u
.
scheme
)
def
urlopen
(
self
,
method
,
url
,
redirect
=
True
,
**
kw
):
"""
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
u
=
parse_url
(
url
)
conn
=
self
.
connection_from_host
(
u
.
host
,
port
=
u
.
port
,
scheme
=
u
.
scheme
)
kw
[
'assert_same_host'
]
=
False
kw
[
'redirect'
]
=
False
if
'headers'
not
in
kw
:
kw
[
'headers'
]
=
self
.
headers
if
self
.
proxy
is
not
None
and
u
.
scheme
==
"http"
:
response
=
conn
.
urlopen
(
method
,
url
,
**
kw
)
else
:
response
=
conn
.
urlopen
(
method
,
u
.
request_uri
,
**
kw
)
redirect_location
=
redirect
and
response
.
get_redirect_location
()
if
not
redirect_location
:
return
response
# Support relative URLs for redirecting.
redirect_location
=
urljoin
(
url
,
redirect_location
)
# RFC 7231, Section 6.4.4
if
response
.
status
==
303
:
method
=
'GET'
retries
=
kw
.
get
(
'retries'
)
if
not
isinstance
(
retries
,
Retry
):
retries
=
Retry
.
from_int
(
retries
,
redirect
=
redirect
)
try
:
retries
=
retries
.
increment
(
method
,
url
,
response
=
response
,
_pool
=
conn
)
except
MaxRetryError
:
if
retries
.
raise_on_redirect
:
raise
return
response
kw
[
'retries'
]
=
retries
kw
[
'redirect'
]
=
redirect
log
.
info
(
"Redirecting
%s
->
%s
"
%
(
url
,
redirect_location
))
return
self
.
urlopen
(
method
,
redirect_location
,
**
kw
)
class
ProxyManager
(
PoolManager
):
"""
Behaves just like :class:`PoolManager`, but sends all requests through
the defined proxy, using the CONNECT method for HTTPS URLs.
:param proxy_url:
The URL of the proxy to be used.
:param proxy_headers:
A dictionary contaning headers that will be sent to the proxy. In case
of HTTP they are being sent with each request, while in the
HTTPS/CONNECT case they are sent only once. Could be used for proxy
authentication.
Example:
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
>>> r1 = proxy.request('GET', 'http://google.com/')
>>> r2 = proxy.request('GET', 'http://httpbin.org/')
>>> len(proxy.pools)
1
>>> r3 = proxy.request('GET', 'https://httpbin.org/')
>>> r4 = proxy.request('GET', 'https://twitter.com/')
>>> len(proxy.pools)
3
"""
def
__init__
(
self
,
proxy_url
,
num_pools
=
10
,
headers
=
None
,
proxy_headers
=
None
,
**
connection_pool_kw
):
if
isinstance
(
proxy_url
,
HTTPConnectionPool
):
proxy_url
=
'
%s
://
%s
:
%i
'
%
(
proxy_url
.
scheme
,
proxy_url
.
host
,
proxy_url
.
port
)
proxy
=
parse_url
(
proxy_url
)
if
not
proxy
.
port
:
port
=
port_by_scheme
.
get
(
proxy
.
scheme
,
80
)
proxy
=
proxy
.
_replace
(
port
=
port
)
if
proxy
.
scheme
not
in
(
"http"
,
"https"
):
raise
ProxySchemeUnknown
(
proxy
.
scheme
)
self
.
proxy
=
proxy
self
.
proxy_headers
=
proxy_headers
or
{}
connection_pool_kw
[
'_proxy'
]
=
self
.
proxy
connection_pool_kw
[
'_proxy_headers'
]
=
self
.
proxy_headers
super
(
ProxyManager
,
self
)
.
__init__
(
num_pools
,
headers
,
**
connection_pool_kw
)
def
connection_from_host
(
self
,
host
,
port
=
None
,
scheme
=
'http'
):
if
scheme
==
"https"
:
return
super
(
ProxyManager
,
self
)
.
connection_from_host
(
host
,
port
,
scheme
)
return
super
(
ProxyManager
,
self
)
.
connection_from_host
(
self
.
proxy
.
host
,
self
.
proxy
.
port
,
self
.
proxy
.
scheme
)
def
_set_proxy_headers
(
self
,
url
,
headers
=
None
):
"""
Sets headers needed by proxies: specifically, the Accept and Host
headers. Only sets headers not provided by the user.
"""
headers_
=
{
'Accept'
:
'*/*'
}
netloc
=
parse_url
(
url
)
.
netloc
if
netloc
:
headers_
[
'Host'
]
=
netloc
if
headers
:
headers_
.
update
(
headers
)
return
headers_
def
urlopen
(
self
,
method
,
url
,
redirect
=
True
,
**
kw
):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u
=
parse_url
(
url
)
if
u
.
scheme
==
"http"
:
# For proxied HTTPS requests, httplib sets the necessary headers
# on the CONNECT to the proxy. For HTTP, we'll definitely
# need to set 'Host' at the very least.
headers
=
kw
.
get
(
'headers'
,
self
.
headers
)
kw
[
'headers'
]
=
self
.
_set_proxy_headers
(
url
,
headers
)
return
super
(
ProxyManager
,
self
)
.
urlopen
(
method
,
url
,
redirect
=
redirect
,
**
kw
)
def
proxy_from_url
(
url
,
**
kw
):
return
ProxyManager
(
proxy_url
=
url
,
**
kw
)
Event Timeline
Log In to Comment