def __str__(self):
    return '<urlopen error %s>' % self.reason

def __init__(self, url, code, msg, hdrs, fp):
    self.code = code
    self.msg = msg
    self.hdrs = hdrs
    self.fp = fp
    self.filename = url
    # The addinfourl classes depend on fp being a valid file
    # object.  In some cases, the HTTPError may not have a valid
    # file object.  If this happens, the simplest workaround is to
    # not initialize the base classes.
    if fp is not None:
        self.__super_init(fp, hdrs, url, code)

def __str__(self):
    return 'HTTP Error %s: %s' % (self.code, self.msg)

# since URLError specifies a .reason attribute, HTTPError should also
#  provide this attribute. See issue13211 fo discussion.
@property
def reason(self):
    return self.msg

# remove port, if present
host = _cut_port_re.sub("", host, 1)
return host.lower()

def __init__(self, url, data=None, headers={},
             origin_req_host=None, unverifiable=False):
    # unwrap('<URL:type:https://host/path>') --> 'type:https://host/path'
    self.__original = unwrap(url)
    self.__original, self.__fragment = splittag(self.__original)
    self.type = None
    # self.__r_type is what's left after doing the splittype
    self.host = None
    self.port = None
    self._tunnel_host = None
    self.data = data
    self.headers = {}
    for key, value in headers.items():
        self.add_header(key, value)
    self.unredirected_hdrs = {}
    if origin_req_host is None:
        origin_req_host = request_host(self)
    self.origin_req_host = origin_req_host
    self.unverifiable = unverifiable

def __getattr__(self, attr):
    # XXX this is a fallback mechanism to guard against these
    # methods getting called in a non-standard order.  this may be
    # too complicated and/or unnecessary.
    # XXX should the __r_XXX attributes be public?
    if attr[:12] == '_Request__r_':
        name = attr[12:]
        if hasattr(Request, 'get_' + name):
            getattr(self, 'get_' + name)()
            return getattr(self, attr)
    raise AttributeError, attr

def get_method(self):
    if self.has_data():
        return "POST"
    else:
        return "GET"

# XXX these helper methods are lame

def add_data(self, data):
    self.data = data

def has_data(self):
    return self.data is not None

def get_data(self):
    return self.data

def get_full_url(self):
    if self.__fragment:
        return '%s#%s' % (self.__original, self.__fragment)
    else:
        return self.__original

def get_type(self):
    if self.type is None:
        self.type, self.__r_type = splittype(self.__original)
        if self.type is None:
            raise ValueError, "unknown url type: %s" % self.__original
    return self.type

def get_host(self):
    if self.host is None:
        self.host, self.__r_host = splithost(self.__r_type)
        if self.host:
            self.host = unquote(self.host)
    return self.host

def get_selector(self):
    return self.__r_host

def set_proxy(self, host, type):
    if self.type == 'https' and not self._tunnel_host:
        self._tunnel_host = self.host
    else:
        self.type = type
        self.__r_host = self.__original

    self.host = host

def has_proxy(self):
    return self.__r_host == self.__original

def get_origin_req_host(self):
    return self.origin_req_host

def is_unverifiable(self):
    return self.unverifiable

def add_header(self, key, val):
    # useful for something like authentication
    self.headers[key.capitalize()] = val

def add_unredirected_header(self, key, val):
    # will not be added to a redirected request
    self.unredirected_hdrs[key.capitalize()] = val

def has_header(self, header_name):
    return (header_name in self.headers or
            header_name in self.unredirected_hdrs)

def get_header(self, header_name, default=None):
    return self.headers.get(
        header_name,
        self.unredirected_hdrs.get(header_name, default))

def header_items(self):
    hdrs = self.unredirected_hdrs.copy()
    hdrs.update(self.headers)
    return hdrs.items()

def add_handler(self, handler):
    if not hasattr(handler, "add_parent"):
        raise TypeError("expected BaseHandler instance, got %r" %
                        type(handler))

    added = False
    for meth in dir(handler):
        if meth in ["redirect_request", "do_open", "proxy_open"]:
            # oops, coincidental match
            continue

        i = meth.find("_")
        protocol = meth[:i]
        condition = meth[i+1:]

        if condition.startswith("error"):
            j = condition.find("_") + i + 1
            kind = meth[j+1:]
            try:
                kind = int(kind)
            except ValueError:
                pass
            lookup = self.handle_error.get(protocol, {})
            self.handle_error[protocol] = lookup
        elif condition == "open":
            kind = protocol
            lookup = self.handle_open
        elif condition == "response":
            kind = protocol
            lookup = self.process_response
        elif condition == "request":
            kind = protocol
            lookup = self.process_request
        else:
            continue

        handlers = lookup.setdefault(kind, [])
        if handlers:
            bisect.insort(handlers, handler)
        else:
            handlers.append(handler)
        added = True

    if added:
        bisect.insort(self.handlers, handler)
        handler.add_parent(self)

def close(self):
    # Only exists for backwards compatibility.
    pass

def _call_chain(self, chain, kind, meth_name, *args):
    # Handlers raise an exception if no one else should try to handle
    # the request, or return None if they can't but another handler
    # could.  Otherwise, they return the response.
    handlers = chain.get(kind, ())
    for handler in handlers:
        func = getattr(handler, meth_name)

        result = func(*args)
        if result is not None:
            return result

def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
    # accept a URL or a Request object
    if isinstance(fullurl, basestring):
        req = Request(fullurl, data)
    else:
        req = fullurl
        if data is not None:
            req.add_data(data)

    req.timeout = timeout
    protocol = req.get_type()

    # pre-process request
    meth_name = protocol+"_request"
    for processor in self.process_request.get(protocol, []):
        meth = getattr(processor, meth_name)
        req = meth(req)

    response = self._open(req, data)

    # post-process response
    meth_name = protocol+"_response"
    for processor in self.process_response.get(protocol, []):
        meth = getattr(processor, meth_name)
        response = meth(req, response)

    return response

def _open(self, req, data=None):
    result = self._call_chain(self.handle_open, 'default',
                              'default_open', req)
    if result:
        return result

    protocol = req.get_type()
    result = self._call_chain(self.handle_open, protocol, protocol +
                              '_open', req)
    if result:
        return result

    return self._call_chain(self.handle_open, 'unknown',
                            'unknown_open', req)

def error(self, proto, *args):
    if proto in ('http', 'https'):
        # XXX http[s] protocols are special-cased
        dict = self.handle_error['http'] # https is not different than http
        proto = args[2]  # YUCK!
        meth_name = 'http_error_%s' % proto
        http_err = 1
        orig_args = args
    else:
        dict = self.handle_error
        meth_name = proto + '_error'
        http_err = 0
    args = (dict, proto, meth_name) + args
    result = self._call_chain(*args)
    if result:
        return result

    if http_err:
        args = (dict, 'default', 'http_error_default') + orig_args
        return self._call_chain(*args)

opener = OpenerDirector()
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                   HTTPDefaultErrorHandler, HTTPRedirectHandler,
                   FTPHandler, FileHandler, HTTPErrorProcessor]
if hasattr(httplib, 'HTTPS'):
    default_classes.append(HTTPSHandler)
skip = set()
for klass in default_classes:
    for check in handlers:
        if isclass(check):
            if issubclass(check, klass):
                skip.add(klass)
        elif isinstance(check, klass):
            skip.add(klass)
for klass in skip:
    default_classes.remove(klass)

for klass in default_classes:
    opener.add_handler(klass())

for h in handlers:
    if isclass(h):
        h = h()
    opener.add_handler(h)
return opener

def add_parent(self, parent):
    self.parent = parent

def close(self):
    # Only exists for backwards compatibility
    pass

def __lt__(self, other):
    if not hasattr(other, "handler_order"):
        # Try to preserve the old behavior of having custom classes
        # inserted after default ones (works only for custom user
        # classes which are not aware of handler_order).
        return True
    return self.handler_order < other.handler_order

def http_response(self, request, response):
    code, msg, hdrs = response.code, response.msg, response.info()

    # According to RFC 2616, "2xx" code indicates that the client's
    # request was successfully received, understood, and accepted.
    if not (200 <= code < 300):
        response = self.parent.error(
            'http', request, response, code, msg, hdrs)

    return response

https_response = http_response

def redirect_request(self, req, fp, code, msg, headers, newurl):
    """Return a Request or None in response to a redirect.
    This is called by the http_error_30x methods when a
    redirection response is received.  If a redirection should
    take place, return a new Request to allow http_error_30x to
    perform the redirect.  Otherwise, raise HTTPError if no-one
    else should try to handle this url.  Return None if you can't
    but another Handler might.
    """
    m = req.get_method()
    if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
        or code in (301, 302, 303) and m == "POST"):
        # Strictly (according to RFC 2616), 301 or 302 in response
        # to a POST MUST NOT cause a redirection without confirmation
        # from the user (of urllib2, in this case).  In practice,
        # essentially all clients do redirect in this case, so we
        # do the same.
        # be conciliant with URIs containing a space
        newurl = newurl.replace(' ', '%20')
        newheaders = dict((k,v) for k,v in req.headers.items()
                          if k.lower() not in ("content-length", "content-type")
                         )
        return Request(newurl,
                       headers=newheaders,
                       origin_req_host=req.get_origin_req_host(),
                       unverifiable=True)
    else:
        raise HTTPError(req.get_full_url(), code, msg, headers, fp)

# Implementation note: To avoid the server sending us into an
# infinite loop, the request object needs to track what URLs we
# have already seen.  Do this by adding a handler-specific
# attribute to the Request object.
def http_error_302(self, req, fp, code, msg, headers):
    # Some servers (incorrectly) return multiple Location headers
    # (so probably same goes for URI).  Use first header.
    if 'location' in headers:
        newurl = headers.getheaders('location')[0]
    elif 'uri' in headers:
        newurl = headers.getheaders('uri')[0]
    else:
        return

    # fix a possible malformed URL
    urlparts = urlparse.urlparse(newurl)
    if not urlparts.path:
        urlparts = list(urlparts)
        urlparts[2] = "/"
    newurl = urlparse.urlunparse(urlparts)

    newurl = urlparse.urljoin(req.get_full_url(), newurl)

    # For security reasons we do not allow redirects to protocols
    # other than HTTP, HTTPS or FTP.
    newurl_lower = newurl.lower()
    if not (newurl_lower.startswith('http:https://') or
            newurl_lower.startswith('https://') or
            newurl_lower.startswith('ftp:https://')):
        raise HTTPError(newurl, code,
                        msg + " - Redirection to url '%s' is not allowed" %
                        newurl,
                        headers, fp)

    # XXX Probably want to forget about the state of the current
    # request, although that might interact poorly with other
    # handlers that also use handler-specific request attributes
    new = self.redirect_request(req, fp, code, msg, headers, newurl)
    if new is None:
        return

    # loop detection
    # .redirect_dict has a key url if url was previously visited.
    if hasattr(req, 'redirect_dict'):
        visited = new.redirect_dict = req.redirect_dict
        if (visited.get(newurl, 0) >= self.max_repeats or
            len(visited) >= self.max_redirections):
            raise HTTPError(req.get_full_url(), code,
                            self.inf_msg + msg, headers, fp)
    else:
        visited = new.redirect_dict = req.redirect_dict = {}
    visited[newurl] = visited.get(newurl, 0) + 1

    # Don't close the fp until we are sure that we won't use it
    # with HTTPError.
    fp.read()
    fp.close()

    return self.parent.open(new, timeout=req.timeout)

http_error_301 = http_error_303 = http_error_307 = http_error_302

inf_msg = "The HTTP server returned a redirect error that would " \
          "lead to an infinite loop.\n" \
          "The last 30x error message was:\n"

def __init__(self, proxies=None):
    if proxies is None:
        proxies = getproxies()
    assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
    self.proxies = proxies
    for type, url in proxies.items():
        setattr(self, '%s_open' % type,
                lambda r, proxy=url, type=type, meth=self.proxy_open: \
                meth(r, proxy, type))

def proxy_open(self, req, proxy, type):
    orig_type = req.get_type()
    proxy_type, user, password, hostport = _parse_proxy(proxy)

    if proxy_type is None:
        proxy_type = orig_type

    if req.host and proxy_bypass(req.host):
        return None

    if user and password:
        user_pass = '%s:%s' % (unquote(user), unquote(password))
        creds = base64.b64encode(user_pass).strip()
        req.add_header('Proxy-authorization', 'Basic ' + creds)
    hostport = unquote(hostport)
    req.set_proxy(hostport, proxy_type)

    if orig_type == proxy_type or orig_type == 'https':
        # let other handlers take care of it
        return None
    else:
        # need to start over, because the other handlers don't
        # grok the proxy's URL type
        # e.g. if we have a constructor arg proxies like so:
        # {'http': 'ftp:https://proxy.example.com'}, we may end up turning
        # a request for http:https://acme.example.com/a into one for
        # ftp:https://proxy.example.com/a
        return self.parent.open(req, timeout=req.timeout)

def __init__(self):
    self.passwd = {}

def add_password(self, realm, uri, user, passwd):
    # uri could be a single URI or a sequence
    if isinstance(uri, basestring):
        uri = [uri]
    if not realm in self.passwd:
        self.passwd[realm] = {}
    for default_port in True, False:
        reduced_uri = tuple(
            [self.reduce_uri(u, default_port) for u in uri])
        self.passwd[realm][reduced_uri] = (user, passwd)

def find_user_password(self, realm, authuri):
    domains = self.passwd.get(realm, {})
    for default_port in True, False:
        reduced_authuri = self.reduce_uri(authuri, default_port)
        for uris, authinfo in domains.iteritems():
            for uri in uris:
                if self.is_suburi(uri, reduced_authuri):
                    return authinfo
    return None, None

def reduce_uri(self, uri, default_port=True):
    """Accept authority or URI and extract only the authority and path."""
    # note HTTP URLs do not have a userinfo component
    parts = urlparse.urlsplit(uri)
    if parts[1]:
        # URI
        scheme = parts[0]
        authority = parts[1]
        path = parts[2] or '/'
    else:
        # host or host:port
        scheme = None
        authority = uri
        path = '/'
    host, port = splitport(authority)
    if default_port and port is None and scheme is not None:
        dport = {"http": 80,
                 "https": 443,
                 }.get(scheme)
        if dport is not None:
            authority = "%s:%d" % (host, dport)
    return authority, path

def is_suburi(self, base, test):
    """Check if test is below base in a URI tree
    Both args must be URIs in reduced form.
    """
    if base == test:
        return True
    if base[0] != test[0]:
        return False
    common = posixpath.commonprefix((base[1], test[1]))
    if len(common) == len(base[1]):
        return True
    return False

def find_user_password(self, realm, authuri):
    user, password = HTTPPasswordMgr.find_user_password(self, realm,
                                                        authuri)
    if user is not None:
        return user, password
    return HTTPPasswordMgr.find_user_password(self, None, authuri)

# XXX this allows for multiple auth-schemes, but will stupidly pick
# the last one with a realm specified.

# allow for double- and single-quoted realm values
# (single quotes are a violation of the RFC, but appear in the wild)
rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
                'realm=(["\'])(.*?)\\2', re.I)

# XXX could pre-emptively send auth info already accepted (RFC 2617,
# end of section 2, and section 1.2 immediately after "credentials"
# production).

def __init__(self, password_mgr=None):
    if password_mgr is None:
        password_mgr = HTTPPasswordMgr()
    self.passwd = password_mgr
    self.add_password = self.passwd.add_password
    self.retried = 0

def reset_retry_count(self):
    self.retried = 0

def http_error_auth_reqed(self, authreq, host, req, headers):
    # host may be an authority (without userinfo) or a URL with an
    # authority
    # XXX could be multiple headers
    authreq = headers.get(authreq, None)

    if self.retried > 5:
        # retry sending the username:password 5 times before failing.
        raise HTTPError(req.get_full_url(), 401, "basic auth failed",
                        headers, None)
    else:
        self.retried += 1

    if authreq:
        mo = AbstractBasicAuthHandler.rx.search(authreq)
        if mo:
            scheme, quote, realm = mo.groups()
            if scheme.lower() == 'basic':
                response = self.retry_http_basic_auth(host, req, realm)
                if response and response.code != 401:
                    self.retried = 0
                return response

def retry_http_basic_auth(self, host, req, realm):
    user, pw = self.passwd.find_user_password(realm, host)
    if pw is not None:
        raw = "%s:%s" % (user, pw)
        auth = 'Basic %s' % base64.b64encode(raw).strip()
        if req.headers.get(self.auth_header, None) == auth:
            return None
        req.add_unredirected_header(self.auth_header, auth)
        return self.parent.open(req, timeout=req.timeout)
    else:
        return None

auth_header = 'Authorization'

def http_error_401(self, req, fp, code, msg, headers):
    url = req.get_full_url()
    response = self.http_error_auth_reqed('www-authenticate',
                                          url, req, headers)
    self.reset_retry_count()
    return response

auth_header = 'Proxy-authorization'

def http_error_407(self, req, fp, code, msg, headers):
    # http_error_auth_reqed requires that there is no userinfo component in
    # authority.  Assume there isn't one, since urllib2 does not (and
    # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
    # userinfo.
    authority = req.get_host()
    response = self.http_error_auth_reqed('proxy-authenticate',
                                      authority, req, headers)
    self.reset_retry_count()
    return response

# XXX The client does not inspect the Authentication-Info header
# in a successful response.

# XXX It should be possible to test this implementation against
# a mock server that just generates a static set of challenges.

# XXX qop="auth-int" supports is shaky

def __init__(self, passwd=None):
    if passwd is None:
        passwd = HTTPPasswordMgr()
    self.passwd = passwd
    self.add_password = self.passwd.add_password
    self.retried = 0
    self.nonce_count = 0
    self.last_nonce = None

def reset_retry_count(self):
    self.retried = 0

def http_error_auth_reqed(self, auth_header, host, req, headers):
    authreq = headers.get(auth_header, None)
    if self.retried > 5:
        # Don't fail endlessly - if we failed once, we'll probably
        # fail a second time. Hm. Unless the Password Manager is
        # prompting for the information. Crap. This isn't great
        # but it's better than the current 'repeat until recursion
        # depth exceeded' approach <wink>
        raise HTTPError(req.get_full_url(), 401, "digest auth failed",
                        headers, None)
    else:
        self.retried += 1
    if authreq:
        scheme = authreq.split()[0]
        if scheme.lower() == 'digest':
            return self.retry_http_digest_auth(req, authreq)

def retry_http_digest_auth(self, req, auth):
    token, challenge = auth.split(' ', 1)
    chal = parse_keqv_list(parse_http_list(challenge))
    auth = self.get_authorization(req, chal)
    if auth:
        auth_val = 'Digest %s' % auth
        if req.headers.get(self.auth_header, None) == auth_val:
            return None
        req.add_unredirected_header(self.auth_header, auth_val)
        resp = self.parent.open(req, timeout=req.timeout)
        return resp

def get_cnonce(self, nonce):
    # The cnonce-value is an opaque
    # quoted string value provided by the client and used by both client
    # and server to avoid chosen plaintext attacks, to provide mutual
    # authentication, and to provide some message integrity protection.
    # This isn't a fabulous effort, but it's probably Good Enough.
    dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
                                        randombytes(8))).hexdigest()
    return dig[:16]

def get_authorization(self, req, chal):
    try:
        realm = chal['realm']
        nonce = chal['nonce']
        qop = chal.get('qop')
        algorithm = chal.get('algorithm', 'MD5')
        # mod_digest doesn't send an opaque, even though it isn't
        # supposed to be optional
        opaque = chal.get('opaque', None)
    except KeyError:
        return None

    H, KD = self.get_algorithm_impls(algorithm)
    if H is None:
        return None

    user, pw = self.passwd.find_user_password(realm, req.get_full_url())
    if user is None:
        return None

    # XXX not implemented yet
    if req.has_data():
        entdig = self.get_entity_digest(req.get_data(), chal)
    else:
        entdig = None

    A1 = "%s:%s:%s" % (user, realm, pw)
    A2 = "%s:%s" % (req.get_method(),
                    # XXX selector: what about proxies and full urls
                    req.get_selector())
    if qop == 'auth':
        if nonce == self.last_nonce:
            self.nonce_count += 1
        else:
            self.nonce_count = 1
            self.last_nonce = nonce

        ncvalue = '%08x' % self.nonce_count
        cnonce = self.get_cnonce(nonce)
        noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
        respdig = KD(H(A1), noncebit)
    elif qop is None:
        respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
    else:
        # XXX handle auth-int.
        raise URLError("qop '%s' is not supported." % qop)

    # XXX should the partial digests be encoded too?

    base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
           'response="%s"' % (user, realm, nonce, req.get_selector(),
                              respdig)
    if opaque:
        base += ', opaque="%s"' % opaque
    if entdig:
        base += ', digest="%s"' % entdig
    base += ', algorithm="%s"' % algorithm
    if qop:
        base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
    return base

def get_algorithm_impls(self, algorithm):
    # algorithm should be case-insensitive according to RFC2617
    algorithm = algorithm.upper()
    # lambdas assume digest modules are imported at the top level
    if algorithm == 'MD5':
        H = lambda x: hashlib.md5(x).hexdigest()
    elif algorithm == 'SHA':
        H = lambda x: hashlib.sha1(x).hexdigest()
    # XXX MD5-sess
    KD = lambda s, d: H("%s:%s" % (s, d))
    return H, KD

def get_entity_digest(self, data, chal):
    # XXX not implemented yet
    return None

auth_header = 'Authorization'
handler_order = 490  # before Basic auth

def http_error_401(self, req, fp, code, msg, headers):
    host = urlparse.urlparse(req.get_full_url())[1]
    retry = self.http_error_auth_reqed('www-authenticate',
                                       host, req, headers)
    self.reset_retry_count()
    return retry

auth_header = 'Proxy-Authorization'
handler_order = 490  # before Basic auth

def http_error_407(self, req, fp, code, msg, headers):
    host = req.get_host()
    retry = self.http_error_auth_reqed('proxy-authenticate',
                                       host, req, headers)
    self.reset_retry_count()
    return retry

def __init__(self, debuglevel=0):
    self._debuglevel = debuglevel

def set_http_debuglevel(self, level):
    self._debuglevel = level

def do_request_(self, request):
    host = request.get_host()
    if not host:
        raise URLError('no host given')

    if request.has_data():  # POST
        data = request.get_data()
        if not request.has_header('Content-type'):
            request.add_unredirected_header(
                'Content-type',
                'application/x-www-form-urlencoded')
        if not request.has_header('Content-length'):
            request.add_unredirected_header(
                'Content-length', '%d' % len(data))

    sel_host = host
    if request.has_proxy():
        scheme, sel = splittype(request.get_selector())
        sel_host, sel_path = splithost(sel)

    if not request.has_header('Host'):
        request.add_unredirected_header('Host', sel_host)
    for name, value in self.parent.addheaders:
        name = name.capitalize()
        if not request.has_header(name):
            request.add_unredirected_header(name, value)

    return request

def do_open(self, http_class, req):
    """Return an addinfourl object for the request, using http_class.
    http_class must implement the HTTPConnection API from httplib.
    The addinfourl return value is a file-like object.  It also
    has methods and attributes including:
        - info(): return a mimetools.Message object for the headers
        - geturl(): return the original request URL
        - code: HTTP status code
    """
    host = req.get_host()
    if not host:
        raise URLError('no host given')

    h = http_class(host, timeout=req.timeout) # will parse host:port
    h.set_debuglevel(self._debuglevel)

    headers = dict(req.unredirected_hdrs)
    headers.update(dict((k, v) for k, v in req.headers.items()
                        if k not in headers))

    # We want to make an HTTP/1.1 request, but the addinfourl
    # class isn't prepared to deal with a persistent connection.
    # It will try to read all remaining data from the socket,
    # which will block while the server waits for the next request.
    # So make sure the connection gets closed after the (only)
    # request.
    headers["Connection"] = "close"
    headers = dict(
        (name.title(), val) for name, val in headers.items())

    if req._tunnel_host:
        tunnel_headers = {}
        proxy_auth_hdr = "Proxy-Authorization"
        if proxy_auth_hdr in headers:
            tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
            # Proxy-Authorization should not be sent to origin
            # server.
            del headers[proxy_auth_hdr]
        h.set_tunnel(req._tunnel_host, headers=tunnel_headers)

    try:
        h.request(req.get_method(), req.get_selector(), req.data, headers)
    except socket.error, err: # XXX what error?
        h.close()
        raise URLError(err)
    else:
        try:
            r = h.getresponse(buffering=True)
        except TypeError: # buffering kw not supported
            r = h.getresponse()

    # Pick apart the HTTPResponse object to get the addinfourl
    # object initialized properly.

    # Wrap the HTTPResponse object in socket's file object adapter
    # for Windows.  That adapter calls recv(), so delegate recv()
    # to read().  This weird wrapping allows the returned object to
    # have readline() and readlines() methods.

    # XXX It might be better to extract the read buffering code
    # out of socket._fileobject() and into a base class.

    r.recv = r.read
    fp = socket._fileobject(r, close=True)

    resp = addinfourl(fp, r.msg, req.get_full_url())
    resp.code = r.status
    resp.msg = r.reason
    return resp

def http_open(self, req):
    return self.do_open(httplib.HTTPConnection, req)

http_request = AbstractHTTPHandler.do_request_

    def https_open(self, req):
        return self.do_open(httplib.HTTPSConnection, req)

    https_request = AbstractHTTPHandler.do_request_

def http_request(self, request):
    self.cookiejar.add_cookie_header(request)
    return request

def http_response(self, request, response):
    self.cookiejar.extract_cookies(response, request)
    return response

https_request = http_request
https_response = http_response

escape = quote = False
for cur in s:
    if escape:
        part += cur
        escape = False
        continue
    if quote:
        if cur == '\\':
            escape = True
            continue
        elif cur == '"':
            quote = False
        part += cur
        continue

    if cur == ',':
        res.append(part)
        part = ''
        continue

    if cur == '"':
        quote = True

    part += cur

# append last part
if part:
    res.append(part)

return [part.strip() for part in res]

# names for the localhost
names = None
def get_names(self):
    if FileHandler.names is None:
        try:
            FileHandler.names = tuple(
                socket.gethostbyname_ex('localhost')[2] +
                socket.gethostbyname_ex(socket.gethostname())[2])
        except socket.gaierror:
            FileHandler.names = (socket.gethostbyname('localhost'),)
    return FileHandler.names

# not entirely sure what the rules are here
def open_local_file(self, req):
    import email.utils
    import mimetypes
    host = req.get_host()
    filename = req.get_selector()
    localfile = url2pathname(filename)
    try:
        stats = os.stat(localfile)
        size = stats.st_size
        modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
        mtype = mimetypes.guess_type(filename)[0]
        headers = mimetools.Message(StringIO(
            'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
            (mtype or 'text/plain', size, modified)))
        if host:
            host, port = splitport(host)
        if not host or \
            (not port and _safe_gethostbyname(host) in self.get_names()):
            if host:
                origurl = 'file:https://' + host + filename
            else:
                origurl = 'file:https://' + filename
            return addinfourl(open(localfile, 'rb'), headers, origurl)
    except OSError, msg:
        # urllib2 users shouldn't expect OSErrors coming from urlopen()
        raise URLError(msg)
    raise URLError('file not on local host')

    # username/password handling
    user, host = splituser(host)
    if user:
        user, passwd = splitpasswd(user)
    else:
        passwd = None
    host = unquote(host)
    user = user or ''
    passwd = passwd or ''

    try:
        host = socket.gethostbyname(host)
    except socket.error, msg:
        raise URLError(msg)
    path, attrs = splitattr(req.get_selector())
    dirs = path.split('/')
    dirs = map(unquote, dirs)
    dirs, file = dirs[:-1], dirs[-1]
    if dirs and not dirs[0]:
        dirs = dirs[1:]
    try:
        fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
        type = file and 'I' or 'D'
        for attr in attrs:
            attr, value = splitvalue(attr)
            if attr.lower() == 'type' and \
               value in ('a', 'A', 'i', 'I', 'd', 'D'):
                type = value.upper()
        fp, retrlen = fw.retrfile(file, type)
        headers = ""
        mtype = mimetypes.guess_type(req.get_full_url())[0]
        if mtype:
            headers += "Content-type: %s\n" % mtype
        if retrlen is not None and retrlen >= 0:
            headers += "Content-length: %d\n" % retrlen
        sf = StringIO(headers)
        headers = mimetools.Message(sf)
        return addinfourl(fp, headers, req.get_full_url())
    except ftplib.all_errors, msg:
        raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]

def connect_ftp(self, user, passwd, host, port, dirs, timeout):
    fw = ftpwrapper(user, passwd, host, port, dirs, timeout,
                    persistent=False)

    return fw

def setTimeout(self, t):
    self.delay = t

def setMaxConns(self, m):
    self.max_conns = m

def connect_ftp(self, user, passwd, host, port, dirs, timeout):
    key = user, host, port, '/'.join(dirs), timeout
    if key in self.cache:
        self.timeout[key] = time.time() + self.delay
    else:
        self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
        self.timeout[key] = time.time() + self.delay
    self.check_cache()
    return self.cache[key]

def check_cache(self):
    # first check for old ones
    t = time.time()
    if self.soonest <= t:
        for k, v in self.timeout.items():
            if v < t:
                self.cache[k].close()
                del self.cache[k]
                del self.timeout[k]
    self.soonest = min(self.timeout.values())

    # then check the size
    if len(self.cache) == self.max_conns:
        for k, v in self.timeout.items():
            if v == self.soonest:
                del self.cache[k]
                del self.timeout[k]
                break
        self.soonest = min(self.timeout.values())

def clear_cache(self):
    for conn in self.cache.values():
        conn.close()
    self.cache.clear()
    self.timeout.clear()