Initial work to support Python 3.3+

This updates most of the code to be forward-compatible with Python 3.3 and
3.4 while still continuing to support 2.6 and 2.7. It **drops** support for
Python 2.5.

Python 3 support is added for common Boto modules (`boto/*.py`) as well as
S3, SQS, Kinesis and CloudTrail. Several other modules may work but have
not been thoroughly tested.

The `tox` configuration has been updated to run tests for all supported
environments, and for now a whitelist is used for Python 3 unit tests.

A new porting guide is included to help community members port other
modules to Python 3, and both the README and Sphinx index list which
modules currently support Python 3.

.travis.yml

@@ -2,9 +2,13 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.3"
+  - "3.4"
 before_install:
   - sudo apt-get update
   - sudo apt-get --reinstall install -qq language-pack-en language-pack-de
   - sudo apt-get install swig
-install: pip install --allow-all-external -r requirements.txt
-script: python tests/test.py unit
+install:
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py26.txt; fi
+  - pip install -r requirements.txt
+script: python tests/test.py default

README.rst

@@ -16,14 +16,24 @@ Introduction
 ************
 
 Boto is a Python package that provides interfaces to Amazon Web Services.
+Currently, all features work with Python 2.6 and 2.7. Work is under way to
+support Python 3.3+ in the same codebase. Modules are being ported one at
+a time with the help of the open source community, so please check below
+for compatibility with Python 3.3+.
+
+To port a module to Python 3.3+, please view our `Contributing Guidelines`_
+and the `Porting Guide`_. If you would like, you can open an issue to let
+others know about your work in progress. Tests **must** pass on Python
+2.6, 2.7, 3.3, and 3.4 for pull requests to be accepted.
+
 At the moment, boto supports:
 
 * Compute
 
   * Amazon Elastic Compute Cloud (EC2)
   * Amazon Elastic Map Reduce (EMR)
   * AutoScaling
-  * Amazon Kinesis
+  * Amazon Kinesis (Python 3)
 
 * Content Delivery
 
@@ -43,7 +53,7 @@ At the moment, boto supports:
   * AWS CloudFormation
   * AWS Data Pipeline
   * AWS Opsworks
-  * AWS CloudTrail
+  * AWS CloudTrail (Python 3)
 
 * Identity & Access
 
@@ -54,7 +64,7 @@ At the moment, boto supports:
   * Amazon CloudSearch
   * Amazon Elastic Transcoder
   * Amazon Simple Workflow Service (SWF)
-  * Amazon Simple Queue Service (SQS)
+  * Amazon Simple Queue Service (SQS) (Python 3)
   * Amazon Simple Notification Server (SNS)
   * Amazon Simple Email Service (SES)
 
@@ -75,7 +85,7 @@ At the moment, boto supports:
 
 * Storage
 
-  * Amazon Simple Storage Service (S3)
+  * Amazon Simple Storage Service (S3) (Python 3)
   * Amazon Glacier
   * Amazon Elastic Block Store (EBS)
   * Google Cloud Storage
@@ -158,6 +168,8 @@ following environment variables to ascertain your credentials:
 Credentials and other boto-related settings can also be stored in a
 boto config file.  See `this`_ for details.
 
+.. _Contributing Guidelines: https://github.com/boto/boto/blob/develop/CONTRIBUTING
+.. _Porting Guide: https://boto.readthedocs.org/en/latest/porting_guide.html
 .. _pip: https://www.pip-installer.org/
 .. _release notes: https://github.com/boto/boto/wiki
 .. _github.com: https://github.com/boto/boto

boto/__init__.py

@@ -34,7 +34,8 @@
 import sys
 import logging
 import logging.config
-import urlparse
+
+from boto.compat import urlparse
 from boto.exception import InvalidUriError
 
 __version__ = '2.29.1'
@@ -492,7 +493,7 @@ def connect_ec2_endpoint(url, aws_access_key_id=None,
     """
     from boto.ec2.regioninfo import RegionInfo
 
-    purl = urlparse.urlparse(url)
+    purl = urlparse(url)
     kwargs['port'] = purl.port
     kwargs['host'] = purl.hostname
     kwargs['path'] = purl.path
@@ -879,7 +880,7 @@ def storage_uri(uri_str, default_scheme='file', debug=0, validate=True,
     version_id = None
     generation = None
 
-    # Manually parse URI components instead of using urlparse.urlparse because
+    # Manually parse URI components instead of using urlparse because
     # what we're calling URIs don't really fit the standard syntax for URIs
     # (the latter includes an optional host/net location part).
     end_scheme_idx = uri_str.find(':https://')

boto/auth.py

@@ -39,10 +39,9 @@
 import os
 import sys
 import time
-import urllib
-import urlparse
 import posixpath
 
+from boto.compat import urllib
 from boto.auth_handler import AuthHandler
 from boto.exception import BotoClientError
 
@@ -65,9 +64,10 @@ def __init__(self, host, config, provider):
 
     def update_provider(self, provider):
         self._provider = provider
-        self._hmac = hmac.new(self._provider.secret_key, digestmod=sha)
+        self._hmac = hmac.new(self._provider.secret_key.encode('utf-8'),
+                              digestmod=sha)
         if sha256:
-            self._hmac_256 = hmac.new(self._provider.secret_key,
+            self._hmac_256 = hmac.new(self._provider.secret_key.encode('utf-8'),
                                       digestmod=sha256)
         else:
             self._hmac_256 = None
@@ -83,13 +83,13 @@ def _get_hmac(self):
             digestmod = sha256
         else:
             digestmod = sha
-        return hmac.new(self._provider.secret_key,
+        return hmac.new(self._provider.secret_key.encode('utf-8'),
                         digestmod=digestmod)
 
     def sign_string(self, string_to_sign):
         new_hmac = self._get_hmac()
-        new_hmac.update(string_to_sign)
-        return base64.encodestring(new_hmac.digest()).strip()
+        new_hmac.update(string_to_sign.encode('utf-8'))
+        return base64.encodestring(new_hmac.digest()).decode('utf-8').strip()
 
     def __getstate__(self):
         pickled_dict = copy.copy(self.__dict__)
@@ -99,7 +99,7 @@ def __getstate__(self):
 
     def __setstate__(self, dct):
         self.__dict__ = dct
-        self.update_provider(self._provider)
+        self.update_provider(self._provider.encode('utf-8'))
 
 
 class AnonAuthHandler(AuthHandler, HmacKeys):
@@ -271,7 +271,7 @@ def add_auth(self, req, **kwargs):
             req.headers['X-Amz-Security-Token'] = self._provider.security_token
         string_to_sign, headers_to_sign = self.string_to_sign(req)
         boto.log.debug('StringToSign:\n%s' % string_to_sign)
-        hash_value = sha256(string_to_sign).digest()
+        hash_value = sha256(string_to_sign.encode('utf-8')).digest()
         b64_hmac = self.sign_string(hash_value)
         s = "AWS3 AWSAccessKeyId=%s," % self._provider.access_key
         s += "Algorithm=%s," % self.algorithm()
@@ -298,6 +298,9 @@ def __init__(self, host, config, provider,
         self.region_name = region_name
 
     def _sign(self, key, msg, hex=False):
+        if not isinstance(key, bytes):
+            key = key.encode('utf-8')
+
         if hex:
             sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
         else:
@@ -330,8 +333,8 @@ def query_string(self, http_request):
         pairs = []
         for pname in parameter_names:
             pval = boto.utils.get_utf8_value(http_request.params[pname])
-            pairs.append(urllib.quote(pname, safe='') + '=' +
-                         urllib.quote(pval, safe='-_~'))
+            pairs.append(urllib.parse.quote(pname, safe='') + '=' +
+                         urllib.parse.quote(pval, safe='-_~'))
         return '&'.join(pairs)
 
     def canonical_query_string(self, http_request):
@@ -342,8 +345,8 @@ def canonical_query_string(self, http_request):
         l = []
         for param in sorted(http_request.params):
             value = boto.utils.get_utf8_value(http_request.params[param])
-            l.append('%s=%s' % (urllib.quote(param, safe='-_.~'),
-                                urllib.quote(value, safe='-_.~')))
+            l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
+                                urllib.parse.quote(value.decode('utf-8'), safe='-_.~')))
         return '&'.join(l)
 
     def canonical_headers(self, headers_to_sign):
@@ -376,7 +379,7 @@ def canonical_uri(self, http_request):
         # in windows normpath('/') will be '\\' so we chane it back to '/'
         normalized = posixpath.normpath(path).replace('\\','/')
         # Then urlencode whatever's left.
-        encoded = urllib.quote(normalized)
+        encoded = urllib.parse.quote(normalized)
         if len(path) > 1 and path.endswith('/'):
             encoded += '/'
         return encoded
@@ -388,7 +391,9 @@ def payload(self, http_request):
         # the entire body into memory.
         if hasattr(body, 'seek') and hasattr(body, 'read'):
             return boto.utils.compute_hash(body, hash_algorithm=sha256)[0]
-        return sha256(http_request.body).hexdigest()
+        elif not isinstance(body, bytes):
+            body = body.encode('utf-8')
+        return sha256(body).hexdigest()
 
     def canonical_request(self, http_request):
         cr = [http_request.method.upper()]
@@ -462,7 +467,7 @@ def string_to_sign(self, http_request, canonical_request):
         sts = ['AWS4-HMAC-SHA256']
         sts.append(http_request.headers['X-Amz-Date'])
         sts.append(self.credential_scope(http_request))
-        sts.append(sha256(canonical_request).hexdigest())
+        sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
         return '\n'.join(sts)
 
     def signature(self, http_request, string_to_sign):
@@ -538,11 +543,11 @@ def clean_region_name(self, region_name):
     def canonical_uri(self, http_request):
         # S3 does **NOT** do path normalization that SigV4 typically does.
         # Urlencode the path, **NOT** ``auth_path`` (because vhosting).
-        path = urlparse.urlparse(http_request.path)
+        path = urllib.parse.urlparse(http_request.path)
         # Because some quoting may have already been applied, let's back it out.
-        unquoted = urllib.unquote(path.path)
+        unquoted = urllib.parse.unquote(path.path)
         # Requote, this time addressing all characters.
-        encoded = urllib.quote(unquoted)
+        encoded = urllib.parse.quote(unquoted)
         return encoded
 
     def host_header(self, host, http_request):
@@ -633,14 +638,14 @@ def mangle_path_and_params(self, req):
         # **ON** the ``path/auth_path``.
         # Rip them apart, so the ``auth_path/params`` can be signed
         # appropriately.
-        parsed_path = urlparse.urlparse(modified_req.auth_path)
+        parsed_path = urllib.parse.urlparse(modified_req.auth_path)
         modified_req.auth_path = parsed_path.path
 
         if modified_req.params is None:
             modified_req.params = {}
 
         raw_qs = parsed_path.query
-        existing_qs = urlparse.parse_qs(
+        existing_qs = urllib.parse.parse_qs(
             raw_qs,
             keep_blank_values=True
         )
@@ -732,16 +737,16 @@ class QueryAuthHandler(AuthHandler):
     capability = ['pure-query']
 
     def _escape_value(self, value):
-        # Would normally be ``return urllib.quote(value)``.
+        # Would normally be ``return urllib.parse.quote(value)``.
         return value
 
     def _build_query_string(self, params):
-        keys = params.keys()
-        keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+        keys = list(params.keys())
+        keys.sort(key=lambda x: x.lower())
         pairs = []
         for key in keys:
             val = boto.utils.get_utf8_value(params[key])
-            pairs.append(key + '=' + self._escape_value(val))
+            pairs.append(key + '=' + self._escape_value(val.decode('utf-8')))
         return '&'.join(pairs)
 
     def add_auth(self, http_request, **kwargs):
@@ -778,15 +783,15 @@ def add_auth(self, http_request, **kwargs):
         boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
         if http_request.method == 'POST':
             headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
-            http_request.body = qs + '&Signature=' + urllib.quote_plus(signature)
+            http_request.body = qs + '&Signature=' + urllib.parse.quote_plus(signature)
             http_request.headers['Content-Length'] = str(len(http_request.body))
         else:
             http_request.body = ''
             # if this is a retried request, the qs from the previous try will
             # already be there, we need to get rid of that and rebuild it
             http_request.path = http_request.path.split('?')[0]
             http_request.path = (http_request.path + '?' + qs +
-                                 '&Signature=' + urllib.quote_plus(signature))
+                                 '&Signature=' + urllib.parse.quote_plus(signature))
 
 
 class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler):
@@ -799,13 +804,13 @@ def _calc_signature(self, params, *args):
         boto.log.debug('using _calc_signature_0')
         hmac = self._get_hmac()
         s = params['Action'] + params['Timestamp']
-        hmac.update(s)
+        hmac.update(s.encode('utf-8'))
         keys = params.keys()
         keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
         pairs = []
         for key in keys:
             val = boto.utils.get_utf8_value(params[key])
-            pairs.append(key + '=' + urllib.quote(val))
+            pairs.append(key + '=' + urllib.parse.quote(val))
         qs = '&'.join(pairs)
         return (qs, base64.b64encode(hmac.digest()))
 
@@ -830,10 +835,10 @@ def _calc_signature(self, params, *args):
         keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
         pairs = []
         for key in keys:
-            hmac.update(key)
+            hmac.update(key.encode('utf-8'))
             val = boto.utils.get_utf8_value(params[key])
             hmac.update(val)
-            pairs.append(key + '=' + urllib.quote(val))
+            pairs.append(key + '=' + urllib.parse.quote(val))
         qs = '&'.join(pairs)
         return (qs, base64.b64encode(hmac.digest()))
 
@@ -856,13 +861,13 @@ def _calc_signature(self, params, verb, path, server_name):
         pairs = []
         for key in keys:
             val = boto.utils.get_utf8_value(params[key])
-            pairs.append(urllib.quote(key, safe='') + '=' +
-                         urllib.quote(val, safe='-_~'))
+            pairs.append(urllib.parse.quote(key, safe='') + '=' +
+                         urllib.parse.quote(val, safe='-_~'))
         qs = '&'.join(pairs)
         boto.log.debug('query string: %s' % qs)
         string_to_sign += qs
         boto.log.debug('string_to_sign: %s' % string_to_sign)
-        hmac.update(string_to_sign)
+        hmac.update(string_to_sign.encode('utf-8'))
         b64 = base64.b64encode(hmac.digest())
         boto.log.debug('len(b64)=%d' % len(b64))
         boto.log.debug('base64 encoded digest: %s' % b64)
@@ -894,7 +899,7 @@ def add_auth(self, req, **kwargs):
         # already be there, we need to get rid of that and rebuild it
         req.path = req.path.split('?')[0]
         req.path = (req.path + '?' + qs +
-                             '&Signature=' + urllib.quote_plus(signature))
+                             '&Signature=' + urllib.parse.quote_plus(signature))
 
 
 def get_auth_handler(host, config, provider, requested_capability=None):

boto/auth_handler.py

@@ -23,7 +23,7 @@
 Defines an interface which all Auth handlers need to implement.
 """
 
-from plugin import Plugin
+from .plugin import Plugin
 
 class NotReadyToAuthenticate(Exception):
   pass

boto/beanstalk/response.py

@@ -1,13 +1,14 @@
 """Classify responses from layer1 and strict type values."""
 from datetime import datetime
+from boto.compat import six
 
 
 class BaseObject(object):
 
     def __repr__(self):
         result = self.__class__.__name__ + '{ '
         counter = 0
-        for key, value in self.__dict__.iteritems():
+        for key, value in six.iteritems(self.__dict__):
             # first iteration no comma
             counter += 1
             if counter > 1:

boto/beanstalk/wrapper.py

@@ -9,7 +9,7 @@ def beanstalk_wrapper(func, name):
     def _wrapped_low_level_api(*args, **kwargs):
         try:
             response = func(*args, **kwargs)
-        except BotoServerError, e:
+        except BotoServerError as e:
             raise exception.simple(e)
         # Turn 'this_is_a_function_name' into 'ThisIsAFunctionNameResponse'.
         cls_name = ''.join([part.capitalize() for part in name.split('_')]) + 'Response'

boto/cloudformation/__init__.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 
-from connection import CloudFormationConnection
+from .connection import CloudFormationConnection
 from boto.regioninfo import RegionInfo, get_regions, load_regions
 
 RegionData = load_regions().get('cloudformation')