From 15c6cfbbd03ad37ce359bdc09dd7ab61bbadecfc Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 15 Nov 2015 16:16:15 +0800 Subject: [PATCH 001/114] [#180] Separate logical block Fit the `clone` function into class. We just move the origin function body into `__init__` . And, in order to keep the `None` be returned, we let `__new__` do the magic. --- couchapp/clone_app.py | 373 +++++++++++++++++++++--------------------- 1 file changed, 191 insertions(+), 182 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 607bbc31..68f2f3b7 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -17,194 +17,203 @@ logger = logging.getLogger(__name__) -def clone(source, dest=None, rev=None): +class clone(object): """ Clone an application from a design_doc given. :param source: the http/https uri of design document """ - try: - dburl, docid = source.split('_design/') - except ValueError: - raise AppError("{0} isn't a valid source".format(source)) - - if not dest: - dest = docid - - path = os.path.normpath(os.path.join(os.getcwd(), dest)) - if not os.path.exists(path): - os.makedirs(path) - - db = client.Database(dburl[:-1], create=False) - if not rev: - doc = db.open_doc("_design/%s" % docid) - else: - doc = db.open_doc("_design/%s" % docid, rev=rev) - docid = doc['_id'] - - metadata = doc.get('couchapp', {}) - - # get manifest - manifest = metadata.get('manifest', {}) - - # get signatures - signatures = metadata.get('signatures', {}) - - # get objects refs - objects = metadata.get('objects', {}) - - # create files from manifest - if manifest: - for filename in manifest: - logger.debug("clone property: %s" % filename) - filepath = os.path.join(path, filename) - if filename.endswith('/'): - if not os.path.isdir(filepath): - os.makedirs(filepath) - elif filename == "couchapp.json": - continue - else: - parts = util.split_path(filename) - fname = parts.pop() - v = doc - while 1: - try: - for key in parts: - v = v[key] - except KeyError: - break - # remove extension - last_key, ext = os.path.splitext(fname) - - # make sure key exist - try: - content = v[last_key] - except KeyError: - break - - if isinstance(content, basestring): - _ref = md5(util.to_bytestring(content)).hexdigest() - if objects and _ref in objects: - content = objects[_ref] - - if content.startswith('base64-encoded;'): - content = base64.b64decode(content[15:]) - - if fname.endswith('.json'): - content = util.json.dumps(content).encode('utf-8') - - del v[last_key] - - # make sure file dir have been created - filedir = os.path.dirname(filepath) - if not os.path.isdir(filedir): - os.makedirs(filedir) - - util.write(filepath, content) - - # remove the key from design doc - temp = doc - for key2 in parts: - if key2 == key: - if not temp[key2]: - del temp[key2] - break - temp = temp[key2] - - # second pass for missing key or in case - # manifest isn't in app - for key in doc.iterkeys(): - if key.startswith('_'): - continue - elif key in ('couchapp'): - app_meta = copy.deepcopy(doc['couchapp']) - if 'signatures' in app_meta: - del app_meta['signatures'] - if 'manifest' in app_meta: - del app_meta['manifest'] - if 'objects' in app_meta: - del app_meta['objects'] - if 'length' in app_meta: - del app_meta['length'] - if app_meta: - couchapp_file = os.path.join(path, 'couchapp.json') - util.write_json(couchapp_file, app_meta) - elif key in ('views'): - vs_dir = os.path.join(path, key) - if not os.path.isdir(vs_dir): - os.makedirs(vs_dir) - for vsname, vs_item in doc[key].iteritems(): - vs_item_dir = os.path.join(vs_dir, vsname) - if not os.path.isdir(vs_item_dir): - os.makedirs(vs_item_dir) - for func_name, func in vs_item.iteritems(): - filename = os.path.join(vs_item_dir, '%s.js' % func_name) - util.write(filename, func) - logger.warning("clone view not in manifest: %s" % filename) - elif key in ('shows', 'lists', 'filter', 'updates'): - showpath = os.path.join(path, key) - if not os.path.isdir(showpath): - os.makedirs(showpath) - for func_name, func in doc[key].iteritems(): - filename = os.path.join(showpath, '%s.js' % func_name) - util.write(filename, func) - logger.warning( - "clone show or list not in manifest: %s" % filename) + def __init__(self, source, dest=None, rev=None): + try: + dburl, docid = source.split('_design/') + except ValueError: + raise AppError("{0} isn't a valid source".format(source)) + + if not dest: + dest = docid + + path = os.path.normpath(os.path.join(os.getcwd(), dest)) + if not os.path.exists(path): + os.makedirs(path) + + db = client.Database(dburl[:-1], create=False) + if not rev: + doc = db.open_doc("_design/%s" % docid) else: - filedir = os.path.join(path, key) - if os.path.exists(filedir): + doc = db.open_doc("_design/%s" % docid, rev=rev) + docid = doc['_id'] + + metadata = doc.get('couchapp', {}) + + # get manifest + manifest = metadata.get('manifest', {}) + + # get signatures + signatures = metadata.get('signatures', {}) + + # get objects refs + objects = metadata.get('objects', {}) + + # create files from manifest + if manifest: + for filename in manifest: + logger.debug("clone property: %s" % filename) + filepath = os.path.join(path, filename) + if filename.endswith('/'): + if not os.path.isdir(filepath): + os.makedirs(filepath) + elif filename == "couchapp.json": + continue + else: + parts = util.split_path(filename) + fname = parts.pop() + v = doc + while 1: + try: + for key in parts: + v = v[key] + except KeyError: + break + # remove extension + last_key, ext = os.path.splitext(fname) + + # make sure key exist + try: + content = v[last_key] + except KeyError: + break + + if isinstance(content, basestring): + _ref = md5(util.to_bytestring(content)).hexdigest() + if objects and _ref in objects: + content = objects[_ref] + + if content.startswith('base64-encoded;'): + content = base64.b64decode(content[15:]) + + if fname.endswith('.json'): + content = util.json.dumps(content).encode('utf-8') + + del v[last_key] + + # make sure file dir have been created + filedir = os.path.dirname(filepath) + if not os.path.isdir(filedir): + os.makedirs(filedir) + + util.write(filepath, content) + + # remove the key from design doc + temp = doc + for key2 in parts: + if key2 == key: + if not temp[key2]: + del temp[key2] + break + temp = temp[key2] + + # second pass for missing key or in case + # manifest isn't in app + for key in doc.iterkeys(): + if key.startswith('_'): continue + elif key in ('couchapp'): + app_meta = copy.deepcopy(doc['couchapp']) + if 'signatures' in app_meta: + del app_meta['signatures'] + if 'manifest' in app_meta: + del app_meta['manifest'] + if 'objects' in app_meta: + del app_meta['objects'] + if 'length' in app_meta: + del app_meta['length'] + if app_meta: + couchapp_file = os.path.join(path, 'couchapp.json') + util.write_json(couchapp_file, app_meta) + elif key in ('views'): + vs_dir = os.path.join(path, key) + if not os.path.isdir(vs_dir): + os.makedirs(vs_dir) + for vsname, vs_item in doc[key].iteritems(): + vs_item_dir = os.path.join(vs_dir, vsname) + if not os.path.isdir(vs_item_dir): + os.makedirs(vs_item_dir) + for func_name, func in vs_item.iteritems(): + filename = os.path.join(vs_item_dir, '%s.js' % func_name) + util.write(filename, func) + logger.warning("clone view not in manifest: %s" % filename) + elif key in ('shows', 'lists', 'filter', 'updates'): + showpath = os.path.join(path, key) + if not os.path.isdir(showpath): + os.makedirs(showpath) + for func_name, func in doc[key].iteritems(): + filename = os.path.join(showpath, '%s.js' % func_name) + util.write(filename, func) + logger.warning( + "clone show or list not in manifest: %s" % filename) else: - logger.warning("clone property not in manifest: %s" % key) - if isinstance(doc[key], (list, tuple,)): - util.write_json(filedir + ".json", doc[key]) - elif isinstance(doc[key], dict): - if not os.path.isdir(filedir): - os.makedirs(filedir) - for field, value in doc[key].iteritems(): - fieldpath = os.path.join(filedir, field) - if isinstance(value, basestring): - if value.startswith('base64-encoded;'): - value = base64.b64decode(content[15:]) - util.write(fieldpath, value) - else: - util.write_json(fieldpath + '.json', value) + filedir = os.path.join(path, key) + if os.path.exists(filedir): + continue else: - value = doc[key] - if not isinstance(value, basestring): - value = str(value) - util.write(filedir, value) - - # save id - idfile = os.path.join(path, '_id') - util.write(idfile, doc['_id']) - - util.write_json(os.path.join(path, '.couchapprc'), {}) - - if '_attachments' in doc: # process attachments - attachdir = os.path.join(path, '_attachments') - if not os.path.isdir(attachdir): - os.makedirs(attachdir) - - for filename in doc['_attachments'].iterkeys(): - if filename.startswith('vendor'): - attach_parts = util.split_path(filename) - vendor_attachdir = os.path.join(path, attach_parts.pop(0), - attach_parts.pop(0), - '_attachments') - filepath = os.path.join(vendor_attachdir, *attach_parts) - else: - filepath = os.path.join(attachdir, filename) - filepath = os.path.normpath(filepath) - currentdir = os.path.dirname(filepath) - if not os.path.isdir(currentdir): - os.makedirs(currentdir) - - if signatures.get(filename) != util.sign(filepath): - resp = db.fetch_attachment(docid, filename) - with open(filepath, 'wb') as f: - for chunk in resp.body_stream(): - f.write(chunk) - logger.debug("clone attachment: %s" % filename) - - logger.info("%s cloned in %s" % (source, dest)) + logger.warning("clone property not in manifest: %s" % key) + if isinstance(doc[key], (list, tuple,)): + util.write_json(filedir + ".json", doc[key]) + elif isinstance(doc[key], dict): + if not os.path.isdir(filedir): + os.makedirs(filedir) + for field, value in doc[key].iteritems(): + fieldpath = os.path.join(filedir, field) + if isinstance(value, basestring): + if value.startswith('base64-encoded;'): + value = base64.b64decode(content[15:]) + util.write(fieldpath, value) + else: + util.write_json(fieldpath + '.json', value) + else: + value = doc[key] + if not isinstance(value, basestring): + value = str(value) + util.write(filedir, value) + + # save id + idfile = os.path.join(path, '_id') + util.write(idfile, doc['_id']) + + util.write_json(os.path.join(path, '.couchapprc'), {}) + + if '_attachments' in doc: # process attachments + attachdir = os.path.join(path, '_attachments') + if not os.path.isdir(attachdir): + os.makedirs(attachdir) + + for filename in doc['_attachments'].iterkeys(): + if filename.startswith('vendor'): + attach_parts = util.split_path(filename) + vendor_attachdir = os.path.join(path, attach_parts.pop(0), + attach_parts.pop(0), + '_attachments') + filepath = os.path.join(vendor_attachdir, *attach_parts) + else: + filepath = os.path.join(attachdir, filename) + filepath = os.path.normpath(filepath) + currentdir = os.path.dirname(filepath) + if not os.path.isdir(currentdir): + os.makedirs(currentdir) + + if signatures.get(filename) != util.sign(filepath): + resp = db.fetch_attachment(docid, filename) + with open(filepath, 'wb') as f: + for chunk in resp.body_stream(): + f.write(chunk) + logger.debug("clone attachment: %s" % filename) + + logger.info("%s cloned in %s" % (source, dest)) + + def __new__(cls, *args, **kwargs): + obj = super(clone, cls).__new__(cls) + + logger.debug('clone obj created: {0}'.format(obj)) + obj.__init__(*args, **kwargs) + + return None From 7b5de2f2f260e57c63962374f22cabfef9e8a559 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 10 Jan 2016 16:38:15 +0800 Subject: [PATCH 002/114] [#200] Prevent user from placing `env` in `couchapp.json` We will ignore the whole `env` field in `couchapp.json`, and show some warnings. --- couchapp/config.py | 20 +++++++++++++++++--- tests/test_config.py | 25 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/couchapp/config.py b/couchapp/config.py index 1d138873..24d9c275 100644 --- a/couchapp/config.py +++ b/couchapp/config.py @@ -3,6 +3,7 @@ # This file is part of couchapp released under the Apache 2 license. # See the NOTICE for more information. +import logging import re import os @@ -11,6 +12,9 @@ from . import util +logger = logging.getLogger(__name__) + + class Config(object): """ main object to read configuration from ~/.couchapp.conf or .couchapprc/couchapp.json in the couchapp folder. @@ -59,13 +63,23 @@ def load_local(self, app_path): """ Load local config from app/couchapp.json and app/.couchapprc. If both of them contain same vars, the latter one will win. + + To prevent user from placing private data like db credentials, + we will ignore ``env`` parts in ``couchapp.json``. """ if not app_path: raise AppError("You aren't in a couchapp.") - fnames = ('couchapp.json', '.couchapprc') - paths = (os.path.join(app_path, fname) for fname in fnames) - return self.load(paths) + json_conf = self.load(os.path.join(app_path, 'couchapp.json')) + # prevent user from place privating data in couchapp.json + if 'env' in json_conf: + logger.warning('Ignore `env` field in couchapp.json. ' + 'Please place your db credentials in .couchapprc.') + del json_conf['env'] + + return self.load( + os.path.join(app_path, '.couchapprc'), + default=json_conf) def update(self, path): ''' diff --git a/tests/test_config.py b/tests/test_config.py index 6a1e7d81..9044efbf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -138,7 +138,7 @@ def test_load_local(self, load): ''' assert self.config.load_local('/mock') == 'mock' - paths = tuple(load.call_args[0][0]) + paths = (load.call_args_list[0][0][0], load.call_args_list[1][0][0]) assert paths == ('/mock/couchapp.json', '/mock/.couchapprc'), paths @raises(AppError) @@ -150,6 +150,29 @@ def test_load_local_apperror(self, load): self.config.load_local(None) assert not load.called + @patch('couchapp.config.Config.load') + def test_load_local_prevent_env(self, load): + ''' + Test case for Config.load_local() with ``env`` field in ``couchapp.json`` + ''' + def load_side_effect(path, default={}): + print(path == '/mock/couchapp.json', path) + if path == '/mock/couchapp.json': + default.update({'env': 'fake_env', 'name': 'MockApp'}) + elif path == '/mock/.couchapprc': + default.update({'hook': 'mock_hook'}) + else: + raise AssertionError('Unknown local config file "{}"'.format( + path)) + return default + + load.side_effect = load_side_effect + conf = self.config.load_local('/mock') + + assert conf['name'] == 'MockApp' + assert conf['hook'] == 'mock_hook' + assert conf.get('env', None) is None + @patch('couchapp.config.Config.load_local', return_value={'mock': True}) def test_update(self, load_local): ''' From 35411bfa7935b5dcc7f3af80d1b966443e6bc155 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 13 Jan 2016 16:05:38 +0800 Subject: [PATCH 003/114] [doc][#200] docs for local configs, couchapp.json and .couchapprc. [ci skip] --- docs/couchapp/config.rst | 172 ++++++++++++++++++++++++++++---------- docs/couchapp/extends.rst | 6 ++ 2 files changed, 136 insertions(+), 42 deletions(-) diff --git a/docs/couchapp/config.rst b/docs/couchapp/config.rst index 0ef2c65f..a07d5cb4 100644 --- a/docs/couchapp/config.rst +++ b/docs/couchapp/config.rst @@ -1,19 +1,93 @@ .. _couchapp-config: Configuration -============= +=============================================================================== -.. hightlight:: javascript +.. highlight:: javascript -``.couchapprc`` ---------------- -Every CouchApp **MUST** have a ``.couchapprc`` file in the application directory. -This file is a JSON object which contains configuration +Local Configuration +---------------------------------------------------------------------- + +The following config files are placed in each CouchApp directory. + + +``.couchapprc`` and ``couchapp.json`` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Every CouchApp **MUST** have a ``.couchapprc`` file in the application directory; +the ``couchapp.json`` is optional. + +Both files are a JSON object which contains configuration parameters that the command-line app uses to build and push your CouchApp. +Note that if they contain the same fields, the ``.couchapprc`` will win. + The ``couchapp generate`` and ``couchapp init`` commands create a default version of this file for you. +So, what's diff between ``.couchapprc`` and ``couchapp.json``? +Usually, we will put not only configs but some metadata into ``couchapp.json``. +``couchapp.json`` will be published via ``couchapp push``; +the ``.couchapprc`` won't. + + +The valid fields in ``.couchapprc``: + +:env: Place your db credentials here. This field is ``.couchapprc`` only. + +:extensions: List of your :ref:`custom extensions `. + +:hooks: Your :ref:`custom hooks `. + +:vendors: List of your :ref:`vendor handlers `. + +:: + + { + "env": { + // ... + }, + "extensions": [ + // ... + ], + "hooks": { + // ... + }, + "vendors": [ + // ... + ] + } + +The valid fields in ``couchapp.json``: + +*Changed in version 1.1* + +The ``env`` is not available here. Also, do not place any private +credentials in this file. This file will be distributed via ``couchapp push``. + +:: + + { + // other metadata here + // "name": "myCouchApp", + // "version": 1.0, + // ... + "extensions": [ + // ... + ], + "hooks": { + // ... + }, + "vendors": [ + // ... + ] + // ... + } + + +Example +************************************************** + The most common use for the ``.couchapprc`` file is to specify one or more CouchDB databases to use as the destination for the ``couchapp push`` command. Destination databases are listed under the @@ -36,45 +110,23 @@ In this example, two environments are specified: ``default``, which pushes to a local CouchDB instance without any authentication, and ``prod``, which pushes to a remote CouchDB that requires authentication. Once these sections are defined in ``.couchapprc``, you can push to your -local CouchDB by running ``couchapp push`` (the environment name -``default`` is used when no environment is specified) and push to the -remote machine using ``couchapp push prod``. For a more complete -discussion of the ``env`` section of the ``.couchapprc`` file, see the -`Managing Design -Documents `__ -chapter of **CouchDB: The Definitive Guide**. - -The ``.couchapprc`` file is also used to configure extensions to the -``couchapp`` tool. See the :ref:`couchapp-extend` page for more details. - - -``~/.couchapp.conf`` --------------------- - -One drawback to declaring environments in the ``.couchapprc`` file is -that any usernames and passwords required to push documents are stored -in that file. If you are using source control for your CouchApp, then -those authentication credentials are checked in to your (possibly -public) source control server. To avoid this problem, the ``couchapp`` -tool can also read environment configurations from a file stored in your -home directory named ``.couchapp.conf``. This file has the same syntax -as ``.couchapprc`` but has the advantage of being outside of the source -tree, so sensitive login information can be protected. If you already -have a working ``.couchapprc`` file, simply move it to -``~/.couchapp.conf`` and run ``couchapp init`` to generate a new, empty -``.couchapprc`` file inside your CouchApp directory. If you don't have a -``.couchapprc`` file, ``couchapp`` will display the dreaded -``couchapp error: You aren't in a couchapp`` message. +local CouchDB by running:: + + couchapp push +(the environment name ``default`` is used when no environment is specified) +and push to the remote machine using:: -``~/.couchapp`` ---------------- + couchapp push prod -Please see :ref:`couchapp-template` +For a more complete discussion of the ``env`` section of the ``.couchapprc`` +file, see the `Managing Design +Documents `_ +chapter of **CouchDB: The Definitive Guide**. ``.couchappignore`` -------------------- +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ A ``.couchappignore`` file specifies intentionally untracked files that couchapp should ignore. It's a simple json file containing an array of @@ -102,7 +154,43 @@ option:: so creating the ``.couchappignore`` file will be a challenge in windows. Possible solutions to creating this file are: - Using cygwin, type: touch .couchappignore cd /to/couchappand then - notepad .couchappignore + Using cygwin, type:: + + cd /path/to/couchapp + touch .couchappignore + + and then notepad ``.couchappignore``. + + +Global Configuration +---------------------------------------------------------------------- + + +``~/.couchapp.conf`` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +One drawback to declaring environments in the ``.couchapprc`` file is +that any usernames and passwords required to push documents are stored +in that file. If you are using source control for your CouchApp, then +those authentication credentials are checked in to your (possibly +public) source control server. To avoid this problem, the ``couchapp`` +tool can also read environment configurations from a file stored in your +home directory named ``.couchapp.conf``. This file has the same syntax +as ``.couchapprc`` but has the advantage of being outside of the source +tree, so sensitive login information can be protected. + +If you already have a working ``.couchapprc`` file, simply move it to +``~/.couchapp.conf`` and run ``couchapp init`` to generate a new, empty +``.couchapprc`` file inside your CouchApp directory. If you don't have a +``.couchapprc`` file, ``couchapp`` will display the dreaded +``couchapp error: You aren't in a couchapp`` message. + + +``~/.couchapp/`` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Please see :ref:`couchapp-template`. + -TODO: more information about other templates like vendor, view, etc. +.. TODO:: + more information about other templates like vendor, view, etc. diff --git a/docs/couchapp/extends.rst b/docs/couchapp/extends.rst index f2c6664c..cb51bc93 100644 --- a/docs/couchapp/extends.rst +++ b/docs/couchapp/extends.rst @@ -19,6 +19,8 @@ scripts. There are 3 kind of extensions: vendor + .. _couchapp-extend-extensions: + Extensions ----------- @@ -61,6 +63,8 @@ the long option (ex: --verbose) ``default`` could be True/False/None/String/Integer +.. _couchapp-extend-hooks: + Hooks ----- @@ -119,6 +123,8 @@ source Date: Wed, 13 Jan 2016 17:03:55 +0800 Subject: [PATCH 004/114] [doc] Fix installation instructs. [ci skip] --- docs/couchapp/install.rst | 46 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/docs/couchapp/install.rst b/docs/couchapp/install.rst index 6cd71561..2bf38102 100644 --- a/docs/couchapp/install.rst +++ b/docs/couchapp/install.rst @@ -22,14 +22,13 @@ Requirements Installing on all UNIXs ----------------------- -To install couchapp using ``easy_install`` you must make sure you have a -recent version of distribute installed: +To install couchapp using ``pip`` you must make sure you have a +recent version of ``pip`` installed: :: - $ curl -O http://python-distribute.org/distribute_setup.py - $ sudo python distribute_setup.py - $ sudo easy_install pip + $ curl -O https://bootstrap.pypa.io/get-pip.py + $ sudo python get-pip.py To install or upgrade to the latest released version of couchapp: @@ -42,37 +41,32 @@ To install/upgrade development version: :: - $ sudo pip install git+http://github.com/couchapp/couchapp.git#egg=Couchapp + $ sudo pip install git+https://github.com/couchapp/couchapp.git -Installing in a sandboxed environnement +Installing in a sandboxed environment --------------------------------------- -If you want to work in a sandboxed environnement which is recommended if -you don't want to not *pollute* your system, you can use -`virtualenv `_ : +If you want to work in a sandboxed environment which is recommended if +you don't want to not *pollute* your system, you can use `virtualenv +`_ :: -:: - - $ curl -O http://python-distribute.org/distribute_setup.py - $ sudo python distribute_setup.py - $ easy_install pip - $ pip install virtualenv - -Then to install couchapp : + $ curl -O https://bootstrap.pypa.io/get-pip.py + $ sudo python get-pip.py + $ sudo pip install virtualenv -:: - - $ pip -E couchapp_env install couchapp +To create a sandboxed environment in ``couchapp_env`` folder, +activate and work in this environment:: -This command create a sandboxed environment in ``couchapp_env`` folder. -To activate and work in this environment: + $ cd /where/the/sandbox/live + $ virtualenv couchapp_env + $ source couchapp_env/bin/activate -:: +Then to install couchapp:: - $ cd couchapp_env && . ./bin/activate + $ pip install couchapp -Then you can work on your couchapps. I usually have a ``couchapps`` +Then you can work on your CouchApps. I usually have a ``couchapps`` folder in ``couchapp_env`` where I put my couchapps. From 2eb1488758b996ea36c0c9ee3d57249d712f3a3e Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 13 Jan 2016 18:52:21 +0800 Subject: [PATCH 005/114] [setup.py] Update copyright, and remove unused script. --- Couchapp.py | 9 --------- resources/scripts/couchapp | 3 ++- setup.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 Couchapp.py diff --git a/Couchapp.py b/Couchapp.py deleted file mode 100644 index 72900139..00000000 --- a/Couchapp.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of couchapp released under the Apache 2 license. -# See the NOTICE for more information. - -from couchapp.dispatch import run - -if __name__ == '__main__': - run() diff --git a/resources/scripts/couchapp b/resources/scripts/couchapp index c982afe1..6f1d8a5d 100755 --- a/resources/scripts/couchapp +++ b/resources/scripts/couchapp @@ -1,10 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# This file is part of couchapp released under the Apache 2 license. +# This file is part of couchapp released under the Apache 2 license. # See the NOTICE for more information. from couchapp.dispatch import run + if __name__ == '__main__': run() diff --git a/setup.py b/setup.py index c8266e42..c83e49f6 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ def get_py2exe_datafiles(): sys.argv.append("-q") extra['console'] = [{'script': os.path.join("resources", "scripts", "couchapp"), - 'copyright': 'Copyright (C) 2008-2011 Benoît Chesneau and others', + 'copyright': 'Copyright (C) 2008-2016 Benoît Chesneau and others', 'product_version': couchapp.__version__ }] From 1812463a27b9e26b4dc59814ef1400aeb6864a88 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 13 Jan 2016 20:20:54 +0800 Subject: [PATCH 006/114] [README] add the badge for readthedocs --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 31a2e096..e3394052 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,10 @@ CouchApp: Standalone CouchDB Application Development Made Simple .. image:: https://img.shields.io/coveralls/couchapp/couchapp/master.png?style=flat-square :target: https://coveralls.io/r/couchapp/couchapp +.. image:: https://readthedocs.org/projects/couchapp/badge/?version=latest&style=flat-square + :target: https://couchapp.readthedocs.org/en/latest + + CouchApp is designed to structure standalone CouchDB application development for maximum application portability. From d485132e681481f1eaea431af3b9bdcf25d9ceed Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 28 Sep 2015 13:12:36 +0800 Subject: [PATCH 007/114] [#176] Test cases for Config.get() --- tests/test_config.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 6a1e7d81..ce15d116 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -330,3 +330,22 @@ def test_iter(self): assert ('env', {'mock': True}) in ls, ls assert ('hooks', {}) in ls, ls + + def test_get(self): + ''' + Test case for Config.get('__init__') + ''' + assert callable(self.config.get('__init__')) + + def test_get_default(self): + ''' + Test case for Config.get('strang', 'default') + ''' + assert self.config.get('strang', 'default') == 'default' + + def test_get_conf(self): + ''' + Test case for Config.get('env') returning value from self.conf + ''' + self.config.conf['env'] = {'mock': True} + assert self.config.get('env') == {'mock': True} From 626b5e5d4625ba7bde3527028241f2b22598e65b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 15 Jan 2016 15:57:29 +0800 Subject: [PATCH 008/114] [#221] Add `vendors` fields to `Config.DEFAULTS` --- couchapp/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchapp/config.py b/couchapp/config.py index 1d138873..baf0e37e 100644 --- a/couchapp/config.py +++ b/couchapp/config.py @@ -20,7 +20,8 @@ class Config(object): DEFAULTS = dict( env={}, extensions=[], - hooks={} + hooks={}, + vendors=[] ) def __init__(self): From ba6c0c297c1b7b557f5392cd766a120ef55ada74 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 15 Jan 2016 16:20:37 +0800 Subject: [PATCH 009/114] [#219] use deepcopy when `Config.load` copy param `default` test case included Merge pull request #220 from couchapp/issue-219 --- couchapp/config.py | 4 +++- tests/test_config.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/couchapp/config.py b/couchapp/config.py index baf0e37e..861082bb 100644 --- a/couchapp/config.py +++ b/couchapp/config.py @@ -6,6 +6,8 @@ import re import os +from copy import deepcopy + from .client import Database from .errors import AppError from . import util @@ -41,7 +43,7 @@ def load(self, path, default=None): :type path: str or iterable """ - conf = default if default is not None else {} + conf = deepcopy(default) if default is not None else {} paths = [path] if isinstance(path, basestring) else path for p in paths: diff --git a/tests/test_config.py b/tests/test_config.py index ce15d116..4ae0fd0c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -131,6 +131,22 @@ def test_load_apperror(self, isfile, read_json): self.config.load('/mock/couchapp.conf') isfile.assert_called_with('/mock/couchapp.conf') + @patch('couchapp.config.util.read_json', return_value={'mock': True}) + @patch('couchapp.config.os.path.isfile', return_value=True) + def test_load_deepcopy_default(self, isfile, read_json): + ''' + Test case for checking Config.load() deepcopy param ``default`` + ''' + default = {'foo': {'bar': 'fake'}} + + conf = self.config.load('/mock.conf', default=default) + + assert conf == { + 'foo': {'bar': 'fake'}, + 'mock': True + } + assert default == {'foo': {'bar': 'fake'}} + @patch('couchapp.config.Config.load', return_value='mock') def test_load_local(self, load): ''' From ebda309ebaca642ff76f2e33e1d217ef0168451a Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 18 Jan 2016 20:38:16 +0800 Subject: [PATCH 010/114] [doc] update download link for win executable --- docs/couchapp/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/couchapp/install.rst b/docs/couchapp/install.rst index 2bf38102..2602c43b 100644 --- a/docs/couchapp/install.rst +++ b/docs/couchapp/install.rst @@ -122,7 +122,7 @@ Installing on Windows There are currently 2 methods to install on windows: - `Standalone Executable - 1.0.1 `_ + 1.0.2 `_ Does not require Python - `Python installer for Python 2.7 `_ Requires Python From a4e784681c587739472730b61d7ad6110b12af23 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 18 Jan 2016 20:50:09 +0800 Subject: [PATCH 011/114] Update version no. for develop branch --- couchapp/__init__.py | 2 +- couchapp/commands.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/couchapp/__init__.py b/couchapp/__init__.py index 5ef33e18..77b63ce6 100644 --- a/couchapp/__init__.py +++ b/couchapp/__init__.py @@ -3,5 +3,5 @@ # This file is part of couchapp released under the Apache 2 license. # See the NOTICE for more information. -version_info = (1, 0, 1) +version_info = (1, 1, 0) __version__ = ".".join(map(str, version_info)) diff --git a/couchapp/commands.py b/couchapp/commands.py index 2cd5f4c9..5a4cc793 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -330,9 +330,9 @@ def version(conf, *args, **opts): from couchapp import __version__ print "Couchapp (version %s)" % __version__ - print "Copyright 2008-2010 Benoît Chesneau " + print "Copyright 2008-2016 Benoît Chesneau " print "Licensed under the Apache License, Version 2.0." - print "" + if opts.get('help', False): usage(conf, *args, **opts) From fc0108f365f509efcde17b49ffc1f3f1b90a9a10 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 15 Nov 2015 20:23:11 +0800 Subject: [PATCH 012/114] [#180] Separate logical blocks: vars init 1. Rename vars - source -> self.source - dest -> self.dest - rev -> self.rev - dburl -> self.dburl - docid -> self.docid - path -> self.path - doc -> self.doc - manifest -> self.manifest - signatures -> self.signatures - objects -> self.objects 2. logical block: `init_path` 3. logical block: `init_metadata` --- couchapp/clone_app.py | 120 ++++++++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 68f2f3b7..0421676d 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -24,41 +24,38 @@ class clone(object): :param source: the http/https uri of design document """ def __init__(self, source, dest=None, rev=None): + self.source = source + self.dest = dest + self.rev = rev + + # init self.docid & self.dburl try: - dburl, docid = source.split('_design/') + self.dburl, self.docid = self.source.split('_design/') except ValueError: - raise AppError("{0} isn't a valid source".format(source)) + raise AppError("{0} isn't a valid source".format(self.source)) - if not dest: - dest = docid + if not self.dest: + self.dest = self.docid - path = os.path.normpath(os.path.join(os.getcwd(), dest)) - if not os.path.exists(path): - os.makedirs(path) + # init self.path + self.init_path() - db = client.Database(dburl[:-1], create=False) - if not rev: - doc = db.open_doc("_design/%s" % docid) + # init self.db + self.db = client.Database(self.dburl[:-1], create=False) + if not self.rev: + self.doc = self.db.open_doc('_design/{0}'.format(self.docid)) else: - doc = db.open_doc("_design/%s" % docid, rev=rev) - docid = doc['_id'] - - metadata = doc.get('couchapp', {}) - - # get manifest - manifest = metadata.get('manifest', {}) - - # get signatures - signatures = metadata.get('signatures', {}) + self.doc = self.db.open_doc('_design/{0}'.format(self.docid), rev=self.rev) + self.docid = self.doc['_id'] - # get objects refs - objects = metadata.get('objects', {}) + # init metadata + self.init_metadata() # create files from manifest - if manifest: - for filename in manifest: + if self.manifest: + for filename in self.manifest: logger.debug("clone property: %s" % filename) - filepath = os.path.join(path, filename) + filepath = os.path.join(self.path, filename) if filename.endswith('/'): if not os.path.isdir(filepath): os.makedirs(filepath) @@ -67,7 +64,7 @@ def __init__(self, source, dest=None, rev=None): else: parts = util.split_path(filename) fname = parts.pop() - v = doc + v = self.doc while 1: try: for key in parts: @@ -85,8 +82,8 @@ def __init__(self, source, dest=None, rev=None): if isinstance(content, basestring): _ref = md5(util.to_bytestring(content)).hexdigest() - if objects and _ref in objects: - content = objects[_ref] + if self.objects and _ref in self.objects: + content = self.objects[_ref] if content.startswith('base64-encoded;'): content = base64.b64decode(content[15:]) @@ -104,7 +101,7 @@ def __init__(self, source, dest=None, rev=None): util.write(filepath, content) # remove the key from design doc - temp = doc + temp = self.doc for key2 in parts: if key2 == key: if not temp[key2]: @@ -114,11 +111,11 @@ def __init__(self, source, dest=None, rev=None): # second pass for missing key or in case # manifest isn't in app - for key in doc.iterkeys(): + for key in self.doc.iterkeys(): if key.startswith('_'): continue elif key in ('couchapp'): - app_meta = copy.deepcopy(doc['couchapp']) + app_meta = copy.deepcopy(self.doc['couchapp']) if 'signatures' in app_meta: del app_meta['signatures'] if 'manifest' in app_meta: @@ -128,13 +125,13 @@ def __init__(self, source, dest=None, rev=None): if 'length' in app_meta: del app_meta['length'] if app_meta: - couchapp_file = os.path.join(path, 'couchapp.json') + couchapp_file = os.path.join(self.path, 'couchapp.json') util.write_json(couchapp_file, app_meta) elif key in ('views'): - vs_dir = os.path.join(path, key) + vs_dir = os.path.join(self.path, key) if not os.path.isdir(vs_dir): os.makedirs(vs_dir) - for vsname, vs_item in doc[key].iteritems(): + for vsname, vs_item in self.doc[key].iteritems(): vs_item_dir = os.path.join(vs_dir, vsname) if not os.path.isdir(vs_item_dir): os.makedirs(vs_item_dir) @@ -143,26 +140,26 @@ def __init__(self, source, dest=None, rev=None): util.write(filename, func) logger.warning("clone view not in manifest: %s" % filename) elif key in ('shows', 'lists', 'filter', 'updates'): - showpath = os.path.join(path, key) + showpath = os.path.join(self.path, key) if not os.path.isdir(showpath): os.makedirs(showpath) - for func_name, func in doc[key].iteritems(): + for func_name, func in self.doc[key].iteritems(): filename = os.path.join(showpath, '%s.js' % func_name) util.write(filename, func) logger.warning( "clone show or list not in manifest: %s" % filename) else: - filedir = os.path.join(path, key) + filedir = os.path.join(self.path, key) if os.path.exists(filedir): continue else: logger.warning("clone property not in manifest: %s" % key) - if isinstance(doc[key], (list, tuple,)): - util.write_json(filedir + ".json", doc[key]) - elif isinstance(doc[key], dict): + if isinstance(self.doc[key], (list, tuple,)): + util.write_json(filedir + ".json", self.doc[key]) + elif isinstance(self.doc[key], dict): if not os.path.isdir(filedir): os.makedirs(filedir) - for field, value in doc[key].iteritems(): + for field, value in self.doc[key].iteritems(): fieldpath = os.path.join(filedir, field) if isinstance(value, basestring): if value.startswith('base64-encoded;'): @@ -171,26 +168,26 @@ def __init__(self, source, dest=None, rev=None): else: util.write_json(fieldpath + '.json', value) else: - value = doc[key] + value = self.doc[key] if not isinstance(value, basestring): value = str(value) util.write(filedir, value) # save id - idfile = os.path.join(path, '_id') - util.write(idfile, doc['_id']) + idfile = os.path.join(self.path, '_id') + util.write(idfile, self.doc['_id']) - util.write_json(os.path.join(path, '.couchapprc'), {}) + util.write_json(os.path.join(self.path, '.couchapprc'), {}) - if '_attachments' in doc: # process attachments - attachdir = os.path.join(path, '_attachments') + if '_attachments' in self.doc: # process attachments + attachdir = os.path.join(self.path, '_attachments') if not os.path.isdir(attachdir): os.makedirs(attachdir) - for filename in doc['_attachments'].iterkeys(): + for filename in self.doc['_attachments'].iterkeys(): if filename.startswith('vendor'): attach_parts = util.split_path(filename) - vendor_attachdir = os.path.join(path, attach_parts.pop(0), + vendor_attachdir = os.path.join(self.path, attach_parts.pop(0), attach_parts.pop(0), '_attachments') filepath = os.path.join(vendor_attachdir, *attach_parts) @@ -201,14 +198,14 @@ def __init__(self, source, dest=None, rev=None): if not os.path.isdir(currentdir): os.makedirs(currentdir) - if signatures.get(filename) != util.sign(filepath): - resp = db.fetch_attachment(docid, filename) + if self.signatures.get(filename) != util.sign(filepath): + resp = self.db.fetch_attachment(self.docid, filename) with open(filepath, 'wb') as f: for chunk in resp.body_stream(): f.write(chunk) logger.debug("clone attachment: %s" % filename) - logger.info("%s cloned in %s" % (source, dest)) + logger.info("%s cloned in %s" % (self.source, self.dest)) def __new__(cls, *args, **kwargs): obj = super(clone, cls).__new__(cls) @@ -217,3 +214,22 @@ def __new__(cls, *args, **kwargs): obj.__init__(*args, **kwargs) return None + + def init_path(self): + self.path = os.path.normpath(os.path.join(os.getcwd(), self.dest)) + + if not os.path.exists(self.path): + os.makedirs(self.path) + + def init_metadata(self): + ''' + Setup + - self.manifest + - self.signatures + - self.objects: objects refs + ''' + metadata = self.doc.get('couchapp', {}) + + self.manifest = metadata.get('manifest', {}) + self.signatures = metadata.get('signatures', {}) + self.objects = metadata.get('objects', {}) From cf62c1b719eb8552a168c67a8a543b541a65220f Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Thu, 19 Nov 2015 14:35:39 +0800 Subject: [PATCH 013/114] [#180] Separate logical blocks: `setup_manifest` And, some doc string. --- couchapp/clone_app.py | 133 ++++++++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 56 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 0421676d..f3822f4e 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -52,62 +52,7 @@ def __init__(self, source, dest=None, rev=None): self.init_metadata() # create files from manifest - if self.manifest: - for filename in self.manifest: - logger.debug("clone property: %s" % filename) - filepath = os.path.join(self.path, filename) - if filename.endswith('/'): - if not os.path.isdir(filepath): - os.makedirs(filepath) - elif filename == "couchapp.json": - continue - else: - parts = util.split_path(filename) - fname = parts.pop() - v = self.doc - while 1: - try: - for key in parts: - v = v[key] - except KeyError: - break - # remove extension - last_key, ext = os.path.splitext(fname) - - # make sure key exist - try: - content = v[last_key] - except KeyError: - break - - if isinstance(content, basestring): - _ref = md5(util.to_bytestring(content)).hexdigest() - if self.objects and _ref in self.objects: - content = self.objects[_ref] - - if content.startswith('base64-encoded;'): - content = base64.b64decode(content[15:]) - - if fname.endswith('.json'): - content = util.json.dumps(content).encode('utf-8') - - del v[last_key] - - # make sure file dir have been created - filedir = os.path.dirname(filepath) - if not os.path.isdir(filedir): - os.makedirs(filedir) - - util.write(filepath, content) - - # remove the key from design doc - temp = self.doc - for key2 in parts: - if key2 == key: - if not temp[key2]: - del temp[key2] - break - temp = temp[key2] + self.setup_manifest() # second pass for missing key or in case # manifest isn't in app @@ -233,3 +178,79 @@ def init_metadata(self): self.manifest = metadata.get('manifest', {}) self.signatures = metadata.get('signatures', {}) self.objects = metadata.get('objects', {}) + + def setup_manifest(self): + ''' + create files/dirs from manifest + + manifest has following format in json: + ``` + "manifest": [ + "some_dir/", + "file_foo", + "bar.json" + ] + ``` + ''' + if not self.manifest: + return + + for filename in self.manifest: + logger.debug('clone property: "{0}"'.format(filename)) + + filepath = os.path.join(self.path, filename) + if filename.endswith('/'): # create dir + if not os.path.isdir(filepath): + os.makedirs(filepath) + continue + elif filename == 'couchapp.json': # we will handle it later + continue + + # create file + parts = util.split_path(filename) + fname = parts.pop() + v = self.doc + while 1: + try: + for key in parts: + v = v[key] + except KeyError: + break + + # remove extension + last_key, ext = os.path.splitext(fname) + + # make sure key exist + try: + content = v[last_key] + except KeyError: + break + + if isinstance(content, basestring): + _ref = md5(util.to_bytestring(content)).hexdigest() + if self.objects and _ref in self.objects: + content = self.objects[_ref] + + if content.startswith('base64-encoded;'): + content = base64.b64decode(content[15:]) + + if fname.endswith('.json'): + content = util.json.dumps(content).encode('utf-8') + + del v[last_key] + + # make sure file dir have been created + filedir = os.path.dirname(filepath) + if not os.path.isdir(filedir): + os.makedirs(filedir) + + util.write(filepath, content) + + # remove the key from design doc + temp = self.doc + for key2 in parts: + if key2 == key: + if not temp[key2]: + del temp[key2] + break + temp = temp[key2] From b8e72c40449e1e74f8a85b7b62d8f37e9044519b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 20 Nov 2015 22:39:43 +0800 Subject: [PATCH 014/114] [#180] Separate logical blocks: `self.setup_missing` For the files not in manifest, or in case of the manifest missing. This blocks is still big, consider to separate it late. And the handlers of `views`, `shows`, and ... should be merged. --- couchapp/clone_app.py | 132 +++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index f3822f4e..89e8c2de 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -23,6 +23,7 @@ class clone(object): :param source: the http/https uri of design document """ + def __init__(self, source, dest=None, rev=None): self.source = source self.dest = dest @@ -56,67 +57,7 @@ def __init__(self, source, dest=None, rev=None): # second pass for missing key or in case # manifest isn't in app - for key in self.doc.iterkeys(): - if key.startswith('_'): - continue - elif key in ('couchapp'): - app_meta = copy.deepcopy(self.doc['couchapp']) - if 'signatures' in app_meta: - del app_meta['signatures'] - if 'manifest' in app_meta: - del app_meta['manifest'] - if 'objects' in app_meta: - del app_meta['objects'] - if 'length' in app_meta: - del app_meta['length'] - if app_meta: - couchapp_file = os.path.join(self.path, 'couchapp.json') - util.write_json(couchapp_file, app_meta) - elif key in ('views'): - vs_dir = os.path.join(self.path, key) - if not os.path.isdir(vs_dir): - os.makedirs(vs_dir) - for vsname, vs_item in self.doc[key].iteritems(): - vs_item_dir = os.path.join(vs_dir, vsname) - if not os.path.isdir(vs_item_dir): - os.makedirs(vs_item_dir) - for func_name, func in vs_item.iteritems(): - filename = os.path.join(vs_item_dir, '%s.js' % func_name) - util.write(filename, func) - logger.warning("clone view not in manifest: %s" % filename) - elif key in ('shows', 'lists', 'filter', 'updates'): - showpath = os.path.join(self.path, key) - if not os.path.isdir(showpath): - os.makedirs(showpath) - for func_name, func in self.doc[key].iteritems(): - filename = os.path.join(showpath, '%s.js' % func_name) - util.write(filename, func) - logger.warning( - "clone show or list not in manifest: %s" % filename) - else: - filedir = os.path.join(self.path, key) - if os.path.exists(filedir): - continue - else: - logger.warning("clone property not in manifest: %s" % key) - if isinstance(self.doc[key], (list, tuple,)): - util.write_json(filedir + ".json", self.doc[key]) - elif isinstance(self.doc[key], dict): - if not os.path.isdir(filedir): - os.makedirs(filedir) - for field, value in self.doc[key].iteritems(): - fieldpath = os.path.join(filedir, field) - if isinstance(value, basestring): - if value.startswith('base64-encoded;'): - value = base64.b64decode(content[15:]) - util.write(fieldpath, value) - else: - util.write_json(fieldpath + '.json', value) - else: - value = self.doc[key] - if not isinstance(value, basestring): - value = str(value) - util.write(filedir, value) + self.setup_missing() # save id idfile = os.path.join(self.path, '_id') @@ -254,3 +195,72 @@ def setup_manifest(self): del temp[key2] break temp = temp[key2] + + def setup_missing(self): + ''' + second pass for missing key or in case manifest isn't in app. + ''' + for key in self.doc.iterkeys(): + if key.startswith('_'): + continue + elif key in ('couchapp'): + app_meta = copy.deepcopy(self.doc['couchapp']) + if 'signatures' in app_meta: + del app_meta['signatures'] + if 'manifest' in app_meta: + del app_meta['manifest'] + if 'objects' in app_meta: + del app_meta['objects'] + if 'length' in app_meta: + del app_meta['length'] + if app_meta: + couchapp_file = os.path.join(self.path, 'couchapp.json') + util.write_json(couchapp_file, app_meta) + elif key in ('views'): + vs_dir = os.path.join(self.path, key) + if not os.path.isdir(vs_dir): + os.makedirs(vs_dir) + for vsname, vs_item in self.doc[key].iteritems(): + vs_item_dir = os.path.join(vs_dir, vsname) + if not os.path.isdir(vs_item_dir): + os.makedirs(vs_item_dir) + for func_name, func in vs_item.iteritems(): + filename = os.path.join(vs_item_dir, + '{0}.js'.format(func_name)) + util.write(filename, func) + logger.warning( + 'clone view not in manifest: "{0}"'.format(filename)) + elif key in ('shows', 'lists', 'filter', 'updates'): + showpath = os.path.join(self.path, key) + if not os.path.isdir(showpath): + os.makedirs(showpath) + for func_name, func in self.doc[key].iteritems(): + filename = os.path.join(showpath, + '{0}.js'.format(func_name)) + util.write(filename, func) + logger.warning( + 'clone show or list not in manifest: {0}'.format(filename)) + else: # handle other property + filedir = os.path.join(self.path, key) + if os.path.exists(filedir): + continue + + logger.warning("clone property not in manifest: {0}".format(key)) + if isinstance(self.doc[key], (list, tuple,)): + util.write_json('{0}.json'.format(filedir), self.doc[key]) + elif isinstance(self.doc[key], dict): + if not os.path.isdir(filedir): + os.makedirs(filedir) + for field, value in self.doc[key].iteritems(): + fieldpath = os.path.join(filedir, field) + if isinstance(value, basestring): + if value.startswith('base64-encoded;'): + value = base64.b64decode(content[15:]) + util.write(fieldpath, value) + else: + util.write_json(fieldpath + '.json', value) + else: + value = self.doc[key] + if not isinstance(value, basestring): + value = str(value) + util.write(filedir, value) From ab45453f5fac033d205c226a302af2f7de151887 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 00:03:26 +0800 Subject: [PATCH 015/114] [#180] Separate logical blocks: `self.setup_couchapp_json` This function separate from `self.setup_missing`. It will handle the `couchapp.json` in app root. --- couchapp/clone_app.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 89e8c2de..de2dc0ce 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -204,18 +204,7 @@ def setup_missing(self): if key.startswith('_'): continue elif key in ('couchapp'): - app_meta = copy.deepcopy(self.doc['couchapp']) - if 'signatures' in app_meta: - del app_meta['signatures'] - if 'manifest' in app_meta: - del app_meta['manifest'] - if 'objects' in app_meta: - del app_meta['objects'] - if 'length' in app_meta: - del app_meta['length'] - if app_meta: - couchapp_file = os.path.join(self.path, 'couchapp.json') - util.write_json(couchapp_file, app_meta) + self.setup_couchapp_json() elif key in ('views'): vs_dir = os.path.join(self.path, key) if not os.path.isdir(vs_dir): @@ -264,3 +253,28 @@ def setup_missing(self): if not isinstance(value, basestring): value = str(value) util.write(filedir, value) + + def setup_couchapp_json(self): + ''' + Create ``couchapp.json`` from ``self.doc['couchapp']``. + + We will exclude the following properties: + - ``signatures`` + - ``manifest`` + - ``objects`` + - ``length`` + ''' + app_meta = copy.deepcopy(self.doc['couchapp']) + + if 'signatures' in app_meta: + del app_meta['signatures'] + if 'manifest' in app_meta: + del app_meta['manifest'] + if 'objects' in app_meta: + del app_meta['objects'] + if 'length' in app_meta: + del app_meta['length'] + + if app_meta: + couchapp_file = os.path.join(self.path, 'couchapp.json') + util.write_json(couchapp_file, app_meta) From cadb66dba41c8d87a4d5edee57a288cf6fc9a8ef Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 00:52:26 +0800 Subject: [PATCH 016/114] [#180] Separate logical blocks: `self.setup_views` This function separeated from `self.setup_missing` --- couchapp/clone_app.py | 44 ++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index de2dc0ce..3c016178 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -206,19 +206,7 @@ def setup_missing(self): elif key in ('couchapp'): self.setup_couchapp_json() elif key in ('views'): - vs_dir = os.path.join(self.path, key) - if not os.path.isdir(vs_dir): - os.makedirs(vs_dir) - for vsname, vs_item in self.doc[key].iteritems(): - vs_item_dir = os.path.join(vs_dir, vsname) - if not os.path.isdir(vs_item_dir): - os.makedirs(vs_item_dir) - for func_name, func in vs_item.iteritems(): - filename = os.path.join(vs_item_dir, - '{0}.js'.format(func_name)) - util.write(filename, func) - logger.warning( - 'clone view not in manifest: "{0}"'.format(filename)) + self.setup_views() elif key in ('shows', 'lists', 'filter', 'updates'): showpath = os.path.join(self.path, key) if not os.path.isdir(showpath): @@ -278,3 +266,33 @@ def setup_couchapp_json(self): if app_meta: couchapp_file = os.path.join(self.path, 'couchapp.json') util.write_json(couchapp_file, app_meta) + + def setup_views(self): + ''' + Create ``views/`` + + ``views`` dir will have following structure: + ``` + views/ + view_name/ + map.js + reduce.js (optional) + view_name2/ + ... + ``` + ''' + vs_dir = os.path.join(self.path, 'views') + + if not os.path.isdir(vs_dir): + os.makedirs(vs_dir) + + for vsname, vs_item in self.doc['views'].iteritems(): + vs_item_dir = os.path.join(vs_dir, vsname) + if not os.path.isdir(vs_item_dir): + os.makedirs(vs_item_dir) + for func_name, func in vs_item.iteritems(): + filename = os.path.join(vs_item_dir, + '{0}.js'.format(func_name)) + util.write(filename, func) + logger.warning( + 'clone view not in manifest: "{0}"'.format(filename)) From a694710e8fabdf7071758214ac4c4b84fedc46bb Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 10:00:28 +0800 Subject: [PATCH 017/114] [#180] Separate logical blocks: `self.setup_func` This function separated from `self.setup_missing`. We will consider to merge `setup_views` into `setup_func` later. --- couchapp/clone_app.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 3c016178..2a2e0541 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -208,15 +208,7 @@ def setup_missing(self): elif key in ('views'): self.setup_views() elif key in ('shows', 'lists', 'filter', 'updates'): - showpath = os.path.join(self.path, key) - if not os.path.isdir(showpath): - os.makedirs(showpath) - for func_name, func in self.doc[key].iteritems(): - filename = os.path.join(showpath, - '{0}.js'.format(func_name)) - util.write(filename, func) - logger.warning( - 'clone show or list not in manifest: {0}'.format(filename)) + self.setup_func(key) else: # handle other property filedir = os.path.join(self.path, key) if os.path.exists(filedir): @@ -296,3 +288,23 @@ def setup_views(self): util.write(filename, func) logger.warning( 'clone view not in manifest: "{0}"'.format(filename)) + + def setup_func(self, func): + ''' + Create dir for function: + - ``shows`` + - ``lists + - ``filters`` + - ``updates`` + ''' + showpath = os.path.join(self.path, func) + + if not os.path.isdir(showpath): + os.makedirs(showpath) + + for func_name, func in self.doc[func].iteritems(): + filename = os.path.join(showpath, '{0}.js'.format(func_name)) + util.write(filename, func) + logger.warning( + 'clone function "{0}" not in manifest: {1}'.format(func, + filename)) From 17d7f341e48ac9af004e386691bb754f3f8d3b5f Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Thu, 28 Jan 2016 12:05:35 +0800 Subject: [PATCH 018/114] [travis] Update testing depends via pip (cherry picked from commit 135222a99efc85b418bc09e7c9667823a91070ab) --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 31ecd824..35ccbb05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ services: - couchdb sudo: false before_script: - - pip install coveralls unittest2 nose-testconfig mock + - pip install -U pip + - pip --version + - pip install -U coveralls unittest2 nose nose-testconfig mock script: - python setup.py install - python setup.py nosetests From c1fba5a2f4a876060ae5ed40c20b67b6e425ace7 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 10:18:06 +0800 Subject: [PATCH 019/114] [#180] Separate logical blocks: `self.setup_id` --- couchapp/clone_app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 2a2e0541..2596bd4e 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -60,8 +60,7 @@ def __init__(self, source, dest=None, rev=None): self.setup_missing() # save id - idfile = os.path.join(self.path, '_id') - util.write(idfile, self.doc['_id']) + self.setup_id() util.write_json(os.path.join(self.path, '.couchapprc'), {}) @@ -308,3 +307,9 @@ def setup_func(self, func): logger.warning( 'clone function "{0}" not in manifest: {1}'.format(func, filename)) + def setup_id(self): + ''' + Create ``_id`` file + ''' + idfile = os.path.join(self.path, '_id') + util.write(idfile, self.doc['_id']) From ad56f13e8f0299b4c71b5091c57602ff465bfcd9 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 11:04:13 +0800 Subject: [PATCH 020/114] [#180] Separate logical blocks: `self.setup_attachments` Also, using the new style str formater. --- couchapp/clone_app.py | 66 ++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 2596bd4e..ae1c4079 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -62,35 +62,14 @@ def __init__(self, source, dest=None, rev=None): # save id self.setup_id() + # setup empty .couchapprc util.write_json(os.path.join(self.path, '.couchapprc'), {}) - if '_attachments' in self.doc: # process attachments - attachdir = os.path.join(self.path, '_attachments') - if not os.path.isdir(attachdir): - os.makedirs(attachdir) - - for filename in self.doc['_attachments'].iterkeys(): - if filename.startswith('vendor'): - attach_parts = util.split_path(filename) - vendor_attachdir = os.path.join(self.path, attach_parts.pop(0), - attach_parts.pop(0), - '_attachments') - filepath = os.path.join(vendor_attachdir, *attach_parts) - else: - filepath = os.path.join(attachdir, filename) - filepath = os.path.normpath(filepath) - currentdir = os.path.dirname(filepath) - if not os.path.isdir(currentdir): - os.makedirs(currentdir) - - if self.signatures.get(filename) != util.sign(filepath): - resp = self.db.fetch_attachment(self.docid, filename) - with open(filepath, 'wb') as f: - for chunk in resp.body_stream(): - f.write(chunk) - logger.debug("clone attachment: %s" % filename) + # process attachments + self.setup_attachments() - logger.info("%s cloned in %s" % (self.source, self.dest)) + logger.info("{src} cloned in {dest}".format(src=self.source, + dest=self.dest)) def __new__(cls, *args, **kwargs): obj = super(clone, cls).__new__(cls) @@ -307,9 +286,44 @@ def setup_func(self, func): logger.warning( 'clone function "{0}" not in manifest: {1}'.format(func, filename)) + def setup_id(self): ''' Create ``_id`` file ''' idfile = os.path.join(self.path, '_id') util.write(idfile, self.doc['_id']) + + def setup_attachments(self): + ''' + Create ``_attachments`` dir + ''' + if '_attachments' not in self.doc: + return + + attachdir = os.path.join(self.path, '_attachments') + + if not os.path.isdir(attachdir): + os.makedirs(attachdir) + + for filename in self.doc['_attachments'].iterkeys(): + if filename.startswith('vendor'): + attach_parts = util.split_path(filename) + vendor_attachdir = os.path.join(self.path, attach_parts.pop(0), + attach_parts.pop(0), + '_attachments') + filepath = os.path.join(vendor_attachdir, *attach_parts) + else: + filepath = os.path.join(attachdir, filename) + + filepath = os.path.normpath(filepath) + currentdir = os.path.dirname(filepath) + if not os.path.isdir(currentdir): + os.makedirs(currentdir) + + if self.signatures.get(filename) != util.sign(filepath): + resp = self.db.fetch_attachment(self.docid, filename) + with open(filepath, 'wb') as f: + for chunk in resp.body_stream(): + f.write(chunk) + logger.debug('clone attachment: {0}'.format(filename)) From 239cfc2ecf7b3f6a124ce52ec97a843ba3edb5fe Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 22 Nov 2015 11:35:45 +0800 Subject: [PATCH 021/114] [#180] Separate logical blocks: `setup_prop` This function separated from `setup_missing`. Also, add the docstring for `setup_prop, and consider to apply the policy strictly later. --- couchapp/clone_app.py | 69 ++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index ae1c4079..c7709346 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -187,30 +187,53 @@ def setup_missing(self): self.setup_views() elif key in ('shows', 'lists', 'filter', 'updates'): self.setup_func(key) - else: # handle other property - filedir = os.path.join(self.path, key) - if os.path.exists(filedir): - continue - - logger.warning("clone property not in manifest: {0}".format(key)) - if isinstance(self.doc[key], (list, tuple,)): - util.write_json('{0}.json'.format(filedir), self.doc[key]) - elif isinstance(self.doc[key], dict): - if not os.path.isdir(filedir): - os.makedirs(filedir) - for field, value in self.doc[key].iteritems(): - fieldpath = os.path.join(filedir, field) - if isinstance(value, basestring): - if value.startswith('base64-encoded;'): - value = base64.b64decode(content[15:]) - util.write(fieldpath, value) - else: - util.write_json(fieldpath + '.json', value) + else: + self.setup_prop(key) + + def setup_prop(self, prop): + ''' + Create file for arbitrary property. + + Policy: + - If the property is a list, we will save it as json file. + + - If the property is a dict, we will create a dir for it and + handle its contents recursively. + + - If the property starts with ``base64-encoded;``, + we decode it and save as binary file. + + - If the property is simple plane text, we just save it. + ''' + if prop not in self.doc: + return + + filedir = os.path.join(self.path, prop) + + if os.path.exists(filedir): + return + + logger.warning('clone property not in manifest: {0}'.format(prop)) + + if isinstance(self.doc[prop], (list, tuple,)): + util.write_json('{0}.json'.format(filedir), self.doc[prop]) + elif isinstance(self.doc[prop], dict): + if not os.path.isdir(filedir): + os.makedirs(filedir) + + for field, value in self.doc[prop].iteritems(): + fieldpath = os.path.join(filedir, field) + if isinstance(value, basestring): + if value.startswith('base64-encoded;'): + value = base64.b64decode(content[15:]) + util.write(fieldpath, value) else: - value = self.doc[key] - if not isinstance(value, basestring): - value = str(value) - util.write(filedir, value) + util.write_json(fieldpath + '.json', value) + else: + value = self.doc[prop] + if not isinstance(value, basestring): + value = str(value) + util.write(filedir, value) def setup_couchapp_json(self): ''' From cee5a2b19529c5267db579eec857671f21641574 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 23 Nov 2015 10:51:28 +0800 Subject: [PATCH 022/114] [#180] New helper function: `setup_dir` It's common for `clone` to create dir again and again, so we encapsulate the `os.makedirs`. Test cases included. --- couchapp/clone_app.py | 20 ++++++++++++++++++++ tests/test_clone_app.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index c7709346..6e6a583b 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -350,3 +350,23 @@ def setup_attachments(self): for chunk in resp.body_stream(): f.write(chunk) logger.debug('clone attachment: {0}'.format(filename)) + + def setup_dir(self, path): + ''' + Create dir recursively. + + :return: True, if create success. + Else, false. + ''' + if not path: + return False + if os.path.exists(path): + logger.warning('file exists: "{0}"'.format(path)) + return False + + try: + os.makedirs(path) + except OSError as e: + logger.debug(e) + return False + return True diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 54a83b58..93189603 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -15,3 +15,36 @@ def test_invalid_source(): If a source uri do not contain ``_design/``, it's invalid. ''' clone('http://foo.bar') + + +class TestCloneMethod(): + ''' + Test cases for the internal-used method of ``clone`` + ''' + + def setup(self): + self.clone = object.__new__(clone) + + def teardown(self): + del self.clone + + @patch('couchapp.clone_app.os.makedirs') + def test_setup_dir(self, makedirs): + assert self.clone.setup_dir('/tmp/mock') + + @patch('couchapp.clone_app.os.makedirs', side_effect=OSError) + def test_setup_dir_failed(self, makedirs): + assert self.clone.setup_dir('/tmp/mock') is False + + @patch('couchapp.clone_app.os.path.exists', return_value=True) + @patch('couchapp.clone_app.os.makedirs') + def test_setup_dir_exists(self, makedirs, exists): + assert self.clone.setup_dir('/tmp/mock') is False + assert makedirs.called is False + + @patch('couchapp.clone_app.os.path.exists') + @patch('couchapp.clone_app.os.makedirs') + def test_setup_dir_empty(self, makedirs, exists): + assert self.clone.setup_dir('') is False + assert exists.called is False + assert makedirs.called is False From 49f3924d621bf3660d29bcecf51f41cd4bca8dff Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 24 Nov 2015 19:39:42 +0800 Subject: [PATCH 023/114] [#180] Test cases and code refinement for `setup_dir` --- couchapp/clone_app.py | 7 +++---- tests/test_clone_app.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 6e6a583b..d27bee7a 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -80,10 +80,9 @@ def __new__(cls, *args, **kwargs): return None def init_path(self): - self.path = os.path.normpath(os.path.join(os.getcwd(), self.dest)) - - if not os.path.exists(self.path): - os.makedirs(self.path) + self.path = os.path.normpath(os.path.join(os.getcwd(), + self.dest or '')) + self.setup_dir(self.path) def init_metadata(self): ''' diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 93189603..03d867e3 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -48,3 +48,33 @@ def test_setup_dir_empty(self, makedirs, exists): assert self.clone.setup_dir('') is False assert exists.called is False assert makedirs.called is False + + @patch('couchapp.clone_app.os.getcwd', return_value='/mock') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_path_cwd(self, setup_dir, getcwd): + ''' + Test case for ``DEST`` not given + + We will use the current working dir. + ''' + self.clone.dest = None + + self.clone.init_path() + + assert getcwd.called + assert self.clone.path == '/mock' + setup_dir.assert_called_with('/mock') + + @patch('couchapp.clone_app.os.getcwd', return_value='/tmp') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_path_dest(self, setup_dir, getcwd): + ''' + Test case for ``DEST`` given + ''' + self.clone.dest = 'mock' + + self.clone.init_path() + + assert getcwd.called + assert self.clone.path == '/tmp/mock' + setup_dir.assert_called_with('/tmp/mock') From 91a94724bb3a48301728b3e4b5ef9f1cdfe7ffee Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 5 Feb 2016 16:58:18 +0800 Subject: [PATCH 024/114] [py3 compat] using the print function form __future__ --- couchapp/commands.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 5a4cc793..4d7630ce 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -3,15 +3,15 @@ # This file is part of couchapp released under the Apache 2 license. # See the NOTICE for more information. +from __future__ import print_function + import logging import os -from couchapp import clone_app +from couchapp import clone_app, generator, util from couchapp.autopush.command import autopush, DEFAULT_UPDATE_DELAY from couchapp.errors import ResourceNotFound, AppError, BulkSaveError -from couchapp import generator from couchapp.localdoc import document -from couchapp import util from couchapp.vendors import vendor_install, vendor_update logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def push(conf, path, *args, **opts): if opts.get('output'): util.write_json(opts.get('output'), doc) else: - print doc.to_json() + print(doc.to_json()) return 0 dbs = conf.get_dbs(dest) @@ -110,7 +110,7 @@ def pushapps(conf, source, dest, *args, **opts): if opts.get('output'): util.write_json(opts.get('output'), jsonobj) else: - print util.json.dumps(jsonobj) + print(util.json.dumps(jsonobj)) return 0 for db in dbs: @@ -169,7 +169,7 @@ def pushdocs(conf, source, dest, *args, **opts): if opts.get('output'): util.write_json(opts.get('output'), jsonobj) else: - print util.json.dumps(jsonobj) + print(util.json.dumps(jsonobj)) else: for db in dbs: docs1 = [] @@ -329,9 +329,9 @@ def browse(conf, path, *args, **opts): def version(conf, *args, **opts): from couchapp import __version__ - print "Couchapp (version %s)" % __version__ - print "Copyright 2008-2016 Benoît Chesneau " - print "Licensed under the Apache License, Version 2.0." + print("Couchapp (version {0})".format(__version__)) + print("Copyright 2008-2016 Benoît Chesneau ") + print("Licensed under the Apache License, Version 2.0.") if opts.get('help', False): usage(conf, *args, **opts) @@ -342,34 +342,34 @@ def version(conf, *args, **opts): def usage(conf, *args, **opts): if opts.get('version', False): version(conf, *args, **opts) - print "Usage: couchapp [OPTIONS] [CMD] [CMDOPTIONS] [ARGS,...]" + print('Usage: couchapp [OPTIONS] [CMD] [CMDOPTIONS] [ARGS,...]') - print "" - print "Options:" + print() + print('Options:') mainopts = [] max_opt_len = len(max(globalopts, key=len)) for opt in globalopts: - print "\t%-*s" % (max_opt_len, get_switch_str(opt)) + print('\t{opt: <{max_len}}'.format(opt=get_switch_str(opt), + max_len=max_opt_len)) mainopts.append(opt[0]) - print "" - print "Commands:" + print() + print('Commands:') commands = sorted(table.keys()) max_len = len(max(commands, key=len)) for cmd in commands: opts = table[cmd] - # Command name is max_len characters. Used by the %-*s formatting code - print "\t%-*s %s" % (max_len, cmd, opts[2]) + print('\t{cmd: <{max_len}} {opts}'.format( + cmd=cmd, max_len=max_len, opts=opts[2])) # Print each command's option list cmd_options = opts[1] if cmd_options: max_opt = max(cmd_options, key=lambda o: len(get_switch_str(o))) max_opt_len = len(get_switch_str(max_opt)) for opt in cmd_options: - print "\t\t%-*s %s" % (max_opt_len, get_switch_str(opt), - opt[3]) - print "" - print "" + print('\t\t{opt_str: <{max_len}} {opts}'.format( + opt_str=get_switch_str(opt), max_len=max_opt_len, + opts=opt[3])) return 0 From 703769f5e40b6850c4a8a0ec150ccb3b7080dfd5 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 5 Feb 2016 17:17:36 +0800 Subject: [PATCH 025/114] [commands.py] use new style format str --- couchapp/commands.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 4d7630ce..f96ab572 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -221,7 +221,7 @@ def startapp(conf, *args, **opts): dest = os.path.normpath(os.path.join(args[0], name)) if util.iscouchapp(dest): - raise AppError("can't create an app at '%s'. " + raise AppError("can't create an app at '{0}'. " "One already exists here.".format(dest)) if util.findcouchapp(dest): raise AppError("can't create an app inside another app '{0}'.".format( @@ -383,10 +383,11 @@ def get_switch_str(opt): default = "[VAL]" if opt[0]: # has a short and long option - return "-%s, --%s %s" % (opt[0], opt[1], default) + return '-{opt[0]}, --{opt[1]} {default}'.format(opt=opt, + default=default) else: # only has a long option - return "--%s %s" % (opt[1], default) + return '--{opt[1]} {default}'.format(opt=opt, default=default) globalopts = [ From ff15985c123bbffc91bc202b565057f66ff37aac Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 5 Feb 2016 17:34:34 +0800 Subject: [PATCH 026/114] [doc] Let command line usage regenerated from `couchapp help` [ci skip] --- docs/Makefile | 5 +++++ docs/couchapp/usage.rst | 46 +---------------------------------------- docs/couchapp/usage.txt | 45 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 docs/couchapp/usage.txt diff --git a/docs/Makefile b/docs/Makefile index a896bd1c..cd792d9d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -191,3 +191,8 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +usage: + @echo '.. GENERATED VIA ``couchapp help``' > couchapp/usage.txt + @echo '' >> couchapp/usage.txt + @couchapp help >> couchapp/usage.txt diff --git a/docs/couchapp/usage.rst b/docs/couchapp/usage.rst index ebe53673..b0ae7d58 100644 --- a/docs/couchapp/usage.rst +++ b/docs/couchapp/usage.rst @@ -18,51 +18,7 @@ Command Line Usage Full command line usage ----------------------- -:: - - Usage: couchapp [OPTIONS] [CMD] [CMDOPTIONS] [ARGS,...] - - Options: - -d, --debug - -h, --help - --version - -v, --verbose - -q, --quiet - - Commands: - autopush [OPTION]... [COUCHAPPDIR] DEST - --no-atomic send attachments one by one - --update-delay [VAL] time between each update - browse [COUCHAPPDIR] DEST - clone [OPTION]...[-r REV] SOURCE [COUCHAPPDIR] - -r, --rev [VAL] clone specific revision - generate [OPTION]... [app|view,list,show,filter,function,vendor] [COUCHAPPDIR] NAME - --template [VAL] template name - help - init [COUCHAPPDIR] - push [OPTION]... [COUCHAPPDIR] DEST - --no-atomic send attachments one by one - --export don't do push, just export doc to stdout - --output [VAL] if export is selected, output to the file - -b, --browse open the couchapp in the browser - --force force attachments sending - --docid [VAL] set docid - pushapps [OPTION]... SOURCE DEST - --no-atomic send attachments one by one - --export don't do push, just export doc to stdout - --output [VAL] if export is selected, output to the file - -b, --browse open the couchapp in the browser - --force force attachments sending - pushdocs [OPTION]... SOURCE DEST - --no-atomic send attachments one by one - --export don't do push, just export doc to stdout - --output [VAL] if export is selected, output to the file - -b, --browse open the couchapp in the browser - --force force attachments sending - startapp [COUCHAPPDIR] NAME - vendor [OPTION]...[-f] install|update [COUCHAPPDIR] SOURCE - -f, --force force install or update - version +.. literalinclude:: usage.txt Commands diff --git a/docs/couchapp/usage.txt b/docs/couchapp/usage.txt new file mode 100644 index 00000000..cf23fa57 --- /dev/null +++ b/docs/couchapp/usage.txt @@ -0,0 +1,45 @@ +.. GENERATED VIA ``couchapp help`` + +Usage: couchapp [OPTIONS] [CMD] [CMDOPTIONS] [ARGS,...] + +Options: + -d, --debug + -h, --help + --version + -v, --verbose + -q, --quiet + +Commands: + autopush [OPTION]... [COUCHAPPDIR] DEST + --no-atomic send attachments one by one + --update-delay [VAL] time between each update + browse [COUCHAPPDIR] DEST + clone [OPTION]...[-r REV] SOURCE [COUCHAPPDIR] + -r, --rev [VAL] clone specific revision + generate [OPTION]... [app|view,list,show,filter,function,vendor] [COUCHAPPDIR] NAME + --template [VAL] template name + help + init [COUCHAPPDIR] + push [OPTION]... [COUCHAPPDIR] DEST + --no-atomic send attachments one by one + --export don't do push, just export doc to stdout + --output [VAL] if export is selected, output to the file + -b, --browse open the couchapp in the browser + --force force attachments sending + --docid [VAL] set docid + pushapps [OPTION]... SOURCE DEST + --no-atomic send attachments one by one + --export don't do push, just export doc to stdout + --output [VAL] if export is selected, output to the file + -b, --browse open the couchapp in the browser + --force force attachments sending + pushdocs [OPTION]... SOURCE DEST + --no-atomic send attachments one by one + --export don't do push, just export doc to stdout + --output [VAL] if export is selected, output to the file + -b, --browse open the couchapp in the browser + --force force attachments sending + startapp [COUCHAPPDIR] NAME + vendor [OPTION]...[-f] install|update [COUCHAPPDIR] SOURCE + -f, --force force install or update + version From 69ef7282ce3dd792e08ba4261a52e73c58420272 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 18:32:22 +0800 Subject: [PATCH 027/114] [clone_app.py] fix warning to show missing function name --- couchapp/clone_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index d27bee7a..d1404a4d 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -306,7 +306,7 @@ def setup_func(self, func): filename = os.path.join(showpath, '{0}.js'.format(func_name)) util.write(filename, func) logger.warning( - 'clone function "{0}" not in manifest: {1}'.format(func, + 'clone function "{0}" not in manifest: {1}'.format(func_name, filename)) def setup_id(self): From 33760fc245bedc2bb6b3dccf5e84c8f16b696ac8 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 20:18:49 +0800 Subject: [PATCH 028/114] [util] deprecate `popen3`, new helper function: `sh_open` Though original implementation invoked subprocess.Popen, the Popen.wait cause intermittent test failures due to PIPE deadlock. In the new helper function, we use Popen.communicate instead. --- couchapp/util.py | 47 +++++++++++--------- couchapp/vendors/backends/git.py | 9 ++-- couchapp/vendors/backends/hg.py | 9 ++-- tests/test_cli.py | 75 +++++++++++++------------------- tests/test_util.py | 10 +++++ 5 files changed, 74 insertions(+), 76 deletions(-) diff --git a/couchapp/util.py b/couchapp/util.py index 511cbd1b..d033d050 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -6,15 +6,17 @@ from __future__ import with_statement import codecs -from hashlib import md5 import imp import inspect import logging import os import re import string +import subprocess import sys +from hashlib import md5 + from couchapp.errors import ScriptError try: @@ -32,21 +34,6 @@ logger = logging.getLogger(__name__) -try: # python 2.6, use subprocess - import subprocess - subprocess.Popen # trigger ImportError early - closefds = os.name == 'posix' - - def popen3(cmd, mode='t', bufsize=0): - p = subprocess.Popen(cmd, shell=True, bufsize=bufsize, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=closefds) - p.wait() - return (p.stdin, p.stdout, p.stderr) -except ImportError: - subprocess = None - popen3 = os.popen3 - try: from importlibe import import_module except ImportError: @@ -503,11 +490,10 @@ def __init__(self, cmd): def hook(self, *args, **options): cmd = self.cmd + " " - (child_stdin, child_stdout, child_stderr) = popen3(cmd) - err = child_stderr.read() - if err: - raise ScriptError(str(err)) - return (child_stdout.read()) + child_stdout, child_stderr = sh_open(cmd) + if child_stderr: + raise ScriptError(str(child_stderr)) + return child_stdout def hook_uri(uri, cfg): @@ -530,3 +516,22 @@ def replace(m): return "" return s return re.sub(re_comment, replace, t) + + +def sh_open(cmd, bufsize=0): + ''' + run shell command with :mod:`subprocess` + + :param str cmd: the command string + :param int bufsize: the bufsize passed to ``subprocess.Popen`` + :return: a tuple contains (stdout, stderr) + ''' + closefds = (os.name == 'posix') + + p = subprocess.Popen(cmd, shell=True, bufsize=bufsize, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=closefds) + # use ``communicate`` to avoid PIPE deadlock + out, err = p.communicate() + + return (out, err) diff --git a/couchapp/vendors/backends/git.py b/couchapp/vendors/backends/git.py index b9674ec4..897717d1 100644 --- a/couchapp/vendors/backends/git.py +++ b/couchapp/vendors/backends/git.py @@ -6,7 +6,7 @@ import logging from couchapp.errors import VendorError -from couchapp.util import locate_program, popen3 +from couchapp.util import locate_program, sh_open from couchapp.vendors.backends.base import BackendVendor logger = logging.getLogger(__name__) @@ -37,8 +37,7 @@ def fetch(self, url, path, *args, **opts): cmd += " clone %s %s" % (url, path) # exec cmd - (child_stdin, child_stdout, child_stderr) = popen3(cmd) - err = child_stderr.read() - if err: - raise VendorError(str(err)) + child_stdout, child_stderr = sh_open(cmd) + if child_stderr: + raise VendorError(str(child_stderr)) logger.debug(child_stdout.read()) diff --git a/couchapp/vendors/backends/hg.py b/couchapp/vendors/backends/hg.py index f90ee9b4..e7c8e1a9 100644 --- a/couchapp/vendors/backends/hg.py +++ b/couchapp/vendors/backends/hg.py @@ -6,7 +6,7 @@ import logging from couchapp.errors import VendorError -from couchapp.util import locate_program, popen3 +from couchapp.util import locate_program, sh_open from couchapp.vendors.backends.base import BackendVendor logger = logging.getLogger(__name__) @@ -38,9 +38,8 @@ def fetch(self, url, path, *args, **opts): cmd += " clone %s %s" % (url, path) # exec cmd - (child_stdin, child_stdout, child_stderr) = popen3(cmd) - err = child_stderr.read() - if err: - raise VendorError(str(err)) + child_stdout, child_stderr = sh_open(cmd) + if child_stderr: + raise VendorError(str(child_stderr)) logger.debug(child_stdout.read()) diff --git a/tests/test_cli.py b/tests/test_cli.py index 42c704b9..b0e9cb64 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,7 @@ from couchapp.errors import ResourceNotFound from couchapp.client import Database -from couchapp.util import popen3, deltree +from couchapp.util import deltree, sh_open couchapp_dir = os.path.join(os.path.dirname(__file__), '../') couchapp_cli = os.path.join(os.path.dirname(__file__), '../bin/couchapp') @@ -64,8 +64,8 @@ def _retrieve_ddoc(self): def testGenerate(self): os.chdir(self.tempdir) - (child_stdin, child_stdout, child_stderr) = popen3("%s generate my-app" - % self.cmd) + child_stdout, child_stderr = sh_open( + '{0} generate my-app'.format(self.cmd)) appdir = os.path.join(self.tempdir, 'my-app') self.assertTrue(os.path.isdir(appdir)) cfile = os.path.join(appdir, '.couchapprc') @@ -83,8 +83,8 @@ def testGenerate(self): def testPush(self): self._make_testapp() - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s push -v my-app %scouchapp-test" % (self.cmd, url)) + child_stdout, child_stderr = sh_open( + '{0} push -v my-app {1}couchapp-test'.format(self.cmd, url)) design_doc = self._retrieve_ddoc() @@ -120,9 +120,8 @@ def testPush(self): def testPushNoAtomic(self): self._make_testapp() - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s push --no-atomic my-app %scouchapp-test" % (self.cmd, - url)) + child_stdout, child_stderr = sh_open( + '{0} push --no-atomic my-app {1}couchapp-test'.format(self.cmd, url)) design_doc = self._retrieve_ddoc() @@ -159,18 +158,15 @@ def testPushNoAtomic(self): def testClone(self): self._make_testapp() - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s push -v my-app %scouchapp-test" % (self.cmd, url)) + child_stdout, child_stderr = sh_open( + '{0} push -v my-app {1}couchapp-test'.format(self.cmd, url)) design_doc = self._retrieve_ddoc() app_dir = os.path.join(self.tempdir, "couchapp-test") - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s clone %s %s" - % (self.cmd, - url + "couchapp-test/_design/my-app", - app_dir)) + child_stdout, child_stderr = sh_open('{0} clone {1} {2}'.format( + self.cmd, url + 'couchapp-test/_design/my-app', app_dir)) # should create .couchapprc self.assertTrue(os.path.isfile(os.path.join(app_dir, ".couchapprc"))) @@ -191,11 +187,8 @@ def testClone(self): design_doc = self.db.save_doc(design_doc) deltree(app_dir) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s clone %s %s" - % (self.cmd, - url + "couchapp-test/_design/my-app", - app_dir)) + child_stdout, child_stderr = sh_open('{0} clone {1} {2}'.format( + self.cmd, url + 'couchapp-test/_design/my-app', app_dir)) self.assertTrue(os.path.isfile(os.path.join(app_dir, 'test.txt'))) # should work when a view is added manually @@ -205,11 +198,8 @@ def testClone(self): design_doc = self.db.save_doc(design_doc) deltree(app_dir) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s clone %s %s" - % (self.cmd, - url + "couchapp-test/_design/my-app", - app_dir)) + child_stdout, child_stderr = sh_open('{0} clone {1} {2}'.format( + self.cmd, url + 'couchapp-test/_design/my-app', app_dir)) self.assertTrue(os.path.isfile(os.path.join(app_dir, 'views/example/map.js'))) @@ -217,11 +207,8 @@ def testClone(self): del design_doc['couchapp']['manifest'] design_doc = self.db.save_doc(design_doc) deltree(app_dir) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s clone %s %s" - % (self.cmd, - url + "couchapp-test/_design/my-app", - app_dir)) + child_stdout, child_stderr = sh_open('{0} clone {1} {2}'.format( + self.cmd, url + 'couchapp-test/_design/my-app', app_dir)) self.assertTrue(os.path.isfile(os.path.join(app_dir, 'views/example/map.js'))) @@ -238,14 +225,13 @@ def testPushApps(self): os.makedirs(docsdir) # create 2 apps - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s generate docs/app1" % self.cmd) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s generate docs/app2" % self.cmd) + child_stdout, child_stderr = sh_open( + '{0} generate docs/app1'.format(self.cmd)) + child_stdout, child_stderr = sh_open( + '{0} generate docs/app2'.format(self.cmd)) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s pushapps docs/ %scouchapp-test" - % (self.cmd, url)) + child_stdout, child_stderr = sh_open( + '{0} pushapps docs/ {1}couchapp-test'.format(self.cmd, url)) alldocs = self.db.all_docs()['rows'] self.assertEqual(len(alldocs), 2) @@ -257,14 +243,13 @@ def testPushDocs(self): os.makedirs(docsdir) # create 2 apps - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s generate docs/app1" % self.cmd) - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s generate docs/app2" % self.cmd) - - (child_stdin, child_stdout, child_stderr) = \ - popen3("%s pushdocs docs/ %scouchapp-test" - % (self.cmd, url)) + child_stdout, child_stderr = sh_open( + '{0} generate docs/app1'.format(self.cmd)) + child_stdout, child_stderr = sh_open( + '{0} generate docs/app2'.format(self.cmd)) + + child_stdout, child_stderr = sh_open( + '{0} pushdocs docs/ {1}couchapp-test'.format(self.cmd, url)) alldocs = self.db.all_docs()['rows'] diff --git a/tests/test_util.py b/tests/test_util.py index 20f57624..714669f3 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,6 +3,7 @@ import os from couchapp.util import discover_apps, iscouchapp, rcpath, split_path +from couchapp.util import sh_open from mock import patch @@ -103,3 +104,12 @@ def test_split_path_abs(): ''' path = os.path.realpath('/foo/bar') assert split_path(path) == [os.path.realpath('/foo'), 'bar'] + + +def test_sh_open(): + ''' + Test case for ``util.sh_open`` + ''' + out, err = sh_open('echo mock') + assert out.startswith('mock'), out + assert not err, err From 26ce6555d8ff0cb977d1b5850805243665534c2a Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 20:37:05 +0800 Subject: [PATCH 029/114] [util] Fix typo for `importlib` --- couchapp/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchapp/util.py b/couchapp/util.py index 511cbd1b..ac11f106 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -48,7 +48,7 @@ def popen3(cmd, mode='t', bufsize=0): popen3 = os.popen3 try: - from importlibe import import_module + from importlib import import_module except ImportError: def _resolve_name(name, package, level): """Return the absolute name of the module to be imported.""" From cac76afe4a57618c48042dcecc086408e1af3f66 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 22:45:19 +0800 Subject: [PATCH 030/114] [commands] Fix #164, `pushapps --export` do not require `DEST` --- couchapp/commands.py | 5 +++-- tests/test_commands.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index f96ab572..60ceec81 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -81,11 +81,11 @@ def push(conf, path, *args, **opts): return 0 -def pushapps(conf, source, dest, *args, **opts): +def pushapps(conf, source, dest=None, *args, **opts): export = opts.get('export', False) noatomic = opts.get('no_atomic', False) browse = opts.get('browse', False) - dbs = conf.get_dbs(dest) + dbs = conf.get_dbs(dest) if not export else None apps = [] source = os.path.normpath(os.path.join(os.getcwd(), source)) appdirs = util.discover_apps(source) @@ -94,6 +94,7 @@ def pushapps(conf, source, dest, *args, **opts): for appdir in appdirs: doc = document(appdir) + # if export mode, the ``dbs`` will be None hook(conf, appdir, "pre-push", dbs=dbs, pushapps=True) if export or not noatomic: apps.append(doc) diff --git a/tests/test_commands.py b/tests/test_commands.py index 4b647d87..e91d9096 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -160,9 +160,9 @@ def test_pushapps_output(discover_apps_, hook, document_, write_json): assert ret_code == 0 discover_apps_.assert_called_with('/mock_dir') hook.assert_any_call(conf, 'foo', 'pre-push', - dbs=conf.get_dbs(), pushapps=True) + dbs=None, pushapps=True) hook.assert_any_call(conf, 'foo', 'post-push', - dbs=conf.get_dbs(), pushapps=True) + dbs=None, pushapps=True) 'file' in write_json.call_args[0] @@ -212,9 +212,9 @@ def test_pushapps_export(discover_apps_, hook, document_, dumps): assert ret_code == 0 discover_apps_.assert_called_with('/mock_dir') hook.assert_any_call(conf, 'foo', 'pre-push', - dbs=conf.get_dbs(), pushapps=True) + dbs=None, pushapps=True) hook.assert_any_call(conf, 'foo', 'post-push', - dbs=conf.get_dbs(), pushapps=True) + dbs=None, pushapps=True) assert dumps.called From ed38f12a9d4d6dc3622a489e22b9db46e1c48876 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Thu, 11 Feb 2016 12:17:06 +0800 Subject: [PATCH 031/114] [util] Fix typo --- couchapp/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchapp/util.py b/couchapp/util.py index fac19ae8..8c83e3ea 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -79,8 +79,8 @@ def user_rcpath(): try: home = os.path.expanduser('~') if sys.getwindowsversion()[3] != 2 and home == '~': - # We are on win < nt: fetch the APPDATA directory location and - # use the parent directory as the user home dir. + # We are on win < nt: fetch the APPDATA directory location and + # use the parent directory as the user home dir. appdir = shell.SHGetPathFromIDList( shell.SHGetSpecialFolderLocation(0, shellcon.CSIDL_APPDATA)) From 8e244899b111fbabda1c736deafe1692ec74305e Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 9 Feb 2016 13:57:33 +0800 Subject: [PATCH 032/114] [setup.py] make py2exe build with `socketpool` ans `pathtools` ref #224 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c83e49f6..09492263 100644 --- a/setup.py +++ b/setup.py @@ -149,7 +149,9 @@ def main(): 'packages': ["http_parser", "restkit", "restkit.contrib", + "pathtools", "pathtools.path", + "socketpool", "watchdog", "watchdog.observers", "watchdog.tricks", From 714991b5437853ec9f5850604b369dc69df9c351 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 13:43:12 +0800 Subject: [PATCH 033/114] [appveyor] test cases for windows standalone exe --- appveyor.yml | 10 +++++++--- tests/test_cli.py | 5 ++++- tests/test_exe.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 tests/test_exe.py diff --git a/appveyor.yml b/appveyor.yml index 3ec7dadc..d051475d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,16 +69,20 @@ build: false # Not a C# project, build stuff at the test step instead. test_script: - python setup.py install - # - nosetests -v --with-coverage --cover-package=couchapp --cover-html -after_test: # Build wheel - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" - # Build standalone binary via py2exe and Inno Setup + # Build standalone binary via py2exe - "%WITH_COMPILER% %PYTHON%/python setup.py build" - python -m py2exe.mf -d resources\scripts\couchapp - python setup.py py2exe + - ls dist\ + + - "nosetests --tests=tests\\test_exe.py:WinExeTestCase" + +after_test: + # Pack ``dist`` via Inno Setup - copy add_path.exe dist\ - ps: ($Env:COUCHAPP_VER = python -c 'import couchapp; print(couchapp.__version__)') - ps: ($Env:COMMIT = $Env:APPVEYOR_REPO_COMMIT.Substring(0, 7)) diff --git a/tests/test_cli.py b/tests/test_cli.py index b0e9cb64..b871bc73 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -40,7 +40,6 @@ def setUp(self): self.tempdir = _tempdir() os.makedirs(self.tempdir) self.app_dir = os.path.join(self.tempdir, "my-app") - self.cmd = "cd %s && couchapp" % self.tempdir self.startdir = os.getcwd() def tearDown(self): @@ -48,6 +47,10 @@ def tearDown(self): deltree(self.tempdir) os.chdir(self.startdir) + @property + def cmd(self): + return 'cd {0} && couchapp'.format(self.tempdir) + def _make_testapp(self): testapp_path = os.path.join(os.path.dirname(__file__), 'testapp') shutil.copytree(testapp_path, self.app_dir) diff --git a/tests/test_exe.py b/tests/test_exe.py new file mode 100644 index 00000000..2eb0b676 --- /dev/null +++ b/tests/test_exe.py @@ -0,0 +1,33 @@ +import os +import sys + +from test_cli import CliTestCase as _CliTestCase + +from nose.plugins.skip import SkipTest + + +if sys.platform != 'win32': + raise SkipTest('windows only testing') + + +class WinExeTestCase(_CliTestCase): + + def __init__(self, *args, **kwargs): + super(WinExeTestCase, self).__init__(*args, **kwargs) + + # check for executable + self.exe + + @property + def exe(self): + exe = os.path.join(os.path.dirname(os.path.realpath(__file__)), + '..', 'dist', 'couchapp.exe') + if not os.path.exists(exe): + raise SkipTest('Windows standalone executable not found ' + 'in {0}'.format(exe)) + return exe + + @property + def cmd(self): + return 'cd {tempdir} && {exe}'.format( + tempdir=self.tempdir, exe=self.exe) From 7c779e574a0802a25f9e10bd84e9a5c5164a8ea1 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 24 Nov 2015 20:29:10 +0800 Subject: [PATCH 034/114] [#180] Test cases for `clone.init_metadata` --- couchapp/clone_app.py | 2 +- tests/test_clone_app.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index d1404a4d..27ed80ad 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -93,7 +93,7 @@ def init_metadata(self): ''' metadata = self.doc.get('couchapp', {}) - self.manifest = metadata.get('manifest', {}) + self.manifest = metadata.get('manifest', []) self.signatures = metadata.get('signatures', {}) self.objects = metadata.get('objects', {}) diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 03d867e3..c23e3a4f 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -78,3 +78,37 @@ def test_setup_path_dest(self, setup_dir, getcwd): assert getcwd.called assert self.clone.path == '/tmp/mock' setup_dir.assert_called_with('/tmp/mock') + + def test_init_metadata(self): + ''' + Test case for extract metadata from a ddoc + ''' + self.clone.doc = { + 'couchapp': { + 'manifest': ['views/'], + 'signatures': {'mock': 'strange_value'}, + 'objects': {'obj_hash': 'obj'} + } + } + + self.clone.init_metadata() + + assert 'views/' in self.clone.manifest + assert 'mock' in self.clone.signatures + assert 'obj_hash' in self.clone.objects + + def test_init_metadata_default(self): + ''' + Test case for extract metadata from an empty ddoc + + Check the default contain correct type + ''' + self.clone.doc = { + 'couchapp': {} + } + + self.clone.init_metadata() + + assert self.clone.manifest == [] + assert self.clone.signatures == {} + assert self.clone.objects == {} From d445c210eab79effb9f15d5881083686a2661c38 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 12 Feb 2016 12:22:33 +0800 Subject: [PATCH 035/114] [util] Test case for `util.remove_comments` --- couchapp/util.py | 22 ++++++++++++++++------ tests/test_util.py | 24 +++++++++++++++++++++++- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/couchapp/util.py b/couchapp/util.py index 8c83e3ea..05eea958 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -505,17 +505,27 @@ def hook_uri(uri, cfg): script_uri = uri return ShellScript(script_uri) -regex_comment = r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"' -re_comment = re.compile(regex_comment, re.DOTALL | re.MULTILINE) +RE_COMMENT = re.compile( + r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"' + , re.DOTALL | re.MULTILINE) -def remove_comments(t): + +def remove_comments(text): + ''' + remove comments string in json text + + :param str text: the json text + ''' def replace(m): + ''' + :param m: the regex match object + ''' s = m.group(0) - if s.startswith("/"): - return "" + if s.startswith('/'): + return '' return s - return re.sub(re_comment, replace, t) + return re.sub(RE_COMMENT, replace, text) def sh_open(cmd, bufsize=0): diff --git a/tests/test_util.py b/tests/test_util.py index 714669f3..793fd3e4 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,7 +3,7 @@ import os from couchapp.util import discover_apps, iscouchapp, rcpath, split_path -from couchapp.util import sh_open +from couchapp.util import sh_open, remove_comments from mock import patch @@ -113,3 +113,25 @@ def test_sh_open(): out, err = sh_open('echo mock') assert out.startswith('mock'), out assert not err, err + + +def test_remove_comments(): + text = '''{ + "mock": 42 // truth +}''' + expect = '{\n "mock": 42 \n}' + + ret = remove_comments(text) + assert ret == expect + + # testing for multiline comments + text = '''{ + "mock": 42, // truth + /* foo + bar + */ + "fake": true}''' + ret = remove_comments(text) + expect = '{\n "mock": 42, \n \n "fake": true}' + + assert ret == expect From bc1e1ea9b06e1058dd96b772a075ad768d332a69 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 10 Feb 2016 13:43:12 +0800 Subject: [PATCH 036/114] [appveyor] cleanup useless script --- appveyor.yml | 9 ++++++++- tests/test_exe.py | 33 --------------------------------- 2 files changed, 8 insertions(+), 34 deletions(-) delete mode 100644 tests/test_exe.py diff --git a/appveyor.yml b/appveyor.yml index d051475d..e9eb3207 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -77,9 +77,16 @@ test_script: - "%WITH_COMPILER% %PYTHON%/python setup.py build" - python -m py2exe.mf -d resources\scripts\couchapp - python setup.py py2exe + + # Add path for couchapp.exe - ls dist\ + - "SET PATH=%CD%\\dist;%PATH%" + + # prepare templates dir for testing + - ps: cp -R dist\couchapp\templates dist\templates - - "nosetests --tests=tests\\test_exe.py:WinExeTestCase" + # tests for standalone binary + - "nosetests --tests=tests\\test_cli.py" after_test: # Pack ``dist`` via Inno Setup diff --git a/tests/test_exe.py b/tests/test_exe.py deleted file mode 100644 index 2eb0b676..00000000 --- a/tests/test_exe.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import sys - -from test_cli import CliTestCase as _CliTestCase - -from nose.plugins.skip import SkipTest - - -if sys.platform != 'win32': - raise SkipTest('windows only testing') - - -class WinExeTestCase(_CliTestCase): - - def __init__(self, *args, **kwargs): - super(WinExeTestCase, self).__init__(*args, **kwargs) - - # check for executable - self.exe - - @property - def exe(self): - exe = os.path.join(os.path.dirname(os.path.realpath(__file__)), - '..', 'dist', 'couchapp.exe') - if not os.path.exists(exe): - raise SkipTest('Windows standalone executable not found ' - 'in {0}'.format(exe)) - return exe - - @property - def cmd(self): - return 'cd {tempdir} && {exe}'.format( - tempdir=self.tempdir, exe=self.exe) From 33c622a85400934005274d0abc5350ef7685469b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 22 Jan 2016 15:54:40 +0800 Subject: [PATCH 037/114] [#180] Separate logical block Code refinement and test cases for clone.setup_manifest The following functions separated from ``setup_manifest``: - extract_property: given a path str in manifest, return the property content. - pop_doc: given a list of traversal node, pop the content from doc. --- couchapp/clone_app.py | 144 +++++++++++++++++++---------- couchapp/errors.py | 4 + tests/test_clone_app.py | 197 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 297 insertions(+), 48 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 27ed80ad..5baf2595 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -11,7 +11,7 @@ from hashlib import md5 from couchapp import client, util -from couchapp.errors import AppError +from couchapp.errors import AppError, MissingContent logger = logging.getLogger(__name__) @@ -109,6 +109,9 @@ def setup_manifest(self): "bar.json" ] ``` + + Once we create a file successfully, we will remove the record from + ``self.doc``. ''' if not self.manifest: return @@ -118,8 +121,7 @@ def setup_manifest(self): filepath = os.path.join(self.path, filename) if filename.endswith('/'): # create dir - if not os.path.isdir(filepath): - os.makedirs(filepath) + self.setup_dir(filepath) continue elif filename == 'couchapp.json': # we will handle it later continue @@ -128,50 +130,98 @@ def setup_manifest(self): parts = util.split_path(filename) fname = parts.pop() v = self.doc - while 1: - try: - for key in parts: - v = v[key] - except KeyError: - break - - # remove extension - last_key, ext = os.path.splitext(fname) - - # make sure key exist - try: - content = v[last_key] - except KeyError: - break - - if isinstance(content, basestring): - _ref = md5(util.to_bytestring(content)).hexdigest() - if self.objects and _ref in self.objects: - content = self.objects[_ref] - - if content.startswith('base64-encoded;'): - content = base64.b64decode(content[15:]) - - if fname.endswith('.json'): - content = util.json.dumps(content).encode('utf-8') - - del v[last_key] - - # make sure file dir have been created - filedir = os.path.dirname(filepath) - if not os.path.isdir(filedir): - os.makedirs(filedir) - - util.write(filepath, content) - - # remove the key from design doc - temp = self.doc - for key2 in parts: - if key2 == key: - if not temp[key2]: - del temp[key2] - break - temp = temp[key2] + + item_pair = self.extract_property(filename) + + if item_pair is None: + continue + _, content = item_pair + + if isinstance(content, basestring): + _ref = md5(util.to_bytestring(content)).hexdigest() + if self.objects and _ref in self.objects: + content = self.objects[_ref] + + if content.startswith('base64-encoded;'): + content = base64.b64decode(content[15:]) + + if fname.endswith('.json'): + content = util.json.dumps(content).encode('utf-8') + + # make sure file dir have been created + filedir = os.path.dirname(filepath) + if not os.path.isdir(filedir): + os.makedirs(filedir) + + util.write(filepath, content) + + def extract_property(self, path): + ''' + Extract the content from ``self.doc``. + Given a listed path in ``self.manifest``, we travel the ``self.doc``. + + Assume we have ``views/some_func/map.js`` in our ``self.manifest`` + Then, there should exist following struct in ``self.doc`` + { + ... + "views": { + "some_func": { + "map": "..." + } + } + ... + } + + :side effect: Remove key from ``self.doc`` if extract sucessfully. + + :return: The ``(content,)`` pair. Note that + if we get path ``a`` and ``{'a': null}``, + then the return will be ``('a', None)``. + + If the extraction failed, return ``None`` + ''' + if not path: + return None + + try: + content = self.pop_doc(util.split_path(path), self.doc) + except MissingContent: + logger.warning( + 'file {0} listed in mastfest missed content'.format(path)) + return None + return (path, content) + + def pop_doc(self, path, doc): + ''' + do doc recursive traversal, and pop the value + + :param path: the list from ``util.split_path`` + :side effect: Remove key from ``self.doc`` if extract sucessfully. + :return: the value. If we pop failed, + raise ``couchapp.errors.MissingContent``. + ''' + try: + head, tail = path[0], path[1:] + except IndexError: # path is empty + raise MissingContent() + + if not tail: # the leaf node of path + prop, _ = os.path.splitext(head) + if prop in doc: + return doc.pop(prop) + else: + raise MissingContent() + + subdoc = doc.get(head, None) + if not isinstance(subdoc, dict): + raise MissingContent() + + ret = self.pop_doc(tail, subdoc) + + if not subdoc: # after subdoc.pop(), if the subdoc is empty + del doc[head] + + return ret def setup_missing(self): ''' diff --git a/couchapp/errors.py b/couchapp/errors.py index 4f303541..4305dd58 100644 --- a/couchapp/errors.py +++ b/couchapp/errors.py @@ -58,3 +58,7 @@ class ScriptError(Exception): class InvalidAttachment(Exception): """ raised when attachment is invalid (bad size, ct, ..)""" + + +class MissingContent(Exception): + """ raised when the clone_app extract content from property failed""" diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index c23e3a4f..5fc1fb22 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from couchapp.clone_app import clone -from couchapp.errors import AppError +from couchapp.errors import AppError, MissingContent from mock import patch from nose.tools import raises @@ -112,3 +112,198 @@ def test_init_metadata_default(self): assert self.clone.manifest == [] assert self.clone.signatures == {} assert self.clone.objects == {} + + def test_pop_doc_str(self): + ''' + Test case for pop str from ``clone.doc`` + ''' + path = ['mock.json'] + doc = { + 'mock': 'fake_data', + 'other': None + } + ret = self.clone.pop_doc(path, doc) + assert ret == 'fake_data' + assert doc == {'other': None} + + def test_pop_doc_unicode(self): + ''' + Test case for pop unicode from ``clone.doc`` + ''' + path = ['mock.json'] + doc = { + u'mock': u'fake_data' + } + ret = self.clone.pop_doc(path, doc) + assert ret == u'fake_data' + assert doc == {} + + path = [u'mock.json'] + doc = { + u'mock': u'fake_data' + } + ret = self.clone.pop_doc(path, doc) + assert ret == u'fake_data' + assert doc == {} + + path = [u'mock.json'] + doc = { + 'mock': u'fake_data' + } + ret = self.clone.pop_doc(path, doc) + assert ret == u'fake_data' + assert doc == {} + + def test_pop_doc_int(self): + ''' + Test case for pop int from ``clone.doc`` + ''' + path = ['truth.json'] + doc = { + 'truth': 42 + } + ret = self.clone.pop_doc(path, doc) + assert isinstance(ret, int) + assert ret == 42 + assert doc == {} + + def test_pop_doc_float(self): + ''' + Test case for pop float from ``clone.doc`` + ''' + path = ['truth.json'] + doc = { + 'truth': 42.0 + } + ret = self.clone.pop_doc(path, doc) + assert isinstance(ret, float) + assert ret == 42.0 + assert doc == {} + + def test_pop_doc_list(self): + ''' + Test case for pop list from ``clone.doc`` + ''' + path = ['mock.json'] + doc = { + 'mock': ['foo', 'bar'] + } + ret = self.clone.pop_doc(path, doc) + assert ret == ['foo', 'bar'] + assert doc == {} + + def test_pop_doc_none(self): + ''' + Test case for pop ``None`` from ``clone.doc`` + + Note that the None come from ``null`` value in json. + ''' + path = ['mock.json'] + doc = { + 'mock': None + } + ret = self.clone.pop_doc(path, doc) + assert ret is None + assert doc == {} + + def test_pop_doc_bool(self): + ''' + Test case for pop boolean from ``clone.doc`` + ''' + path = ['mock.json'] + doc = { + 'mock': True + } + ret = self.clone.pop_doc(path, doc) + assert ret is True + assert doc == {} + + def test_pop_doc_deep(self): + ''' + Test case for pop data from deeper prop of ``clone.doc`` + ''' + path = ['shows', 'mock.js'] + doc = { + 'shows': { + 'mock': u'fake_script' + } + } + ret = self.clone.pop_doc(path, doc) + assert ret == u'fake_script' + assert doc == {} + + path = ['shows.lol', 'mock.bak.js'] + doc = { + 'shows.lol': { + 'mock.bak': u'fake_script' + } + } + ret = self.clone.pop_doc(path, doc) + assert ret == u'fake_script' + assert doc == {} + + @raises(MissingContent) + def test_pop_doc_miss(self): + ''' + Test case for pop a missing prop + ''' + path = ['fake.json'] + doc = { + 'mock': 'mock_data' + } + + self.clone.pop_doc(path, doc) + assert doc == {'mock': 'mock_data'} + + @raises(MissingContent) + def test_pop_doc_wrong_prop(self): + ''' + Test case for pop a wrong prop + ''' + path = ['mock', 'fake.json'] + doc = { + 'mock': ['fake'] + } + + self.clone.pop_doc(path, doc) + + @raises(MissingContent) + def test_pop_doc_empty_args(self): + ''' + Test case for ``clone.pop_doc`` with empty args + ''' + doc = { + 'mock': 'fake' + } + self.clone.pop_doc([], doc) + + def test_extract_property_empty_args(self): + ''' + Test case for ``extract_property`` with empty args + ''' + self.clone.doc = { + 'mock': 'no_effect' + } + + assert self.clone.extract_property('') is None + assert self.clone.doc == {'mock': 'no_effect'} + + @patch('couchapp.clone_app.clone.pop_doc', return_value='mock_content') + def test_extract_property(self, pop_doc): + ''' + Test case for extract prop successfully + ''' + self.clone.doc = {} + + ret = self.clone.extract_property('mock') + + assert ret == ('mock', 'mock_content') + assert pop_doc.called + + @patch('couchapp.clone_app.clone.pop_doc', side_effect=MissingContent) + def test_extract_property_fail(self, pop_doc): + ''' + Test case for extract prop failed + ''' + self.clone.doc = {} + assert self.clone.extract_property('mock') is None From e1591f2ea98ef06b05df9afe83a9486921c546d0 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 22 Jan 2016 19:48:17 +0800 Subject: [PATCH 038/114] [#180] Separate logical block The function came from `setup_manifest`: - `decode_content`: given a content, resolve it if base64 encoded or refer to `doc.objects`. - `dump_file`: write the content to file --- couchapp/clone_app.py | 58 ++++++++++++++++++++++--------------- tests/test_clone_app.py | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 5baf2595..012977c9 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -127,33 +127,12 @@ def setup_manifest(self): continue # create file - parts = util.split_path(filename) - fname = parts.pop() - v = self.doc - item_pair = self.extract_property(filename) - if item_pair is None: continue _, content = item_pair - if isinstance(content, basestring): - _ref = md5(util.to_bytestring(content)).hexdigest() - if self.objects and _ref in self.objects: - content = self.objects[_ref] - - if content.startswith('base64-encoded;'): - content = base64.b64decode(content[15:]) - - if fname.endswith('.json'): - content = util.json.dumps(content).encode('utf-8') - - # make sure file dir have been created - filedir = os.path.dirname(filepath) - if not os.path.isdir(filedir): - os.makedirs(filedir) - - util.write(filepath, content) + self.dump_file(filepath, self.decode_content(content)) def extract_property(self, path): ''' @@ -174,7 +153,7 @@ def extract_property(self, path): :side effect: Remove key from ``self.doc`` if extract sucessfully. - :return: The ``(content,)`` pair. Note that + :return: The ``(path, content)`` pair. Note that if we get path ``a`` and ``{'a': null}``, then the return will be ``('a', None)``. @@ -223,6 +202,39 @@ def pop_doc(self, path, doc): return ret + def decode_content(self, content): + ''' + Decode for base64 string, or get the content refered via objects + ''' + if not isinstance(content, basestring): + return content + + _ref = md5(util.to_bytestring(content)).hexdigest() + if self.objects and _ref in self.objects: + content = self.objects[_ref] + + if content.startswith('base64-encoded;'): + content = base64.b64decode(content[15:]) + + return content + + def dump_file(self, path, content): + ''' + Dump the content of doc to file + ''' + if not path: + return + + if path.endswith('.json'): + content = util.json.dumps(content).encode('utf-8') + + # make sure file dir have been created + filedir = os.path.dirname(path) + if not os.path.isdir(filedir): + self.setup_dir(filedir) + + util.write(path, content) + def setup_missing(self): ''' second pass for missing key or in case manifest isn't in app. diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 5fc1fb22..83bd00fb 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -307,3 +307,66 @@ def test_extract_property_fail(self, pop_doc): ''' self.clone.doc = {} assert self.clone.extract_property('mock') is None + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_dump_file_empty_args(self, util_write, setup_dir): + ''' + Test case for dump_file with empty ``path`` + ''' + self.clone.objects = {} + + assert self.clone.dump_file('', 'mock') is None + assert not util_write.called + assert not setup_dir.called + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_dump_file_json_str(self, util_write, setup_dir): + ''' + Test case for dump_file to json file with normal str + ''' + self.clone.objects = {} + + ret = self.clone.dump_file('/mock/fake.json', 'foobar\n') + + assert setup_dir.called + assert util_write.call_args_list[0][0] == ('/mock/fake.json', + '"foobar\\n"') + + def test_decode_content_str(self): + ''' + Test case for decode_content with normal str + ''' + self.clone.objects = {} + + assert self.clone.decode_content('foobar\n') == 'foobar\n' + + def test_decode_content_base64(self): + ''' + Test case for decode_content with base64 str + ''' + self.clone.objects = {} + b64_content = 'base64-encoded;Zm9vYmFyCg==' # foobar\n + + assert self.clone.decode_content(b64_content) == 'foobar\n' + + def test_decode_content_objects(self): + ''' + Test case for decode_content with content refered to ``objects`` + ''' + self.clone.objects = { + '17404a596cbd0d1e6c7d23fcd845ab82': 'mock_data' + } + + assert self.clone.decode_content('mock') == 'mock_data' + + def test_decode_content_non_str(self): + ''' + Test case for decode_content with non-str + ''' + assert self.clone.decode_content(42) == 42 + assert self.clone.decode_content(1.1) == 1.1 + assert self.clone.decode_content(['foo']) == ['foo'] + assert self.clone.decode_content(None) is None + assert self.clone.decode_content(True) is True From 951b58d68c695d78b3cfeb6ffe06a20a0926d293 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 22 Jan 2016 23:07:06 +0800 Subject: [PATCH 039/114] [#180] Test cases for ``clone.setup_manifest`` --- tests/test_clone_app.py | 85 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 83bd00fb..c694a279 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -370,3 +370,88 @@ def test_decode_content_non_str(self): assert self.clone.decode_content(['foo']) == ['foo'] assert self.clone.decode_content(None) is None assert self.clone.decode_content(True) is True + + @patch('couchapp.clone_app.clone.dump_file') + def test_setup_manifest_empty(self, dump_file): + ''' + Test case for setup_manifest with empty ``manifest`` + ''' + self.clone.manifest = [] + self.clone.setup_manifest() + assert not dump_file.called + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.dump_file') + @patch('couchapp.clone_app.clone.extract_property') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_manifest_dirs(self, setup_dir, extract_property, + dump_file, decode_content): + ''' + Test case for setup_manifest with dirs + ''' + self.clone.path = '/mock' + self.clone.manifest = ['views/', 'views/mockview/', + 'shows/', '_attachments/'] + + self.clone.setup_manifest() + assert setup_dir.call_count == 4 + assert not extract_property.called + assert not decode_content.called + assert not dump_file.called + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.extract_property') + @patch('couchapp.clone_app.clone.dump_file') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_manifest_couchapp_json(self, setup_dir, dump_file, + extract_property, decode_content): + ''' + Test case for setup_manifest with ``couchapp.json`` + ''' + self.clone.path = '/mock' + self.clone.manifest = ['couchapp.json'] + + self.clone.setup_manifest() + assert not setup_dir.called + assert not extract_property.called + assert not decode_content.called + assert not dump_file.called + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.extract_property') + @patch('couchapp.clone_app.clone.dump_file') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_manifest_create_files_fail(self, setup_dir, dump_file, + extract_property, decode_content): + ''' + Test case for setup_manifest create files failed + ''' + self.clone.path = '/mock' + self.clone.manifest = ['mock.json', 'fake.txt'] + extract_property.return_value = None + + self.clone.setup_manifest() + assert not setup_dir.called + assert extract_property.call_count == 2 + assert not decode_content.called + assert not dump_file.called + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.extract_property') + @patch('couchapp.clone_app.clone.dump_file') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_manifest_create_files(self, setup_dir, dump_file, + extract_property, decode_content): + ''' + Test case for setup_manifest create files + ''' + + self.clone.path = '/mock' + self.clone.manifest = ['mock.json', 'fake.txt'] + extract_property.return_value = ('mock', 'joke') + + self.clone.setup_manifest() + assert not setup_dir.called + assert extract_property.call_count == 2 + assert dump_file.called + assert decode_content.called From 07fdb4495037e027a5d0b114a500dbc97db903e3 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 12:40:17 +0800 Subject: [PATCH 040/114] [clone_app] Fix key comparson, close #189 --- couchapp/clone_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 012977c9..1ba438b5 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -242,9 +242,9 @@ def setup_missing(self): for key in self.doc.iterkeys(): if key.startswith('_'): continue - elif key in ('couchapp'): + elif key in ('couchapp',): self.setup_couchapp_json() - elif key in ('views'): + elif key in ('views',): self.setup_views() elif key in ('shows', 'lists', 'filter', 'updates'): self.setup_func(key) From bf6b29e30f60e48ec245835bba54629da1c16da6 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 13:02:05 +0800 Subject: [PATCH 041/114] [clone_app] Fix missing `s` for `filters` function close #207 --- couchapp/clone_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 1ba438b5..474dfa3f 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -246,7 +246,7 @@ def setup_missing(self): self.setup_couchapp_json() elif key in ('views',): self.setup_views() - elif key in ('shows', 'lists', 'filter', 'updates'): + elif key in ('shows', 'lists', 'filters', 'updates'): self.setup_func(key) else: self.setup_prop(key) From d4d204b426096138504a9e15b1109aa1bf26c246 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 13:54:07 +0800 Subject: [PATCH 042/114] [#180] Separate logical block: `setup_couchapp_json` - This function came from `setup_missing`. - Test cases included. --- couchapp/clone_app.py | 18 ++++++++++----- tests/test_clone_app.py | 51 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 474dfa3f..089785be 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -59,6 +59,9 @@ def __init__(self, source, dest=None, rev=None): # manifest isn't in app self.setup_missing() + # create couchapp.json + self.setup_couchapp_json() + # save id self.setup_id() @@ -123,8 +126,8 @@ def setup_manifest(self): if filename.endswith('/'): # create dir self.setup_dir(filepath) continue - elif filename == 'couchapp.json': # we will handle it later - continue + elif filename == 'couchapp.json': + continue # we will setup it later in ``self.__init__`` # create file item_pair = self.extract_property(filename) @@ -243,7 +246,7 @@ def setup_missing(self): if key.startswith('_'): continue elif key in ('couchapp',): - self.setup_couchapp_json() + continue # we will setup it later in ``self.__init__`` elif key in ('views',): self.setup_views() elif key in ('shows', 'lists', 'filters', 'updates'): @@ -306,6 +309,10 @@ def setup_couchapp_json(self): - ``objects`` - ``length`` ''' + if 'couchapp' not in self.doc: + logger.warning('missing `couchapp` property in document') + return + app_meta = copy.deepcopy(self.doc['couchapp']) if 'signatures' in app_meta: @@ -317,9 +324,8 @@ def setup_couchapp_json(self): if 'length' in app_meta: del app_meta['length'] - if app_meta: - couchapp_file = os.path.join(self.path, 'couchapp.json') - util.write_json(couchapp_file, app_meta) + couchapp_file = os.path.join(self.path, 'couchapp.json') + util.write_json(couchapp_file, app_meta) def setup_views(self): ''' diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index c694a279..52046af4 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -455,3 +455,54 @@ def test_setup_manifest_create_files(self, setup_dir, dump_file, assert extract_property.call_count == 2 assert dump_file.called assert decode_content.called + + @patch('couchapp.clone_app.util.write_json') + def test_setup_couchapp_json_miss(self, write_json): + ''' + Test case for setup_couchapp_json with missing ``self.doc['couchapp']`` + ''' + self.clone.doc = {} + self.clone.setup_couchapp_json() + assert not write_json.called + + @patch('couchapp.clone_app.util.write_json') + def test_setup_couchapp_json_empty(self, write_json): + ''' + Test case for setup_couchapp_json with empty json writted finally + ''' + self.clone.doc = { + 'couchapp': { + 'signatures': {}, + 'objects': {}, + 'manifest': [], + 'length': 42 + } + } + self.clone.path = '/mock' + + self.clone.setup_couchapp_json() + + assert '/mock/couchapp.json' in write_json.call_args_list[0][0] + assert {} in write_json.call_args_list[0][0] + + @patch('couchapp.clone_app.util.write_json') + def test_setup_couchapp_json(self, write_json): + ''' + Test case for setup_couchapp_json + ''' + self.clone.doc = { + 'couchapp': { + 'name': 'foo', + 'signatures': {}, + 'objects': {}, + 'manifest': [], + 'length': 42, + 'truth': 42, + }, + } + self.clone.path = '/mock' + + self.clone.setup_couchapp_json() + + assert '/mock/couchapp.json' in write_json.call_args_list[0][0] + assert {'name': 'foo', 'truth': 42} in write_json.call_args_list[0][0] From 787550597c2999ecfe47eef131b5db99b6327003 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 14:50:04 +0800 Subject: [PATCH 043/114] [#180] Separate logical block: `setup_views` - This function came from `setup_missing` - Test cases included --- couchapp/clone_app.py | 23 ++++++++++++----------- tests/test_clone_app.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 089785be..b2347ee8 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -331,25 +331,26 @@ def setup_views(self): ''' Create ``views/`` - ``views`` dir will have following structure: - ``` - views/ - view_name/ - map.js - reduce.js (optional) - view_name2/ - ... - ``` + ``views`` dir will have following structure:: + + views/ + view_name/ + map.js + reduce.js (optional) + view_name2/ + ... + ''' vs_dir = os.path.join(self.path, 'views') if not os.path.isdir(vs_dir): - os.makedirs(vs_dir) + self.setup_dir(vs_dir) for vsname, vs_item in self.doc['views'].iteritems(): vs_item_dir = os.path.join(vs_dir, vsname) if not os.path.isdir(vs_item_dir): - os.makedirs(vs_item_dir) + self.setup_dir(vs_item_dir) + for func_name, func in vs_item.iteritems(): filename = os.path.join(vs_item_dir, '{0}.js'.format(func_name)) diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 52046af4..f2c80391 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -506,3 +506,40 @@ def test_setup_couchapp_json(self, write_json): assert '/mock/couchapp.json' in write_json.call_args_list[0][0] assert {'name': 'foo', 'truth': 42} in write_json.call_args_list[0][0] + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_setup_views_empty(self, util_write, setup_dir): + ''' + Test case for ``setup_views`` with empty ``views`` prop + ''' + self.clone.doc = { + 'views': {} + } + self.clone.path = '/mock' + + self.clone.setup_views() + + assert not util_write.called + assert setup_dir.called + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_setup_views(self, util_write, setup_dir): + ''' + Test case for ``setup_views`` with ``mockview`` written + ''' + self.clone.doc = { + 'views': { + 'mockview': { + 'map': 'function(){}', + 'redurce': 'function(){}', + } + } + } + self.clone.path = '/mock' + + self.clone.setup_views() + + assert util_write.call_count == 2 + assert setup_dir.called From 5b93c8ceb10f311366056186968f41ff50f6fa44 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 15:31:16 +0800 Subject: [PATCH 044/114] [#180] Separate logical block: `setup_func` - This function came from `setup_missing` - Test cases included --- couchapp/clone_app.py | 8 ++++---- tests/test_clone_app.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index b2347ee8..0a8b3bc1 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -366,13 +366,13 @@ def setup_func(self, func): - ``filters`` - ``updates`` ''' - showpath = os.path.join(self.path, func) + func_dir = os.path.join(self.path, func) - if not os.path.isdir(showpath): - os.makedirs(showpath) + if not os.path.isdir(func_dir): + self.setup_dir(func_dir) for func_name, func in self.doc[func].iteritems(): - filename = os.path.join(showpath, '{0}.js'.format(func_name)) + filename = os.path.join(func_dir, '{0}.js'.format(func_name)) util.write(filename, func) logger.warning( 'clone function "{0}" not in manifest: {1}'.format(func_name, diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index f2c80391..bede0984 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -543,3 +543,38 @@ def test_setup_views(self, util_write, setup_dir): assert util_write.call_count == 2 assert setup_dir.called + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_setup_func_empty(self, util_write, setup_dir): + ''' + Test case for ``setup_func`` with empty func prop + ''' + self.clone.doc = { + 'shows': {} + } + self.clone.path = '/mock' + + self.clone.setup_func('shows') + + assert setup_dir.called + assert not util_write.called + + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.util.write') + def test_setup_func_shows(self, util_write, setup_dir): + ''' + Test case for ``setup_func`` have ``shows`` written + ''' + self.clone.doc = { + 'shows': { + 'mock': 'function(){}', + 'fake': 'function(){}' + } + } + self.clone.path = '/mock' + + self.clone.setup_func('shows') + + assert setup_dir.called + assert util_write.call_count == 2 From a5d333de5a429cabe4067b66ca58cdd36ea52ec0 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 17:08:55 +0800 Subject: [PATCH 045/114] [#180] new helper function: `clone.flatten_doc` Test case included. --- couchapp/clone_app.py | 51 ++++++++++++++++++++++++++++++----------- tests/test_clone_app.py | 20 ++++++++++++++++ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 0a8b3bc1..b38222e1 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -259,7 +259,8 @@ def setup_prop(self, prop): Create file for arbitrary property. Policy: - - If the property is a list, we will save it as json file. + - If the property is a list/int/float/bool/null, + we will save it as json file. - If the property is a dict, we will create a dir for it and handle its contents recursively. @@ -272,32 +273,30 @@ def setup_prop(self, prop): if prop not in self.doc: return - filedir = os.path.join(self.path, prop) + filepath = os.path.join(self.path, prop) - if os.path.exists(filedir): + if os.path.exists(filepath): return logger.warning('clone property not in manifest: {0}'.format(prop)) - if isinstance(self.doc[prop], (list, tuple,)): - util.write_json('{0}.json'.format(filedir), self.doc[prop]) - elif isinstance(self.doc[prop], dict): - if not os.path.isdir(filedir): - os.makedirs(filedir) + value = self.doc[prop] + if isinstance(value, (list, tuple, int, float, bool)) or value is None: + self.dump_file('{0}.json'.format(filepath), value) + elif isinstance(value, dict): + if not os.path.isdir(filepath): + self.setup_dir(filepath) for field, value in self.doc[prop].iteritems(): - fieldpath = os.path.join(filedir, field) + fieldpath = os.path.join(filepath, field) if isinstance(value, basestring): if value.startswith('base64-encoded;'): value = base64.b64decode(content[15:]) util.write(fieldpath, value) else: util.write_json(fieldpath + '.json', value) - else: - value = self.doc[prop] - if not isinstance(value, basestring): - value = str(value) - util.write(filedir, value) + else: # in case of ``string`` + self.dump_file(filepath, self.decode_content(value)) def setup_couchapp_json(self): ''' @@ -438,3 +437,27 @@ def setup_dir(self, path): logger.debug(e) return False return True + + def flatten_doc(self, doc): + ''' + flatten a nested doc with filesystem map + + :param doc: { + 'foo': { + 'bar': 'fake' + } + } + + :return: { + 'foo/bar': 'fake' + } + ''' + ret = {} + for key, val in doc.iteritems(): + if not isinstance(val, dict): + ret[key] = val + continue + + for subkey, subval in self.flatten_doc(val).iteritems(): + ret['{0}/{1}'.format(key, subkey)] = subval + return ret diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index bede0984..1809d164 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -578,3 +578,23 @@ def test_setup_func_shows(self, util_write, setup_dir): assert setup_dir.called assert util_write.call_count == 2 + + def test_flatten_doc(self): + ''' + Test case for ``flatten_doc`` + ''' + doc = { + 'views': { + 'mock': { + 'map': 'map_func', + 'reduce': 'reduce_func', + } + }, + 'truth': 42 + } + expect = { + 'views/mock/map': 'map_func', + 'views/mock/reduce': 'reduce_func', + 'truth': 42 + } + assert self.clone.flatten_doc(doc) == expect From bc3869936a297ae80427e492d418fb3350ca25d9 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 22:05:16 +0800 Subject: [PATCH 046/114] [#180] Separate logical block: `setup_prop` - This function came from `setup_missing` - Test cases included --- couchapp/clone_app.py | 18 +++---- tests/test_clone_app.py | 105 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index b38222e1..185c2b80 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -278,7 +278,7 @@ def setup_prop(self, prop): if os.path.exists(filepath): return - logger.warning('clone property not in manifest: {0}'.format(prop)) + logger.warning('clone property not in manifest: "{0}"'.format(prop)) value = self.doc[prop] if isinstance(value, (list, tuple, int, float, bool)) or value is None: @@ -287,15 +287,15 @@ def setup_prop(self, prop): if not os.path.isdir(filepath): self.setup_dir(filepath) - for field, value in self.doc[prop].iteritems(): + for field, content in value.iteritems(): fieldpath = os.path.join(filepath, field) - if isinstance(value, basestring): - if value.startswith('base64-encoded;'): - value = base64.b64decode(content[15:]) - util.write(fieldpath, value) - else: - util.write_json(fieldpath + '.json', value) - else: # in case of ``string`` + content = self.decode_content(content) + + if not isinstance(content, basestring): + fieldpath = '{0}.json'.format(fieldpath) + + self.dump_file(fieldpath, content) + else: # in case of content is ``string`` self.dump_file(filepath, self.decode_content(value)) def setup_couchapp_json(self): diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 1809d164..218634fd 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -598,3 +598,108 @@ def test_flatten_doc(self): 'truth': 42 } assert self.clone.flatten_doc(doc) == expect + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.clone.dump_file') + def test_setup_prop_nonexist(self, dump_file, setup_dir, decode_content): + ''' + Test case for ``clone.setup_prop`` with nonexist prop + ''' + self.clone.doc = {} + self.clone.setup_prop('nonexist') + assert not decode_content.called + assert not setup_dir.called + assert not dump_file.called + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.clone.dump_file') + def test_setup_prop_dict(self, dump_file, setup_dir, decode_content): + ''' + Test case for ``clone.setup_prop`` with dict + ''' + self.clone.path = '/mockapp' + self.clone.doc = { + 'mock': { + 'foo': { + 'bar': 42, + 'baz': None, + } + }, + 'fake': { + 'fun': 'data', + } + } + decode_content.side_effect = lambda x: x + + self.clone.setup_prop('mock') + assert setup_dir.call_count == 1 + assert decode_content.called + assert '/mockapp/mock/foo.json' in dump_file.call_args_list[0][0] + assert {'bar': 42, 'baz': None} in dump_file.call_args_list[0][0] + + setup_dir.reset_mock() + dump_file.reset_mock() + + self.clone.setup_prop('fake') + assert setup_dir.called == 1 + assert decode_content.called + assert '/mockapp/fake/fun' in dump_file.call_args_list[0][0] + assert 'data' in dump_file.call_args_list[0][0] + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.clone.dump_file') + def test_setup_prop_str(self, dump_file, setup_dir, decode_content): + ''' + Test case for ``clone.setup_prop`` with str value + ''' + self.clone.path = '/mockapp' + self.clone.doc = { + 'foo': 'bar', + } + decode_content.side_effect = lambda x: x + self.clone.setup_prop('foo') + + assert not setup_dir.called + assert decode_content.called + assert '/mockapp/foo' in dump_file.call_args_list[0][0] + assert 'bar' in dump_file.call_args_list[0][0] + + @patch('couchapp.clone_app.clone.decode_content') + @patch('couchapp.clone_app.clone.setup_dir') + @patch('couchapp.clone_app.clone.dump_file') + def test_setup_prop_non_str(self, dump_file, setup_dir, decode_content): + ''' + Test case for ``clone.setup_prop`` with non-str value + ''' + self.clone.path = '/mockapp' + self.clone.doc = { + 'foo': 42, + 'bar': None, + 'baz': True, + 'qux': [1, 2, 3], + } + decode_content.side_effect = lambda x: x + self.clone.setup_prop('foo') + + assert not setup_dir.called + assert not decode_content.called + assert '/mockapp/foo.json' in dump_file.call_args_list[0][0] + assert 42 in dump_file.call_args_list[0][0] + + dump_file.reset_mock() + self.clone.setup_prop('bar') + assert '/mockapp/bar.json' in dump_file.call_args_list[0][0] + assert None in dump_file.call_args_list[0][0] + + dump_file.reset_mock() + self.clone.setup_prop('baz') + assert '/mockapp/baz.json' in dump_file.call_args_list[0][0] + assert True in dump_file.call_args_list[0][0] + + dump_file.reset_mock() + self.clone.setup_prop('qux') + assert '/mockapp/qux.json' in dump_file.call_args_list[0][0] + assert [1, 2, 3] in dump_file.call_args_list[0][0] From e53153e7caea80d0d70055d69fef0064ff3ce9df Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 22:30:55 +0800 Subject: [PATCH 047/114] [#180] Tese cases for ``clone.setup_missing`` --- couchapp/clone_app.py | 2 +- tests/test_clone_app.py | 91 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 185c2b80..1cb8ce47 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -242,7 +242,7 @@ def setup_missing(self): ''' second pass for missing key or in case manifest isn't in app. ''' - for key in self.doc.iterkeys(): + for key in self.doc: if key.startswith('_'): continue elif key in ('couchapp',): diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 218634fd..402a576d 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -703,3 +703,94 @@ def test_setup_prop_non_str(self, dump_file, setup_dir, decode_content): self.clone.setup_prop('qux') assert '/mockapp/qux.json' in dump_file.call_args_list[0][0] assert [1, 2, 3] in dump_file.call_args_list[0][0] + + @patch('couchapp.clone_app.clone.setup_prop') + @patch('couchapp.clone_app.clone.setup_func') + @patch('couchapp.clone_app.clone.setup_views') + def test_setup_missing_underlinekey(self, setup_views, setup_func, setup_prop): + ''' + Test case for ``clone.setup_missing`` handle ``_key`` + ''' + self.clone.doc = { + '_id': 'foo', + '_rev': 'bar', + } + self.clone.setup_missing() + + assert not setup_views.called + assert not setup_func.called + assert not setup_prop.called + + @patch('couchapp.clone_app.clone.setup_prop') + @patch('couchapp.clone_app.clone.setup_func') + @patch('couchapp.clone_app.clone.setup_views') + def test_setup_missing_couchapp_json(self, setup_views, setup_func, setup_prop): + ''' + Test case for ``clone.setup_missing`` encounter key ``couchapp`` + ''' + self.clone.doc = { + 'couchapp': {} + } + self.clone.setup_missing() + + assert not setup_views.called + assert not setup_func.called + assert not setup_prop.called + + @patch('couchapp.clone_app.clone.setup_prop') + @patch('couchapp.clone_app.clone.setup_func') + @patch('couchapp.clone_app.clone.setup_views') + def test_setup_missing_views(self, setup_views, setup_func, setup_prop): + ''' + Test case for ``clone.setup_missing`` handle views + ''' + self.clone.doc = { + 'views': { + 'mock': { + 'map': 'func', + 'reduce': 'func', + } + } + } + self.clone.setup_missing() + + assert setup_views.called + assert not setup_func.called + assert not setup_prop.called + + @patch('couchapp.clone_app.clone.setup_prop') + @patch('couchapp.clone_app.clone.setup_func') + @patch('couchapp.clone_app.clone.setup_views') + def test_setup_missing_func(self, setup_views, setup_func, setup_prop): + ''' + Test case for ``clone.setup_missing`` handle func like ``shows`` + ''' + self.clone.doc = { + 'shows': {}, + 'updates': {}, + 'filters': {}, + 'lists': {}, + } + self.clone.setup_missing() + + assert not setup_views.called + assert setup_func.call_count == 4 + assert not setup_prop.called + + @patch('couchapp.clone_app.clone.setup_prop') + @patch('couchapp.clone_app.clone.setup_func') + @patch('couchapp.clone_app.clone.setup_views') + def test_setup_missing_prop(self, setup_views, setup_func, setup_prop): + ''' + Test case for ``clone.setup_missing`` handle prop + ''' + self.clone.doc = { + 'mock': 'fake', + 'foo': 'bar', + 'baz': 'qux', + } + self.clone.setup_missing() + + assert not setup_views.called + assert not setup_func.called + assert setup_prop.call_count == 3 From 4ba6e1f284ead2871b889eb81cb8287d54161f1d Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 23 Jan 2016 23:10:32 +0800 Subject: [PATCH 048/114] [#180] Test case for ``setup_id`` --- tests/test_clone_app.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 402a576d..6e5de366 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -794,3 +794,17 @@ def test_setup_missing_prop(self, setup_views, setup_func, setup_prop): assert not setup_views.called assert not setup_func.called assert setup_prop.call_count == 3 + + @patch('couchapp.clone_app.util.write') + def test_setup_id(self, write): + ''' + Test case for ``clone.setup_id`` + ''' + self.clone.path = '/mock' + self.clone.doc = { + '_id': '_design/mock', + } + + self.clone.setup_id() + assert '/mock/_id' in write.call_args_list[0][0] + assert '_design/mock' in write.call_args_list[0][0] From f7fe3f7fe0f5028767b8277e2e1e4cae2e0cc1bc Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 00:00:59 +0800 Subject: [PATCH 049/114] [#180] New helper function: `vendor_attach_dir` Given a vendor path listed in `_attachments`, map it to filesystem path. Test cases included. --- couchapp/clone_app.py | 19 +++++++++++++++++++ tests/test_clone_app.py | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 1cb8ce47..8643a1e2 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -418,6 +418,25 @@ def setup_attachments(self): f.write(chunk) logger.debug('clone attachment: {0}'.format(filename)) + def vendor_attach_dir(self, path): + ''' + Map the vendor attachments to filesystem path + + Assume that we have ``vendor/couchapp/index.html`` + in ``_attachments`` section. It should be stored in + ``/path/to/app/vendor/couchapp/_attachments/index.html`` + + :return: the path string, + if the ``path`` not starts with ``vendor``, + return the original path. + ''' + if not path.startswith('vendor'): + return path + + path = util.split_path(path) + path.insert(2, '_attachments') + return os.path.join(self.path, *path) + def setup_dir(self, path): ''' Create dir recursively. diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 6e5de366..b7a72074 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -808,3 +808,24 @@ def test_setup_id(self, write): self.clone.setup_id() assert '/mock/_id' in write.call_args_list[0][0] assert '_design/mock' in write.call_args_list[0][0] + + def test_vendor_attach_dir(self): + ''' + Test case for ``clone.vendor_attach_dir`` + ''' + self.clone.path = '/mock' + orig_path = 'vendor/couchapp/app.js' + expect = '/mock/vendor/couchapp/_attachments/app.js' + + ret = self.clone.vendor_attach_dir(orig_path) + assert ret == expect, '{0} != {1}'.format(ret, expect) + + def test_vendor_attach_dir_invalid(self): + ''' + Test case for ``clone.vendor_attach_dir`` with invalid path + ''' + self.clone.path = '/mock' + orig_path = 'nonvendor/couchapp/app.js' + + ret = self.clone.vendor_attach_dir(orig_path) + assert ret == orig_path, '{0} != {1}'.format(ret, orig_path) From 7842d40fa72534e69e29eecf069998e2e28cfd08 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 11:40:41 +0800 Subject: [PATCH 050/114] [#180] New helper function: `dump_attachment` This function came from `setup_attachments`. We separated it, in order to deal with testing the side effect. --- couchapp/clone_app.py | 15 +++++++++++++++ tests/test_clone_app.py | 24 +++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 8643a1e2..c748df83 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -437,6 +437,21 @@ def vendor_attach_dir(self, path): path.insert(2, '_attachments') return os.path.join(self.path, *path) + def dump_attachment(self, url, path): + ''' + dump the attachment to filesystem. + + :param url: the relative url in document + :param path: the filesystem path + ''' + if not url or not path: + return + + resp = self.db.fetch_attachment(self.docid, url) + with open(path, 'wb') as f: + for chunk in resp.body_stream(): + f.write(chunk) + def setup_dir(self, path): ''' Create dir recursively. diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index b7a72074..31c16b48 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -3,7 +3,7 @@ from couchapp.clone_app import clone from couchapp.errors import AppError, MissingContent -from mock import patch +from mock import MagicMock, call, patch from nose.tools import raises @@ -829,3 +829,25 @@ def test_vendor_attach_dir_invalid(self): ret = self.clone.vendor_attach_dir(orig_path) assert ret == orig_path, '{0} != {1}'.format(ret, orig_path) + + @patch('couchapp.clone_app.open', side_effect=AssertionError) + def test_dump_attachment_empty(self, mock_open): + ''' + Test case for ``clone.dump_attachment`` with empty args + ''' + self.clone.dump_attachment('', '') + self.clone.dump_attachment(None, None) + + @patch('couchapp.clone_app.open') + def test_dump_attachment(self, mock_open): + ''' + Test case for ``clone.dump_attachment`` + ''' + self.clone.docid = 'fakeid' + self.clone.db = MagicMock(name='db') + self.clone.db.fetch_attachment().body_stream.return_value = [None] + self.clone.dump_attachment('js/app.js', '/mock/_attachments/js/app.js') + expect_call = call.fetch_attachment('fakeid', 'js/app.js') + + assert mock_open.called + assert expect_call in self.clone.db.mock_calls From 3419316fc79e387cfb36746b23cb1daa9c797348 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 14:42:33 +0800 Subject: [PATCH 051/114] [#180] Test cases for `setup_attachments` Also, rename `vendor_attach_dir` to `locate_attach_dir`; make this function point out correct abs path. --- couchapp/clone_app.py | 44 ++++++++----------- tests/test_clone_app.py | 97 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 33 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index c748df83..8b2f53f3 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -386,7 +386,7 @@ def setup_id(self): def setup_attachments(self): ''' - Create ``_attachments`` dir + Create ``_attachments`` ''' if '_attachments' not in self.doc: return @@ -394,44 +394,36 @@ def setup_attachments(self): attachdir = os.path.join(self.path, '_attachments') if not os.path.isdir(attachdir): - os.makedirs(attachdir) - - for filename in self.doc['_attachments'].iterkeys(): - if filename.startswith('vendor'): - attach_parts = util.split_path(filename) - vendor_attachdir = os.path.join(self.path, attach_parts.pop(0), - attach_parts.pop(0), - '_attachments') - filepath = os.path.join(vendor_attachdir, *attach_parts) - else: - filepath = os.path.join(attachdir, filename) + self.setup_dir(attachdir) - filepath = os.path.normpath(filepath) + for filename in self.doc['_attachments']: + filepath = os.path.normpath(self.locate_attach_dir(filename)) currentdir = os.path.dirname(filepath) + if not os.path.isdir(currentdir): - os.makedirs(currentdir) + self.setup_dir(currentdir) + + if self.signatures.get(filename) == util.sign(filepath): + continue # we already have the same file on fs + + self.dump_attachment(filename, filepath) - if self.signatures.get(filename) != util.sign(filepath): - resp = self.db.fetch_attachment(self.docid, filename) - with open(filepath, 'wb') as f: - for chunk in resp.body_stream(): - f.write(chunk) - logger.debug('clone attachment: {0}'.format(filename)) + logger.debug('clone attachment: "{0}"'.format(filename)) - def vendor_attach_dir(self, path): + def locate_attach_dir(self, path): ''' - Map the vendor attachments to filesystem path + Map the attachments dir to filesystem path - Assume that we have ``vendor/couchapp/index.html`` - in ``_attachments`` section. It should be stored in + Note that if we have ``vendor/couchapp/index.html`` + in ``_attachments`` section, it should be stored in ``/path/to/app/vendor/couchapp/_attachments/index.html`` :return: the path string, if the ``path`` not starts with ``vendor``, - return the original path. + return the normal attachment dir ''' if not path.startswith('vendor'): - return path + return os.path.join(self.path, '_attachments', path) path = util.split_path(path) path.insert(2, '_attachments') diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 31c16b48..b6b1b73c 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -809,26 +809,27 @@ def test_setup_id(self, write): assert '/mock/_id' in write.call_args_list[0][0] assert '_design/mock' in write.call_args_list[0][0] - def test_vendor_attach_dir(self): + def test_locate_attach_dir_vendor(self): ''' - Test case for ``clone.vendor_attach_dir`` + Test case for ``clone.locate_attach_dir`` with vendor dir ''' self.clone.path = '/mock' orig_path = 'vendor/couchapp/app.js' expect = '/mock/vendor/couchapp/_attachments/app.js' - ret = self.clone.vendor_attach_dir(orig_path) + ret = self.clone.locate_attach_dir(orig_path) assert ret == expect, '{0} != {1}'.format(ret, expect) - def test_vendor_attach_dir_invalid(self): + def test_locate_attach_dir(self): ''' - Test case for ``clone.vendor_attach_dir`` with invalid path + Test case for ``clone.locate_attach_dir`` ''' self.clone.path = '/mock' orig_path = 'nonvendor/couchapp/app.js' + expect = '/mock/_attachments/nonvendor/couchapp/app.js' - ret = self.clone.vendor_attach_dir(orig_path) - assert ret == orig_path, '{0} != {1}'.format(ret, orig_path) + ret = self.clone.locate_attach_dir(orig_path) + assert ret == expect, '{0} != {1}'.format(ret, expect) @patch('couchapp.clone_app.open', side_effect=AssertionError) def test_dump_attachment_empty(self, mock_open): @@ -851,3 +852,85 @@ def test_dump_attachment(self, mock_open): assert mock_open.called assert expect_call in self.clone.db.mock_calls + + @patch('couchapp.clone_app.clone.dump_attachment') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_attachments_empty(self, setup_dir, dump_attachment): + ''' + Test case for ``clone.setup_attachments`` with empty args + ''' + self.clone.doc = {} + + self.clone.setup_attachments() + + assert not setup_dir.called + assert not dump_attachment.called + + @patch('couchapp.clone_app.clone.dump_attachment') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_attachments(self, setup_dir, dump_attachment): + ''' + Test case for ``clone.setup_attachments`` + ''' + self.clone.path = '/mock' + self.clone.doc = { + '_attachments': { + 'js/mock.js': { + 'content_type': 'application/javascript', + } + } + } + self.clone.signatures = {} + dump_call = call('js/mock.js', '/mock/_attachments/js/mock.js') + + self.clone.setup_attachments() + + assert setup_dir.called + assert dump_call in dump_attachment.mock_calls + + @patch('couchapp.clone_app.clone.dump_attachment') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_attachments_vendor(self, setup_dir, dump_attachment): + ''' + Test case for ``clone.setup_attachments`` with vendor attachements + ''' + self.clone.path = '/mock' + self.clone.doc = { + '_attachments': { + 'vendor/couchapp/magic.js': { + 'content_type': 'application/javascript', + } + } + } + self.clone.signatures = {} + dump_call = call('vendor/couchapp/magic.js', + '/mock/vendor/couchapp/_attachments/magic.js') + + self.clone.setup_attachments() + + assert setup_dir.called + assert dump_call in dump_attachment.mock_calls + + @patch('couchapp.clone_app.util.sign', return_value='mock_sign') + @patch('couchapp.clone_app.clone.dump_attachment') + @patch('couchapp.clone_app.clone.setup_dir') + def test_setup_attachments_ignore(self, setup_dir, dump_attachment, sign): + ''' + Test case for ``clone.setup_attachments`` ignore existed file on fs + ''' + self.clone.path = '/mock' + self.clone.doc = { + '_attachments': { + 'vendor/couchapp/magic.js': { + 'content_type': 'application/javascript', + } + } + } + self.clone.signatures = { + 'vendor/couchapp/magic.js': 'mock_sign' + } + + self.clone.setup_attachments() + + assert setup_dir.called + assert not dump_attachment.called From abd46b61f702522cb6c6300c91f044bc492b2198 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 15:17:41 +0800 Subject: [PATCH 052/114] [#180] Separate logical block: `init_db` Test cases include. --- couchapp/clone_app.py | 32 +++++++++++++++++++++++++------- tests/test_clone_app.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 8b2f53f3..718f8a4c 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -41,13 +41,11 @@ def __init__(self, source, dest=None, rev=None): # init self.path self.init_path() - # init self.db - self.db = client.Database(self.dburl[:-1], create=False) - if not self.rev: - self.doc = self.db.open_doc('_design/{0}'.format(self.docid)) - else: - self.doc = self.db.open_doc('_design/{0}'.format(self.docid), rev=self.rev) - self.docid = self.doc['_id'] + # init self.db related vars here + # affected: + # - self.docid + # - self.doc + self.init_db() # init metadata self.init_metadata() @@ -83,10 +81,30 @@ def __new__(cls, *args, **kwargs): return None def init_path(self): + ''' + data dependency: + - self.dest + ''' self.path = os.path.normpath(os.path.join(os.getcwd(), self.dest or '')) self.setup_dir(self.path) + def init_db(self): + ''' + init self.db related vars here + + affected: + - self.docid + - self.doc + ''' + self.db = client.Database(self.dburl[:-1], create=False) + if not self.rev: + self.doc = self.db.open_doc('_design/{0}'.format(self.docid)) + else: + self.doc = self.db.open_doc('_design/{0}'.format(self.docid), + rev=self.rev) + self.docid = self.doc['_id'] + def init_metadata(self): ''' Setup diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index b6b1b73c..dbdbe7d6 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -934,3 +934,35 @@ def test_setup_attachments_ignore(self, setup_dir, dump_attachment, sign): assert setup_dir.called assert not dump_attachment.called + + @patch('couchapp.clone_app.client.Database') + def test_init_db_no_rev(self, db): + ''' + Test case for ``clone.init_db`` without ``self.rev`` + ''' + self.clone.rev = None + self.clone.docid = 'mockapp' + self.clone.dburl = 'http://localhost:5984/test_db/' + db_call = call('http://localhost:5984/test_db', create=False) + connect_call = call('_design/mockapp') + + self.clone.init_db() + + assert db_call in db.mock_calls + assert connect_call in db().open_doc.mock_calls + + @patch('couchapp.clone_app.client.Database') + def test_init_db_rev(self, db): + ''' + Test case for ``clone.init_db`` with ``self.rev`` + ''' + self.clone.rev = 'mockrev' + self.clone.docid = 'mockapp' + self.clone.dburl = 'http://localhost:5984/test_db/' + db_call = call('http://localhost:5984/test_db', create=False) + connect_call = call('_design/mockapp', rev='mockrev') + + self.clone.init_db() + + assert db_call in db.mock_calls + assert connect_call in db().open_doc.mock_calls From 4d6af1133056ece32d5102ba29af481a6acdba1b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 15:30:46 +0800 Subject: [PATCH 053/114] [#180] Separate logical block: `setup_couchapprc` Currently, we just setup an empty `.couchapprc` via this function. Considering to auto store db url to it in future. Test cases included. --- couchapp/clone_app.py | 10 ++++++++-- tests/test_clone_app.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/couchapp/clone_app.py b/couchapp/clone_app.py index 718f8a4c..9569ca8e 100644 --- a/couchapp/clone_app.py +++ b/couchapp/clone_app.py @@ -63,8 +63,8 @@ def __init__(self, source, dest=None, rev=None): # save id self.setup_id() - # setup empty .couchapprc - util.write_json(os.path.join(self.path, '.couchapprc'), {}) + # setup .couchapprc + self.setup_couchapprc() # process attachments self.setup_attachments() @@ -402,6 +402,12 @@ def setup_id(self): idfile = os.path.join(self.path, '_id') util.write(idfile, self.doc['_id']) + def setup_couchapprc(self): + ''' + Setup empty .couchapprc + ''' + util.write_json(os.path.join(self.path, '.couchapprc'), {}) + def setup_attachments(self): ''' Create ``_attachments`` diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index dbdbe7d6..831737d3 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -966,3 +966,14 @@ def test_init_db_rev(self, db): assert db_call in db.mock_calls assert connect_call in db().open_doc.mock_calls + + @patch('couchapp.clone_app.util.write_json') + def test_setup_couchapprc(self, write_json): + ''' + Test case for ``clone.setup_couchapprc`` + ''' + self.clone.path = '/mock' + expect_call = call('/mock/.couchapprc', {}) + + self.clone.setup_couchapprc() + assert expect_call in write_json.mock_calls From 82c054aea7661abb2b955a2c5a51eb4e3a060065 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 24 Jan 2016 15:41:51 +0800 Subject: [PATCH 054/114] [#180] Test case for `clone.__init__`, close #180 --- tests/test_clone_app.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_clone_app.py b/tests/test_clone_app.py index 831737d3..5b22ba51 100644 --- a/tests/test_clone_app.py +++ b/tests/test_clone_app.py @@ -977,3 +977,23 @@ def test_setup_couchapprc(self, write_json): self.clone.setup_couchapprc() assert expect_call in write_json.mock_calls + + @patch('couchapp.clone_app.clone.setup_attachments') + @patch('couchapp.clone_app.clone.setup_couchapprc') + @patch('couchapp.clone_app.clone.setup_id') + @patch('couchapp.clone_app.clone.setup_couchapp_json') + @patch('couchapp.clone_app.clone.setup_missing') + @patch('couchapp.clone_app.clone.setup_manifest') + @patch('couchapp.clone_app.clone.init_metadata') + @patch('couchapp.clone_app.clone.init_db') + @patch('couchapp.clone_app.clone.init_path') + def test_init(self, *args): + ''' + Test case for ``clone__init__`` + ''' + src = 'http://localhost:5984/testdb/_design/mockapp' + self.clone.__init__(src) + + for mock in args: + assert mock.called, mock + assert self.clone.dest == 'mockapp', self.clone.dest From 443071698085af70865a870a0df6c2337db31a4e Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 5 Feb 2016 18:25:16 +0800 Subject: [PATCH 055/114] [localdoc] invoke `os.path.realpath` to increase portability --- couchapp/localdoc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchapp/localdoc.py b/couchapp/localdoc.py index 477ff50c..0666c506 100644 --- a/couchapp/localdoc.py +++ b/couchapp/localdoc.py @@ -3,8 +3,8 @@ # This file is part of couchapp released under the Apache 2 license. # See the NOTICE for more information. - from __future__ import with_statement + import base64 import logging import mimetypes @@ -73,7 +73,7 @@ def __init__(self, path, create=False, docid=None, is_ddoc=True): def get_id(self): """ - if there is an _id file, docid is extracted from it, + if there is an ``_id`` file, docid is extracted from it, else we take the current folder name. """ idfile = os.path.join(self.docdir, '_id') @@ -436,4 +436,5 @@ def to_json(self): def document(path, create=False, docid=None, is_ddoc=True): - return LocalDoc(path, create=create, docid=docid, is_ddoc=is_ddoc) + return LocalDoc(os.path.realpath(path), create=create, docid=docid, + is_ddoc=is_ddoc) From cb42ba8eddfd07b9daf031646e4306ed0f27b07c Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 1 Mar 2016 01:24:49 +0800 Subject: [PATCH 056/114] [util] new helper functions: `is_empty_dir` and `setup_dir` test cases included --- couchapp/util.py | 27 ++++++++++- tests/test_util.py | 118 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/couchapp/util.py b/couchapp/util.py index 05eea958..094002f6 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -17,7 +17,7 @@ from hashlib import md5 -from couchapp.errors import ScriptError +from couchapp.errors import AppError, ScriptError try: import json @@ -545,3 +545,28 @@ def sh_open(cmd, bufsize=0): out, err = p.communicate() return (out, err) + + +def is_empty_dir(path): + if not os.listdir(path): + return True + return False + + +def setup_dir(path, require_empty=True): + ''' + If dir exists, check it empty or not. + If dir does not exist, make one. + ''' + isdir = os.path.isdir(path) + + if isdir and not require_empty: + return + elif isdir and require_empty and is_empty_dir(path): + return + elif isdir and require_empty and not is_empty_dir(path): + raise AppError("dir '{0}' is not empty".format(path)) + elif os.path.exists(path) and not isdir: + raise AppError("'{0}': File exists".format(path)) + + os.mkdir(path) diff --git a/tests/test_util.py b/tests/test_util.py index 793fd3e4..956b895f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- +import shutil +import tempfile import os +from couchapp.errors import AppError from couchapp.util import discover_apps, iscouchapp, rcpath, split_path -from couchapp.util import sh_open, remove_comments +from couchapp.util import sh_open, remove_comments, is_empty_dir, setup_dir from mock import patch +from nose.tools import raises @patch('couchapp.util.user_rcpath') @@ -135,3 +139,115 @@ def test_remove_comments(): expect = '{\n "mock": 42, \n \n "fake": true}' assert ret == expect + + +class test_is_empty_dir_true(): + def setup(self): + self.tmpdir = tempfile.mkdtemp() + + def test(self): + assert is_empty_dir(self.tmpdir) is True + + def teardown(self): + shutil.rmtree(self.tmpdir) + + +class test_is_empty_dir_false(): + def setup(self): + self.tmpdir = tempfile.mkdtemp() + self.tmpfile = tempfile.NamedTemporaryFile(dir=self.tmpdir, + delete=False) + self.tmpfile.close() + + def test(self): + assert is_empty_dir(self.tmpdir) is False + + def teardown(self): + shutil.rmtree(self.tmpdir) + + +@patch('couchapp.util.os.mkdir') +def test_setup_dir(mkdir): + setup_dir('/mock') + assert mkdir.called + + +class test_setup_dir_exists(): + def setup(self): + ''' + /tmpdir/ + *empty* + ''' + self.tmpdir = tempfile.mkdtemp() + + @patch('couchapp.util.os.mkdir') + def test(self, mkdir): + setup_dir(self.tmpdir, require_empty=True) + assert not mkdir.called + + def teardown(self): + shutil.rmtree(self.tmpdir) + + +class test_setup_dir_exists_not_empty(): + def setup(self): + ''' + /tmpdir/ + tmpfile + ''' + self.tmpdir = tempfile.mkdtemp() + self.tmpfile = tempfile.NamedTemporaryFile(dir=self.tmpdir, + delete=False) + self.tmpfile.close() + + @raises(AppError) + def main(self): + setup_dir(self.tmpdir, require_empty=True) + + @patch('couchapp.util.os.mkdir') + def test(self, mkdir): + self.main() + assert not mkdir.called + + def teardown(self): + shutil.rmtree(self.tmpdir) + + +class test_setup_dir_exists_empty_not_required(): + def setup(self): + ''' + /tmpdir/ + tmpfile + ''' + self.tmpdir = tempfile.mkdtemp() + self.tmpfile = tempfile.NamedTemporaryFile(dir=self.tmpdir, + delete=False) + self.tmpfile.close() + + @patch('couchapp.util.os.mkdir') + def test(self, mkdir): + setup_dir(self.tmpdir, require_empty=False) + assert not mkdir.called + + def teardown(self): + shutil.rmtree(self.tmpdir) + + +class test_setup_dir_exists_not_dir(): + def setup(self): + ''' + /strangefile + ''' + self.tmpfile = tempfile.NamedTemporaryFile(delete=True) + + @raises(AppError) + def main(self): + setup_dir(self.tmpfile.name) + + @patch('couchapp.util.os.mkdir') + def test(self, mkdir): + self.main() + assert not mkdir.called + + def teardown(self): + del self.tmpfile From 75359b2576122c57db542f998368165f93a6a207 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 1 Mar 2016 01:32:49 +0800 Subject: [PATCH 057/114] [localdoc] let `couchapp ini newdir` create dir correctly --- couchapp/localdoc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchapp/localdoc.py b/couchapp/localdoc.py index 0666c506..e095bf58 100644 --- a/couchapp/localdoc.py +++ b/couchapp/localdoc.py @@ -24,9 +24,9 @@ desktopcouch = None +from couchapp import util from couchapp.errors import ResourceNotFound, AppError from couchapp.macros import package_shows, package_views -from couchapp import util if os.name == 'nt': def _replace_backslash(name): @@ -94,8 +94,7 @@ def __str__(self): return util.json.dumps(self.doc()) def create(self): - if not os.path.isdir(self.docdir): - logger.error("%s directory doesn't exist." % self.docdir) + util.setup_dir(self.docdir, require_empty=False) rcfile = os.path.join(self.docdir, '.couchapprc') ignfile = os.path.join(self.docdir, '.couchappignore') From fa8e61e6c33ae6a089fe84ef937c7f9d25ca6e7f Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 7 Mar 2016 11:29:28 +0800 Subject: [PATCH 058/114] [util] new helper funcs: `is_py2exe` and `is_windows` --- couchapp/generator.py | 8 ++++---- couchapp/util.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index a0da6c2a..b9cda59b 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -12,7 +12,7 @@ from couchapp.errors import AppError from couchapp import localdoc -from couchapp.util import user_path, relpath +from couchapp.util import is_py2exe, is_windows, relpath, user_path __all__ = ["generate_app", "generate_function", "generate"] @@ -185,15 +185,15 @@ def copy_helper(path, directory, tname="templates"): def find_template_dir(name, directory=''): paths = ['%s' % name, os.path.join('..', name)] - if hasattr(sys, 'frozen'): # py2exe + if is_py2exe(): modpath = sys.executable - elif sys.platform == "win32" or os.name == "nt": + elif is_windows(): modpath = os.path.join(sys.prefix, "Lib", "site-packages", "couchapp", "templates") else: modpath = __file__ - if sys.platform != "win32" and os.name != "nt": + if not is_windows(): default_locations = [ "/usr/share/couchapp/templates/%s" % directory, "/usr/local/share/couchapp/templates/%s" % directory, diff --git a/couchapp/util.py b/couchapp/util.py index 094002f6..e8981225 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -71,7 +71,16 @@ def import_module(name, package=None): __import__(name) return sys.modules[name] -if os.name == 'nt': + +def is_windows(): + return sys.platform == 'win32' or os.name == 'nt' + + +def is_py2exe(): + return is_windows() and hasattr(sys, 'frozen') + + +if is_windows(): from win32com.shell import shell, shellcon def user_rcpath(): From b6c6a6f5c78e654be45a96f16d5518750411dda8 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 1 Mar 2016 17:03:53 +0800 Subject: [PATCH 059/114] [generator] rename `start_app` to `init_basic` - add docstring - setup app dir via `util.setup_dir` ref #198 --- couchapp/generator.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index b9cda59b..169addae 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -12,7 +12,7 @@ from couchapp.errors import AppError from couchapp import localdoc -from couchapp.util import is_py2exe, is_windows, relpath, user_path +from couchapp.util import is_py2exe, is_windows, relpath, setup_dir, user_path __all__ = ["generate_app", "generate_function", "generate"] @@ -26,12 +26,20 @@ 'views'] -def start_app(path): - try: - os.makedirs(path) - except OSError, e: - errno, message = e - raise AppError("Can't create a CouchApp in %s: %s" % (path, message)) +def init_basic(path): + ''' + Generate a basic CouchApp which contain following files:: + + /path/ + .couchapprc + .couchappignore + _attachments/ + lists/ + shows/ + updates/ + views/ + ''' + setup_dir(path, require_empty=True) for n in DEFAULT_APP_TREE: tp = os.path.join(path, n) @@ -40,10 +48,9 @@ def start_app(path): fid = os.path.join(path, '_id') if not os.path.isfile(fid): with open(fid, 'wb') as f: - f.write('_design/%s' % os.path.split(path)[1]) + f.write('_design/{0}'.format(os.path.split(path)[1])) localdoc.document(path, create=True) - logger.info("%s created." % path) def generate_app(path, template=None, create=False): From 64ebf4925945a8f84fe36edd2e46ab9eb2f8f042 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 7 Mar 2016 16:19:40 +0800 Subject: [PATCH 060/114] [generator] rename `generate_app` to `init_template` - setup app dir via `util.setup_dir` ref #198 --- couchapp/generator.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index 169addae..05c03cd7 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -14,7 +14,7 @@ from couchapp import localdoc from couchapp.util import is_py2exe, is_windows, relpath, setup_dir, user_path -__all__ = ["generate_app", "generate_function", "generate"] +__all__ = ["init_basic", "init_template", "generate_function", "generate"] logger = logging.getLogger(__name__) @@ -53,24 +53,14 @@ def init_basic(path): localdoc.document(path, create=True) -def generate_app(path, template=None, create=False): - """ Generates a CouchApp in app_dir - - :attr verbose: boolean, default False - :return: boolean, dict. { 'ok': True } if ok, - { 'ok': False, 'error': message } - if something was wrong. - """ - +def init_template(path, template=None): + ''' + Generates a CouchApp via template + ''' TEMPLATES = ['app'] - prefix = '' - if template is not None: - prefix = os.path.join(*template.split('/')) - try: - os.makedirs(path) - except OSError as e: - errno, message = e - raise AppError("Can't create a CouchApp in %s: %s" % (path, message)) + prefix = os.path.join(*template.split('/')) if template is not None else '' + + setup_dir(path, require_empty=True) for n in DEFAULT_APP_TREE: tp = os.path.join(path, n) @@ -99,12 +89,9 @@ def generate_app(path, template=None, create=False): fid = os.path.join(appdir, '_id') if not os.path.isfile(fid): with open(fid, 'wb') as f: - f.write('_design/%s' % os.path.split(appdir)[1]) + f.write('_design/{0}'.format(os.path.split(appdir)[1])) - if create: - localdoc.document(path, create=True) - - logger.info("%s generated." % path) + localdoc.document(path, create=True) def generate_function(path, kind, name, template=None): From 16bfcdeaab1204ee213a2148e03cbf22bc6f8c13 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 7 Mar 2016 16:34:02 +0800 Subject: [PATCH 061/114] [commands] New `init` command Different level of 'init': 1. default 2. empty 3. template 1. level 'default', this will invoke `couchapp.generator.init_basic` $ couchapp init dir dir/ .couchapprc .couchappignore _attachments/ lists/ shows/ updates/ views/ 2. level 'empty' $ couchapp init -e dir dir/ .couchapprc .couchappignore 3. level 'template', invoke `couchapp.generator.init_template` $ couchapp init -t mytemplate dir dir/ .couchapprc .couchappignore ... vendors ... ref #198 --- couchapp/commands.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 60ceec81..5e1badd1 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -24,16 +24,41 @@ def hook(conf, path, hook_type, *args, **kwargs): h.hook(path, hook_type, *args, **kwargs) -def init(conf, path, *args, **opts): +def init(conf, *args, **opts): if not args: dest = os.getcwd() else: dest = os.path.normpath(os.path.join(os.getcwd(), args[0])) if dest is None: - raise AppError("Unknown dest") + raise AppError('Unknown dest') - document(dest, create=True) + if opts['empty'] and opts['template']: + raise AppError('option "empty" cannot use with "template"') + + if util.iscouchapp(dest): + raise AppError("can't create an app at '{0}'. " + "One already exists here.".format(dest)) + + if util.findcouchapp(dest): + raise AppError("can't create an app inside another app '{0}'.".format( + util.findcouchapp(dest))) + + # ``couchapp init -e dest`` + if opts['empty']: + document(dest, create=True) + + # ``couchapp init -t template_name dest`` + elif opts['template']: + generator.init_template(dest, template=opts['template']) + + # ``couchapp init dest`` + else: + generator.init_basic(dest) + + logger.info('{0} created.'.format(dest)) + + return 0 def push(conf, path, *args, **opts): @@ -410,8 +435,9 @@ def get_switch_str(opt): table = { "init": ( init, - [], - "[COUCHAPPDIR]" + [('e', 'empty', False, 'create .couchapprc and .couchappignore only'), + ('t', 'template', '', 'create from template')], + "[OPTION]... [COUCHAPPDIR]" ), "push": ( push, @@ -465,4 +491,4 @@ def get_switch_str(opt): } withcmd = ['generate', 'vendor'] -incouchapp = ['init', 'push', 'generate', 'vendor', 'autopush'] +incouchapp = ['push', 'generate', 'vendor', 'autopush'] From a436ac158bc627eceeca1aedc7d5214d811700d6 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 7 Mar 2016 16:53:27 +0800 Subject: [PATCH 062/114] [commands] Add deprecated warning for command `startapp` --- couchapp/commands.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 5e1badd1..7ed6f60e 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -236,25 +236,13 @@ def clone(conf, source, *args, **opts): def startapp(conf, *args, **opts): - if len(args) < 1: - raise AppError("Can't start an app, name or path is missing") + opts['empty'] = False + opts['template'] = '' - if len(args) == 1: - name = args[0] - dest = os.path.normpath(os.path.join(os.getcwd(), ".", name)) - elif len(args) == 2: - name = args[1] - dest = os.path.normpath(os.path.join(args[0], name)) + logger.warning('"startapp" will be deprecated in future release. ' + 'Please use "init" instead.') - if util.iscouchapp(dest): - raise AppError("can't create an app at '{0}'. " - "One already exists here.".format(dest)) - if util.findcouchapp(dest): - raise AppError("can't create an app inside another app '{0}'.".format( - util.findcouchapp(dest))) - - generator.generate(dest, "startapp", name, **opts) - return 0 + return init(conf, *args, **opts) def generate(conf, path, *args, **opts): From ff2d796d19704ff2e7fbc027c860f5ad5137f2ce Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 7 Mar 2016 17:26:40 +0800 Subject: [PATCH 063/114] [commands] Add deprecated warning for `generate app` --- couchapp/commands.py | 19 +++++++++++-------- couchapp/generator.py | 21 ++++++++------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 7ed6f60e..41d918b6 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -264,15 +264,18 @@ def generate(conf, path, *args, **opts): dest = args[1] name = args[2] + if kind == 'app': # deprecated warning + logger.warning('"genrate app" will be deprecated in future release. ' + 'Please use "init -t TEMPLATE" instead.') + args = (dest,) if dest is not None else tuple() + kwargs = { + 'template': opts['template'] if opts['template'] else 'app', + 'empty': False + } + return init(conf, *args, **kwargs) + if dest is None: - if kind == "app": - dest = os.path.normpath(os.path.join(os.getcwd(), name)) - opts['create'] = True - else: - raise AppError("You aren't in a couchapp.") - elif dest and kind == 'app': - raise AppError("can't create an app inside another app '{0}'.".format( - dest)) + raise AppError("You aren't in a couchapp.") hook(conf, dest, "pre-generate") generator.generate(dest, kind, name, **opts) diff --git a/couchapp/generator.py b/couchapp/generator.py index 05c03cd7..39d64f00 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -222,18 +222,13 @@ def find_template_dir(name, directory=''): def generate(path, kind, name, **opts): - if kind not in ['startapp', 'app', 'view', 'list', 'show', 'filter', + if kind not in ['view', 'list', 'show', 'filter', 'function', 'vendor', 'update', 'spatial']: - raise AppError( - "Can't generate %s in your couchapp. generator is unknown" % kind) + raise AppError("Can't generate {0} in your couchapp. " + 'generator is unknown'.format(kind)) - if kind == "app": - generate_app(path, template=opts.get("template"), - create=opts.get('create', False)) - elif kind == "startapp": - start_app(path) - else: - if name is None: - raise AppError("Can't generate %s function, name is missing" % - kind) - generate_function(path, kind, name, opts.get("template")) + if name is None: + raise AppError("Can't generate {0} function, " + "name is missing".format(kind)) + + generate_function(path, kind, name, opts.get("template")) From 158c92ac8392356c54302e11b6abd28d2ad523d6 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 3 Apr 2016 16:02:05 +0800 Subject: [PATCH 064/114] [commands] Add test cases for new init cmd ref: #198 --- tests/test_cli.py | 12 +-- tests/test_commands.py | 173 +++++++++++------------------------------ 2 files changed, 52 insertions(+), 133 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index b871bc73..8819baa5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -65,10 +65,10 @@ def _retrieve_ddoc(self): self.assertIsNotNone(design_doc) return design_doc - def testGenerate(self): + def test_init_template(self): os.chdir(self.tempdir) child_stdout, child_stderr = sh_open( - '{0} generate my-app'.format(self.cmd)) + '{0} init -t app my-app'.format(self.cmd)) appdir = os.path.join(self.tempdir, 'my-app') self.assertTrue(os.path.isdir(appdir)) cfile = os.path.join(appdir, '.couchapprc') @@ -229,9 +229,9 @@ def testPushApps(self): # create 2 apps child_stdout, child_stderr = sh_open( - '{0} generate docs/app1'.format(self.cmd)) + '{0} init -t app docs/app1'.format(self.cmd)) child_stdout, child_stderr = sh_open( - '{0} generate docs/app2'.format(self.cmd)) + '{0} init -t app docs/app2'.format(self.cmd)) child_stdout, child_stderr = sh_open( '{0} pushapps docs/ {1}couchapp-test'.format(self.cmd, url)) @@ -247,9 +247,9 @@ def testPushDocs(self): # create 2 apps child_stdout, child_stderr = sh_open( - '{0} generate docs/app1'.format(self.cmd)) + '{0} init -t app docs/app1'.format(self.cmd)) child_stdout, child_stderr = sh_open( - '{0} generate docs/app2'.format(self.cmd)) + '{0} init -t app docs/app2'.format(self.cmd)) child_stdout, child_stderr = sh_open( '{0} pushdocs docs/ {1}couchapp-test'.format(self.cmd, url)) diff --git a/tests/test_commands.py b/tests/test_commands.py index e91d9096..c8865b38 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -11,23 +11,57 @@ @patch('couchapp.commands.document') -def test_init_dest(mock_doc): - commands.init(None, None, '/tmp/mk') +def test_init_empty(mock_doc): + ''' + couchapp init -e + ''' + commands.init(None, '/tmp/mk', empty=True, template='') mock_doc.assert_called_once_with('/tmp/mk', create=True) @patch('os.getcwd', return_value='/mock_dir') -@patch('couchapp.commands.document') -def test_init_dest_auto(mock_doc, mock_cwd): - commands.init(None, None) - mock_doc.assert_called_once_with('/mock_dir', create=True) +@patch('couchapp.commands.generator.init_basic') +def test_init_dest_cwd(init_basic, mock_cwd): + ''' + couchapp init + ''' + commands.init(None, empty=False, template='') + init_basic.assert_called_once_with('/mock_dir') + + +@patch('os.path.join', return_value='/mock_dir') +@patch('couchapp.commands.generator.init_basic') +def test_init_dest(init_basic, mock_join): + ''' + couchapp init /mock_dir + ''' + commands.init(None, '/mock_dir', empty=False, template='') + init_basic.assert_called_once_with('/mock_dir') + + +@patch('os.path.join', return_value='/mock_dir') +@patch('couchapp.commands.generator.init_template') +@patch('couchapp.commands.generator.init_basic') +def test_init_template(init_basic, init_template, mock_join): + ''' + couchapp init -t app /mock_dir + ''' + commands.init(None, '/mock_dir', empty=False, template='app') + + assert not init_basic.called + init_template.called_with('/mock_dir', template='app') @raises(AppError) @patch('os.getcwd', return_value=None) @patch('couchapp.commands.document') def test_init_dest_none(mock_doc, mock_cwd): - commands.init(None, None) + commands.init(None) + + +@raises(AppError) +def test_init_dest_opt_conflict(): + commands.init(None, '/mock/app', empty=True, template=True) @patch('couchapp.commands.hook') @@ -311,106 +345,6 @@ def test_clone_default(clone, hook): hook.assert_any_call(conf, dest, 'post-clone', source=src) -@patch('couchapp.commands.os.getcwd', return_value='/') -@patch('couchapp.commands.generator.generate') -def test_startapp_default(generate, getcwd): - ''' - $ couchapp startapp {name} - ''' - conf = NonCallableMock(name='conf') - name = 'mock' - - ret_code = commands.startapp(conf, name) - assert ret_code == 0 - generate.assert_called_with('/mock', 'startapp', name) - - -@patch('couchapp.commands.os.getcwd') -@patch('couchapp.commands.generator.generate') -def test_startapp_default(generate, getcwd): - ''' - $ couchapp startapp {dir} {name} - ''' - conf = NonCallableMock(name='conf') - dir_ = '/' - name = 'mock' - - ret_code = commands.startapp(conf, dir_, name) - assert ret_code == 0 - assert not getcwd.called - generate.assert_called_with('/mock', 'startapp', name) - - -@raises(AppError) -@patch('couchapp.commands.os.getcwd', return_value='/') -@patch('couchapp.commands.generator.generate') -def test_startapp_without_name(generate, getcwd): - ''' - $ couchapp startapp - ''' - conf = NonCallableMock(name='conf') - - ret_code = commands.startapp(conf) - assert not generate.called - - -@raises(AppError) -@patch('couchapp.commands.util.iscouchapp', return_value=True) -@patch('couchapp.commands.os.getcwd', return_value='/') -@patch('couchapp.commands.generator.generate') -def test_startapp_exists(generate, getcwd, iscouchapp): - ''' - $ couchapp startapp {already exists app} - ''' - conf = NonCallableMock(name='conf') - - ret_code = commands.startapp(conf) - assert not generate.called - - -@raises(AppError) -@patch('couchapp.commands.util.iscouchapp', return_value=True) -@patch('couchapp.commands.os.getcwd', return_value='/') -@patch('couchapp.commands.generator.generate') -def test_startapp_exists(generate, getcwd, iscouchapp): - ''' - $ couchapp startapp {already exists app} - ''' - conf = NonCallableMock(name='conf') - name = 'mock' - - ret_code = commands.startapp(conf, name) - assert iscouchapp.assert_called_with('/mock') - assert not generate.called - - -@raises(AppError) -@patch('couchapp.commands.util.findcouchapp', return_value=True) -@patch('couchapp.commands.util.iscouchapp', return_value=False) -@patch('couchapp.commands.os.getcwd', return_value='/') -@patch('couchapp.commands.generator.generate') -def test_startapp_inside_app(generate, getcwd, iscouchapp, findcouchapp): - ''' - $ couchapp startapp {path in another app} - - e.g. Assume there is a couchapp ``app1`` - - :: - app1/ - .couchapprc - ... - - We try to ``couchapp startapp app1/app2``, - and this should raise `AppError`. - ''' - conf = NonCallableMock(name='conf') - name = 'mock' - - ret_code = commands.startapp(conf, name) - assert findcouchapp.assert_called_with('/mock') - assert not generate.called - - @patch('couchapp.commands.util.iscouchapp', return_value=True) @patch('couchapp.commands.document') def test_browse_default(document, iscouchapp): @@ -461,7 +395,8 @@ def test_browse_exist(document, iscouchapp): @raises(AppError) -def test_generate_inside(): +@patch('couchapp.commands.util.iscouchapp', return_value=True) +def test_generate_inside(iscouchapp): ''' $ couchapp generate app {path inside another app} @@ -470,7 +405,9 @@ def test_generate_inside(): conf = NonCallableMock(name='conf') app = '/mock/app' - commands.generate(conf, app, 'app', 'mockapp') + commands.generate(conf, app, 'app', 'mockapp', template='') + + assert iscouchapp.called @raises(AppError) @@ -499,24 +436,6 @@ def test_generate_view_without_app(): commands.generate(conf, None, 'view', 'myview') -@patch('couchapp.commands.os.getcwd', return_value='/mock') -@patch('couchapp.commands.generator.generate') -@patch('couchapp.commands.hook') -def test_generate_app(hook, generate, getcwd): - ''' - $ couchapp generate myapp - ''' - conf = NonCallableMock(name='conf') - kind = 'app' - name = 'myapp' - - ret_code = commands.generate(conf, None, name) - assert ret_code == 0 - generate.assert_called_with('/mock/myapp', kind, name, create=True) - hook.assert_any_call(conf, '/mock/myapp', 'pre-generate') - hook.assert_any_call(conf, '/mock/myapp', 'post-generate') - - @patch('couchapp.commands.os.getcwd', return_value='/mock') @patch('couchapp.commands.generator.generate') @patch('couchapp.commands.hook') From dbd96cfe2c7c346c3d229e94269430cbe69096f5 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 3 Apr 2016 18:05:52 +0800 Subject: [PATCH 065/114] [setup.cfg] Add `bdist_wheel` option --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index e4bf05bc..2903adbb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,3 +8,6 @@ detailed-errors=1 tests=tests verbosity=3 with-coverage=1 + +[bdist_wheel] +python-tag=py2 From 5b08d2e78366f74472314e8ee8a03c55f8ceae13 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 3 Apr 2016 18:21:45 +0800 Subject: [PATCH 066/114] [setup.py] reindent setup options --- setup.py | 98 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index 09492263..b66dd7b8 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def get_packages_data(): packagedata = {'couchapp': []} for root in ('templates',): - for curdir, dirs, files in os.walk(os.path.join("couchapp", root)): + for curdir, dirs, files in os.walk(os.path.join('couchapp', root)): curdir = curdir.split(os.sep, 1)[1] dirs[:] = filter(ordinarypath, dirs) for f in filter(ordinarypath, files): @@ -124,53 +124,59 @@ def main(): except ImportError: INSTALL_REQUIRES.append('simplejson') - options = dict(name='Couchapp', - version=couchapp.__version__, - url='http://github.com/couchapp/couchapp/tree/master', - license='Apache License 2', - author='Benoit Chesneau', - author_email='benoitc@e-engura.org', - description='Standalone CouchDB Application Development Made Simple.', - long_description=long_description, - tests_require = ['unittest2', 'nose', 'coverage', - 'nose-testconfig', 'mock'], - test_suite="tests", - keywords='couchdb couchapp', - platforms=['any'], - classifiers=CLASSIFIERS, - packages=find_packages(), - data_files=DATA_FILES, - include_package_data=True, - zip_safe=False, - install_requires=INSTALL_REQUIRES, - scripts=get_scripts(), - options=dict(py2exe={'dll_excludes': ["kernelbase.dll", - "powrprof.dll"], - 'packages': ["http_parser", - "restkit", - "restkit.contrib", - "pathtools", - "pathtools.path", - "socketpool", - "watchdog", - "watchdog.observers", - "watchdog.tricks", - "watchdog.utils", - "win32pdh", - "win32pdhutil", - "win32api", - "win32con", - "subprocess" - ] - }, - bdist_mpkg=dict(zipdist=True, - license='LICENSE', - readme='resources/macosx/Readme.html', - welcome='resources/macosx/Welcome.html') - ) - ) + options = dict( + name='Couchapp', + version=couchapp.__version__, + url='http://github.com/couchapp/couchapp/tree/master', + license='Apache License 2', + author='Benoit Chesneau', + author_email='benoitc@e-engura.org', + description='Standalone CouchDB Application Development Made Simple.', + long_description=long_description, + tests_require = ['unittest2', 'nose', 'coverage', + 'nose-testconfig', 'mock'], + test_suite="tests", + keywords='couchdb couchapp', + platforms=['any'], + classifiers=CLASSIFIERS, + packages=find_packages(), + data_files=DATA_FILES, + include_package_data=True, + zip_safe=False, + install_requires=INSTALL_REQUIRES, + scripts=get_scripts(), + options=dict( + py2exe={ + 'dll_excludes': ["kernelbase.dll", "powrprof.dll"], + 'packages': [ + "http_parser", + "restkit", + "restkit.contrib", + "pathtools", + "pathtools.path", + "socketpool", + "watchdog", + "watchdog.observers", + "watchdog.tricks", + "watchdog.utils", + "win32pdh", + "win32pdhutil", + "win32api", + "win32con", + "subprocess", + ] + }, + bdist_mpkg=dict( + zipdist=True, + license='LICENSE', + readme='resources/macosx/Readme.html', + welcome='resources/macosx/Welcome.html' + ), + ), + ) options.update(extra) setup(**options) + if __name__ == "__main__": main() From ae1f4f6fbc956905e8d8a61c9498877befda9f4d Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Fri, 15 Apr 2016 22:59:41 +0800 Subject: [PATCH 067/114] [doc] Fix version info in docs/conf.py --- docs/conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 91552d2c..d8942efc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,9 +61,12 @@ # built documents. # # The short X.Y version. -version = '1.0.1' +from couchapp import version_info +version = '.'.join(map(str, version_info)) # The full version, including alpha/beta/rc tags. -release = '1.0.1' +release = '.'.join(map(str, version_info)) + +del version_info # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 97f6d45279b26b7d3ef93e10d2bb503128b18aa9 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 18 Apr 2016 17:51:45 +0800 Subject: [PATCH 068/114] [util.py] new helper func `setup_dirs` test case included --- couchapp/util.py | 12 ++++++++++++ tests/test_util.py | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/couchapp/util.py b/couchapp/util.py index e8981225..0b87c22d 100644 --- a/couchapp/util.py +++ b/couchapp/util.py @@ -579,3 +579,15 @@ def setup_dir(path, require_empty=True): raise AppError("'{0}': File exists".format(path)) os.mkdir(path) + + +def setup_dirs(path_list, *args, **kwargs): + ''' + setup a list of dirs. + + :param path_list: iterable + + Other arguments please refer to ``setup_dir``. + ''' + for p in path_list: + setup_dir(p, *args, **kwargs) diff --git a/tests/test_util.py b/tests/test_util.py index 956b895f..a9c815a8 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -7,6 +7,7 @@ from couchapp.errors import AppError from couchapp.util import discover_apps, iscouchapp, rcpath, split_path from couchapp.util import sh_open, remove_comments, is_empty_dir, setup_dir +from couchapp.util import setup_dirs from mock import patch from nose.tools import raises @@ -251,3 +252,11 @@ def test(self, mkdir): def teardown(self): del self.tmpfile + + +@patch('couchapp.util.setup_dir') +def test_setup_dirs(setup_dir): + plist = ['/mock', '/fake', '/mock/app', '/42'] + setup_dirs(plist) + + assert setup_dir.called From cbc20a115f2d556aaeb191ff0049f81978b07ebf Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 19 Apr 2016 12:19:04 +0800 Subject: [PATCH 069/114] [util.py] Add test case for `setup_dirs` --- tests/test_util.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index a9c815a8..1d6e3bfb 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -260,3 +260,19 @@ def test_setup_dirs(setup_dir): setup_dirs(plist) assert setup_dir.called + setup_dir.assert_any_call('/mock') + setup_dir.assert_any_call('/fake') + setup_dir.assert_any_call('/mock/app') + setup_dir.assert_any_call('/42') + + +@patch('couchapp.util.setup_dir') +def test_setup_dirs_args(setup_dir): + plist = ['/mock', '/fake', '/mock/app', '/42'] + setup_dirs(plist, require_empty=True) + + assert setup_dir.called + setup_dir.assert_any_call('/mock', require_empty=True) + setup_dir.assert_any_call('/fake', require_empty=True) + setup_dir.assert_any_call('/mock/app', require_empty=True) + setup_dir.assert_any_call('/42', require_empty=True) From e744c8a398a8ca3e33f0a5149e0a63ab2bd9c9e8 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 19 Apr 2016 12:40:11 +0800 Subject: [PATCH 070/114] [generator] new helper function `save_id` ref: #230 --- couchapp/generator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/couchapp/generator.py b/couchapp/generator.py index 39d64f00..ad6cc6e9 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -232,3 +232,15 @@ def generate(path, kind, name, **opts): "name is missing".format(kind)) generate_function(path, kind, name, opts.get("template")) + + +def save_id(app_path, name): + ''' + Save ``name`` into ``app_path/_id`` file. + if file exists, we will overwride it. + + :param str app_dir: + :param str name: + ''' + with open(os.path.join(app_path, '_id'), 'wb') as f: + f.write(name) From 0e13afddc3bfadd62b96dacd3fd8ca48a65a4163 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 19 Apr 2016 12:42:22 +0800 Subject: [PATCH 071/114] [generator] Test cases for `init_basic` and `save_id` ref: #230 --- couchapp/generator.py | 12 +++--------- tests/test_generator.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 tests/test_generator.py diff --git a/couchapp/generator.py b/couchapp/generator.py index ad6cc6e9..22ec20a0 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -13,6 +13,7 @@ from couchapp.errors import AppError from couchapp import localdoc from couchapp.util import is_py2exe, is_windows, relpath, setup_dir, user_path +from couchapp.util import setup_dirs __all__ = ["init_basic", "init_template", "generate_function", "generate"] @@ -40,16 +41,9 @@ def init_basic(path): views/ ''' setup_dir(path, require_empty=True) + setup_dirs(os.path.join(path, n) for n in DEFAULT_APP_TREE) - for n in DEFAULT_APP_TREE: - tp = os.path.join(path, n) - os.makedirs(tp) - - fid = os.path.join(path, '_id') - if not os.path.isfile(fid): - with open(fid, 'wb') as f: - f.write('_design/{0}'.format(os.path.split(path)[1])) - + save_id(path, '_design/{0}'.format(os.path.split(path)[-1])) localdoc.document(path, create=True) diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 00000000..573b1033 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from couchapp.generator import init_basic, save_id + +from mock import patch + + +@patch('couchapp.generator.save_id') +@patch('couchapp.generator.setup_dirs') +@patch('couchapp.generator.setup_dir') +@patch('couchapp.generator.localdoc') +def test_init_basic(localdoc, setup_dir, setup_dirs, save_id): + init_basic('/mock/app') + + assert setup_dir.called + assert setup_dirs.called + assert save_id.called + assert localdoc.document.called + + +@patch('couchapp.generator.open') +def test_save_id(open_): + save_id('/mock/app', 'someid') + + open_.assert_called_with('/mock/app/_id', 'wb') From 0939f25691ece5bf5fbc9b0ad0c7676635a47cdd Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 3 May 2016 21:30:24 +0800 Subject: [PATCH 072/114] [generator] reimplement `find_template_dir` - Document the search path in diff platforms - Let user custom dir win first close #231 see also #228 ref #230 --- couchapp/generator.py | 122 +++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 43 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index 22ec20a0..17c5ea0f 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -93,9 +93,9 @@ def generate_function(path, kind, name, template=None): if template: functions_path = [] _relpath = os.path.join(*template.split('/')) - template_dir = find_template_dir("templates", _relpath) + template_dir = find_template_dir(_relpath) else: - template_dir = find_template_dir("templates") + template_dir = find_template_dir() if template_dir: functions = [] if kind == "view": @@ -171,48 +171,84 @@ def copy_helper(path, directory, tname="templates"): (path)) -def find_template_dir(name, directory=''): - paths = ['%s' % name, os.path.join('..', name)] - if is_py2exe(): - modpath = sys.executable +def find_template_dir(tmpl_name, tmpl_type, raise_error=False): + ''' + Find template dir for different platform + + :param tmpl_name: The template name under ``templates``. + It can be empty string. + e.g. ``mytmpl`` mentioned in the docstring of + :py:func:`~couchapp.generate.init_template` + :param tmpl_type: the type of template. + e.g. 'app', 'functions', 'vendor' + :param bool raise_error: raise ``AppError`` if not found + :return: the absolute path or ``None`` if not found + + We will check the ``/templates//`` is + dir or not. The first matched win. + + For posix platform, the search locations are following: + - ~/.couchapp/ + - / + - /../ + - /usr/share/couchapp/ + - /usr/local/share/couchapp/ + - /opt/couchapp/ + + For darwin (OSX) platform, we have some extra search locations: + - ${HOME}/Library/Application Support/Couchapp/ + + For windows with standlone binary (py2exe): + - / + - /../ + + For windows with python interpreter: + - ${USERPROFILE}/.couchapp/ + - / + - /../ + - /Lib/site-packages/couchapp/ + ''' + modpath = os.path.dirname(__file__) + search_paths = user_path() + [ + modpath, + os.path.join(modpath, '..'), + ] + + if os.name == 'posix': + search_paths.extend([ + '/usr/share/couchapp', + '/usr/local/share/couchapp', + '/opt/couchapp', + ]) + elif is_py2exe(): + search_paths.append(os.path.dirname(sys.executable)) elif is_windows(): - modpath = os.path.join(sys.prefix, "Lib", "site-packages", "couchapp", - "templates") - else: - modpath = __file__ - - if not is_windows(): - default_locations = [ - "/usr/share/couchapp/templates/%s" % directory, - "/usr/local/share/couchapp/templates/%s" % directory, - "/opt/couchapp/templates/%s" % directory] - - else: - default_locations = [] - - default_locations.extend([os.path.join(os.path.dirname(modpath), p, - directory) for p in paths]) - - if sys.platform == "darwin": - home = os.path.expanduser('~'), - data_path = "%s/Library/Application Support/Couchapp" % home - default_locations.extend(["%s/%s/%s" % (data_path, p, directory) - for p in paths]) - - if directory: - for user_location in user_path(): - default_locations.append(os.path.join(user_location, name, - directory)) - - found = False - for location in default_locations: - template_dir = os.path.normpath(location) - if os.path.isdir(template_dir): - found = True - break - if found: - return template_dir - return False + search_paths.append( + os.path.join(sys.prefix, 'Lib', 'site-packages', 'couchapp') + ) + + # extra path for darwin + if sys.platform.startswith('darwin'): + search_paths.append( + os.path.expanduser('~/Library/Application Support/Couchapp') + ) + + # the first win! + for path in search_paths: + path = os.path.normpath(path) + path = os.path.join(path, 'templates', tmpl_name, tmpl_type) + if os.path.isdir(path): + logger.debug('template path match: "{0}"'.format(path)) + return path + + logger.debug('template search path: "{0}" not found'.format(path)) + + if raise_error: + logger.info('please use "-d" to checkout search paths.') + raise AppError('template "{0}/{1}" not found.'.format( + tmpl_name, tmpl_type)) + + return None def generate(path, kind, name, **opts): From df943910fc0f54c31a052c95ff0921cbfd93c164 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 3 May 2016 22:25:24 +0800 Subject: [PATCH 073/114] [generator] test cases for `find_template_dir` ref #230 --- couchapp/generator.py | 5 ++++- tests/test_generator.py | 50 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index 17c5ea0f..176085ec 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -171,7 +171,7 @@ def copy_helper(path, directory, tname="templates"): (path)) -def find_template_dir(tmpl_name, tmpl_type, raise_error=False): +def find_template_dir(tmpl_name='', tmpl_type='', raise_error=False): ''' Find template dir for different platform @@ -208,6 +208,9 @@ def find_template_dir(tmpl_name, tmpl_type, raise_error=False): - /../ - /Lib/site-packages/couchapp/ ''' + if tmpl_type and tmpl_type not in TEMPLATE_TYPES: + raise AppError('invalid template type "{0}"'.format(tmpl_type)) + modpath = os.path.dirname(__file__) search_paths = user_path() + [ modpath, diff --git a/tests/test_generator.py b/tests/test_generator.py index 573b1033..7883a6ea 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- -from couchapp.generator import init_basic, save_id +from couchapp.errors import AppError +from couchapp.generator import find_template_dir, init_basic, save_id from mock import patch +from nose.tools import raises @patch('couchapp.generator.save_id') @@ -23,3 +25,49 @@ def test_save_id(open_): save_id('/mock/app', 'someid') open_.assert_called_with('/mock/app/_id', 'wb') + + +@patch('couchapp.generator.os.path.isdir', return_value=False) +def test_find_template_dir_not_found(isdir): + assert find_template_dir() is None + assert isdir.called + + +@patch('couchapp.generator.os.path.isdir', return_value=False) +def test_find_template_dir_not_found_raise(isdir): + @raises(AppError) + def f(): + find_template_dir(raise_error=True) + + f() + assert isdir.called + + +@patch('couchapp.generator.os.path.isdir', return_value=False) +def test_find_template_dir_template_type_error(isdir): + @raises(AppError) + def f(): + find_template_dir(tmpl_type='mock_type') + + f() + assert not isdir.called + + +@patch('couchapp.generator.user_path', return_value=['/mock/.couchapp']) +@patch('couchapp.generator.os.path.isdir', return_value=True) +def test_find_template_dir_user_dir_first(isdir, user_path): + ret = find_template_dir() + + assert ret == '/mock/.couchapp/templates/', ret + assert user_path.called + assert isdir.called + + +@patch('couchapp.generator.user_path', return_value=['/mock/.couchapp']) +@patch('couchapp.generator.os.path.isdir', return_value=True) +def test_find_template_dir_user_dir_first_with_type(isdir, user_path): + ret = find_template_dir(tmpl_type='app') + + assert ret == '/mock/.couchapp/templates/app', ret + assert user_path.called + assert isdir.called From 6901ede0e0749b4e1ca5b06f62add0253b505461 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 3 May 2016 23:49:51 +0800 Subject: [PATCH 074/114] [generator] reimplement `copy_helper` Let `copy_helper` behave like `shutil.copytree`, but it do not require dest dir non-exist. Test cases included. ref: #230 --- couchapp/generator.py | 77 ++++++++++++++++++++++++----------------- tests/test_generator.py | 47 ++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 33 deletions(-) diff --git a/couchapp/generator.py b/couchapp/generator.py index 176085ec..4fd0aebb 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -7,11 +7,12 @@ import logging import os -import shutil import sys -from couchapp.errors import AppError +from shutil import copy2, copytree + from couchapp import localdoc +from couchapp.errors import AppError from couchapp.util import is_py2exe, is_windows, relpath, setup_dir, user_path from couchapp.util import setup_dirs @@ -39,6 +40,8 @@ def init_basic(path): shows/ updates/ views/ + + .. versionadded:: 1.1 ''' setup_dir(path, require_empty=True) setup_dirs(os.path.join(path, n) for n in DEFAULT_APP_TREE) @@ -138,45 +141,50 @@ def generate_function(path, kind, name, template=None): raise AppError("Defaults templates not found. Check your install.") -def copy_helper(path, directory, tname="templates"): - """ copy helper used to generate an app""" - if tname == "vendor": - tname = os.path.join("templates", tname) +def copy_helper(src, dest): + ''' + copy helper similar to ``shutil.copytree`` - templatedir = find_template_dir(tname, directory) - if templatedir: - if directory == "vendor": - path = os.path.join(path, directory) - try: - os.makedirs(path) - except: - pass + But we do not require ``dest`` non-exist - for root, dirs, files in os.walk(templatedir): - rel = relpath(root, templatedir) - if rel == ".": - rel = "" - target_path = os.path.join(path, rel) - for d in dirs: - try: - os.makedirs(os.path.join(target_path, d)) - except: - continue - for f in files: - shutil.copy2(os.path.join(root, f), - os.path.join(target_path, f)) - else: - raise AppError( - "Can't create a CouchApp in %s: default template not found." % - (path)) + :param str src: source dir + :param str dest: destination dir + + e.g:: + + foo/ + bar.txt + baz/ + *empty dir* -def find_template_dir(tmpl_name='', tmpl_type='', raise_error=False): + ``copy_helper('foo', 'bar')`` will copy ``bar.txt`` as ``baz/bar.txt``. + + ..versionchanged: 1.1 + ''' + if not os.path.isdir(src): + raise OSError('source "{0}" is not a directory'.format(src)) + + setup_dir(dest, require_empty=False) + + for p in os.listdir(src): + _src = os.path.join(src, p) + _dest = os.path.join(dest, p) + + if os.path.isdir(_src): + copytree(_src, _dest) + else: + copy2(_src, _dest) + + +def find_template_dir(tmpl_name='default', tmpl_type='', raise_error=False): ''' Find template dir for different platform :param tmpl_name: The template name under ``templates``. It can be empty string. + If it is set to ``default``, we will use consider + the tmpl_name as empty. e.g. ``mytmpl`` mentioned in the docstring of :py:func:`~couchapp.generate.init_template` :param tmpl_type: the type of template. @@ -207,10 +215,15 @@ def find_template_dir(tmpl_name='', tmpl_type='', raise_error=False): - / - /../ - /Lib/site-packages/couchapp/ + + ..versionchanged:: 1.1 ''' if tmpl_type and tmpl_type not in TEMPLATE_TYPES: raise AppError('invalid template type "{0}"'.format(tmpl_type)) + if tmpl_name == 'default': + tmpl_name = '' + modpath = os.path.dirname(__file__) search_paths = user_path() + [ modpath, diff --git a/tests/test_generator.py b/tests/test_generator.py index 7883a6ea..460f6ab7 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- +import tempfile + from couchapp.errors import AppError +from couchapp.generator import copy_helper from couchapp.generator import find_template_dir, init_basic, save_id from mock import patch -from nose.tools import raises +from nose.tools import raises, with_setup @patch('couchapp.generator.save_id') @@ -71,3 +74,45 @@ def test_find_template_dir_user_dir_first_with_type(isdir, user_path): assert ret == '/mock/.couchapp/templates/app', ret assert user_path.called assert isdir.called + + +@raises(OSError) +def test_copy_helper_invalid_src(): + copy_helper('/mock', '/fake') + + +@patch('couchapp.generator.copytree') +@patch('couchapp.generator.copy2') +@patch('couchapp.generator.os.listdir', return_value=['foo', 'bar']) +@patch('couchapp.generator.os.path.isdir') +@patch('couchapp.generator.setup_dir') +def test_copy_helper_file_only(setup_dir, isdir, listdir, copy2, copytree): + def _isdir(p): + if p == '/mock': + return True + return False + + isdir.side_effect = _isdir + + copy_helper('/mock', '/fake') + + assert setup_dir.called + assert copy2.called + assert not copytree.called + copy2.assert_any_call('/mock/foo', '/fake/foo') + copy2.assert_any_call('/mock/bar', '/fake/bar') + + +@patch('couchapp.generator.copytree') +@patch('couchapp.generator.copy2') +@patch('couchapp.generator.os.listdir', return_value=['foo', 'bar']) +@patch('couchapp.generator.os.path.isdir', return_value=True) +@patch('couchapp.generator.setup_dir') +def test_copy_helper_dir_only(setup_dir, isdir, listdir, copy2, copytree): + copy_helper('/mock', '/fake') + + assert setup_dir.called + assert copytree.called + assert not copy2.called + copytree.assert_any_call('/mock/foo', '/fake/foo') + copytree.assert_any_call('/mock/bar', '/fake/bar') From 83a0b35d8380ff4a03aa18ea39861fca32dc7b95 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 4 May 2016 10:20:07 +0800 Subject: [PATCH 075/114] [generator] code refinement for `init_template` We redefine the structure of ``templates`` dir. The ``default`` template is defined as following. If we invoke ``couchapp init -t default myapp``, we will get a copy of ``templates/app`` and ``templates/vendor``. tempaltes/ app/ functions/ vendor/ There is three types of templates -- 'app', 'function', and 'vendor' We can identify tempalte set via name. e.g. we have template set named as ``mytmpl`` templates/ ... mytmpl/ app/ functions/ vendor/ Ref #230 Close #232 --- couchapp/commands.py | 2 +- couchapp/generator.py | 100 +++++++++++++++++++++++++++--------------- tests/test_cli.py | 10 ++--- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/couchapp/commands.py b/couchapp/commands.py index 41d918b6..35ce22c0 100644 --- a/couchapp/commands.py +++ b/couchapp/commands.py @@ -269,7 +269,7 @@ def generate(conf, path, *args, **opts): 'Please use "init -t TEMPLATE" instead.') args = (dest,) if dest is not None else tuple() kwargs = { - 'template': opts['template'] if opts['template'] else 'app', + 'template': opts['template'] if opts['template'] else 'default', 'empty': False } return init(conf, *args, **kwargs) diff --git a/couchapp/generator.py b/couchapp/generator.py index 4fd0aebb..6fb8b84f 100644 --- a/couchapp/generator.py +++ b/couchapp/generator.py @@ -20,12 +20,19 @@ logger = logging.getLogger(__name__) +DEFAULT_APP_TREE = ( + '_attachments', + 'lists', + 'shows', + 'updates', + 'views', +) -DEFAULT_APP_TREE = ['_attachments', - 'lists', - 'shows', - 'updates', - 'views'] +TEMPLATE_TYPES = ( + 'app', + 'functions', + 'vendor', +) def init_basic(path): @@ -50,44 +57,67 @@ def init_basic(path): localdoc.document(path, create=True) -def init_template(path, template=None): +def init_template(path, template='default'): ''' Generates a CouchApp via template + + :param str path: the app dir + :param str template: the templates set name. In following example, it is + ``mytmpl``. + + We expect template dir has following structure:: + + templates/ + app/ + functions/ + vendor/ + + mytmpl/ + app/ + functions/ + vendor/ + + vuejs/ + myvue/ + app/ + functions/ + vendor/ + vueform/ + app/ + functions/ + vendor/ + + The ``templates/app`` will be used as default app template. + ``templates/functions`` and ``templates/vender`` are default, also. + + And we can create a dir ``mytmpl`` as custom template set. + The template set name can be nested, e.g. ``vuejs/myvue``. + + ..versionadded:: 1.1 ''' - TEMPLATES = ['app'] - prefix = os.path.join(*template.split('/')) if template is not None else '' + if template in TEMPLATE_TYPES: + raise AppError('template name connot be {0}.'.format(TEMPLATE_TYPES)) - setup_dir(path, require_empty=True) + tmpl_name = os.path.normpath(template) if template else '' - for n in DEFAULT_APP_TREE: - tp = os.path.join(path, n) - os.makedirs(tp) - - for t in TEMPLATES: - appdir = path - if prefix: - # we do the job twice for now to make sure an app or vendor - # template exist in user template location - # fast on linux since there is only one user dir location - # but could be a little slower on windows - for user_location in user_path(): - location = os.path.join(user_location, 'templates', prefix, t) - if os.path.exists(location): - t = os.path.join(prefix, t) - break - - copy_helper(appdir, t) + # copy ``