Update manage_externals for Python 3+ big fix, add support for Python…

… 3.7 and 3.8 (#23)

manage_externals/.gitignore

@@ -12,3 +12,6 @@ components/
 
 # generated python files
 *.pyc
+
+# test tmp file
+test/tmp

manage_externals/.travis.yml

@@ -1,25 +1,12 @@
-# NOTE(bja, 2017-11) travis-ci dosen't support python language builds
-# on mac os. As a work around, we use built-in python on linux, and
-# declare osx a 'generic' language, and create our own python env.
-
 language: python
 os: linux
 python: 
  - "2.7"
  - "3.4"
  - "3.5"
  - "3.6"
-matrix:
- include:
- - os: osx
- language: generic
- before_install:
- # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough!
- # - brew update
- # - brew outdated python2 || brew upgrade python2
- - pip install virtualenv
- - virtualenv env -p python2
- - source env/bin/activate
+ - "3.7"
+ - "3.8"
 install:
  - pip install -r test/requirements.txt
 before_script:

manage_externals/manic/checkout.py

@@ -279,6 +279,9 @@ def commandline_arguments(args=None):
  help='The externals description filename. '
  'Default: %(default)s.')
 
+ parser.add_argument('-x', '--exclude', nargs='*',
+ help='Component(s) listed in the externals file which should be ignored.')
+
  parser.add_argument('-o', '--optional', action='store_true', default=False,
  help='By default only the required externals '
  'are checked out. This flag will also checkout the '
@@ -362,7 +365,7 @@ def main(args):
  root_dir = os.path.abspath(os.getcwd())
  external_data = read_externals_description_file(root_dir, args.externals)
  external = create_externals_description(
- external_data, components=args.components)
+ external_data, components=args.components, exclude=args.exclude)
 
  for comp in args.components:
  if comp not in external.keys():
@@ -389,17 +392,25 @@ def main(args):
  # exit gracefully
  msg = """The external repositories labeled with 'M' above are not in a clean state.
 
-The following are two options for how to proceed:
+The following are three options for how to proceed:
 
-(1) Go into each external that is not in a clean state and issue either
- an 'svn status' or a 'git status' command. Either revert or commit
- your changes so that all externals are in a clean state. (Note,
- though, that it is okay to have untracked files in your working
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+ an 'svn status' command (depending on whether the external is managed by git or
+ svn). Either revert or commit your changes so that all externals are in a clean
+ state. (To revert changes in git, follow the instructions given when you run 'git
+ status'.) (Note, though, that it is okay to have untracked files in your working
  directory.) Then rerun {program_name}.
 
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you
- can manually update out-of-sync externals (labeled with 's' above)
- as described in the configuration file {config_file}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+ update out-of-sync externals (labeled with 's' above) as described in the
+ configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+ commands to checkout the appropriate tags for each external, as given in
+ {config_file}.)
+
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+ one or more externals to ignore using the '-x' or '--exclude' argument to
+ {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+ update the other, non-excluded externals.
 
 
 The external repositories labeled with '?' above are not under version

manage_externals/manic/externals_description.py

@@ -180,6 +180,9 @@ def parse_submodules_desc_section(section_items, file_path):
  path = item[1].strip()
  elif name == 'url':
  url = item[1].strip()
+ elif name == 'branch':
+ # We do not care about branch since we have a hash - silently ignore
+ pass
  else:
  msg = 'WARNING: Ignoring unknown {} property, in {}'
  msg = msg.format(item[0], file_path) # fool pylint
@@ -261,18 +264,18 @@ def read_gitmodules_file(root_dir, file_name):
  return externals_description
 
 def create_externals_description(
- model_data, model_format='cfg', components=None, parent_repo=None):
+ model_data, model_format='cfg', components=None, exclude=None, parent_repo=None):
  """Create the a externals description object from the provided data
  """
  externals_description = None
  if model_format == 'dict':
  externals_description = ExternalsDescriptionDict(
- model_data, components=components)
+ model_data, components=components, exclude=exclude)
  elif model_format == 'cfg':
  major, _, _ = get_cfg_schema_version(model_data)
  if major == 1:
  externals_description = ExternalsDescriptionConfigV1(
- model_data, components=components, parent_repo=parent_repo)
+ model_data, components=components, exclude=exclude, parent_repo=parent_repo)
  else:
  msg = ('Externals description file has unsupported schema '
  'version "{0}".'.format(major))
@@ -707,7 +710,7 @@ class ExternalsDescriptionDict(ExternalsDescription):
 
  """
 
- def __init__(self, model_data, components=None):
+ def __init__(self, model_data, components=None, exclude=None):
  """Parse a native dictionary into a externals description.
  """
  ExternalsDescription.__init__(self)
@@ -719,10 +722,15 @@ def __init__(self, model_data, components=None):
  self._input_patch = 0
  self._verify_schema_version()
  if components:
- for key in model_data.items():
+ for key in list(model_data.keys()):
  if key not in components:
  del model_data[key]
 
+ if exclude:
+ for key in list(model_data.keys()):
+ if key in exclude:
+ del model_data[key]
+
  self.update(model_data)
  self._check_user_input()
 
@@ -733,7 +741,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription):
 
  """
 
- def __init__(self, model_data, components=None, parent_repo=None):
+ def __init__(self, model_data, components=None, exclude=None, parent_repo=None):
  """Convert the config data into a standardized dict that can be used to
  construct the source objects
 
@@ -746,7 +754,7 @@ def __init__(self, model_data, components=None, parent_repo=None):
  get_cfg_schema_version(model_data)
  self._verify_schema_version()
  self._remove_metadata(model_data)
- self._parse_cfg(model_data, components=components)
+ self._parse_cfg(model_data, components=components, exclude=exclude)
  self._check_user_input()
 
  @staticmethod
@@ -758,7 +766,7 @@ def _remove_metadata(model_data):
  """
  model_data.remove_section(DESCRIPTION_SECTION)
 
- def _parse_cfg(self, cfg_data, components=None):
+ def _parse_cfg(self, cfg_data, components=None, exclude=None):
  """Parse a config_parser object into a externals description.
  """
  def list_to_dict(input_list, convert_to_lower_case=True):
@@ -775,7 +783,7 @@ def list_to_dict(input_list, convert_to_lower_case=True):
 
  for section in cfg_data.sections():
  name = config_string_cleaner(section.lower().strip())
- if components and name not in components:
+ if (components and name not in components) or (exclude and name in exclude):
  continue
  self[name] = {}
  self[name].update(list_to_dict(cfg_data.items(section)))

manage_externals/manic/repository_svn.py

@@ -220,9 +220,8 @@ def xml_status_is_dirty(svn_output):
  continue
  if item == SVN_UNVERSIONED:
  continue
- else:
- is_dirty = True
- break
+ is_dirty = True
+ break
  return is_dirty
 
  # ----------------------------------------------------------------

manage_externals/manic/sourcetree.py

@@ -299,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR):
  for comp in load_comps:
  printlog('{0}, '.format(comp), end='')
  stat = self._all_components[comp].status()
+ stat_final = {}
  for name in stat.keys():
  # check if we need to append the relative_path_base to
  # the path so it will be sorted in the correct order.
- if not stat[name].path.startswith(relative_path_base):
- stat[name].path = os.path.join(relative_path_base,
- stat[name].path)
- # store under key = updated path, and delete the
- # old key.
- comp_stat = stat[name]
- del stat[name]
- stat[comp_stat.path] = comp_stat
- summary.update(stat)
+ if stat[name].path.startswith(relative_path_base):
+ # use as is, without any changes to path
+ stat_final[name] = stat[name]
+ else:
+ # append relative_path_base to path and store under key = updated path
+ modified_path = os.path.join(relative_path_base,
+ stat[name].path)
+ stat_final[modified_path] = stat[name]
+ stat_final[modified_path].path = modified_path
+ summary.update(stat_final)
 
  return summary
 

manage_externals/test/test_sys_checkout.py

@@ -819,10 +819,15 @@ def _check_container_component_post_checkout(self, overall, tree):
 
  def _check_container_component_post_checkout2(self, overall, tree):
  self.assertEqual(overall, 0)
- self._check_simple_opt_ok(tree)
  self._check_simple_tag_empty(tree)
  self._check_simple_branch_ok(tree)
 
+ def _check_container_component_post_checkout3(self, overall, tree):
+ self.assertEqual(overall, 0)
+ self.assertFalse("simp_opt" in tree)
+ self._check_simple_tag_ok(tree)
+ self._check_simple_branch_ok(tree)
+
  def _check_container_full_post_checkout(self, overall, tree):
  self.assertEqual(overall, 0)
  self._check_simple_tag_ok(tree)
@@ -1347,6 +1352,23 @@ def test_container_component(self):
  self.status_args)
  self._check_container_component_post_checkout2(overall, tree)
 
+ def test_container_exclude_component(self):
+ """Verify that exclude component checkout works
+ """
+ # create the test repository
+ under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME)
+
+ # create the top level externals file
+ self._generator.container_full(under_test_dir)
+
+ # inital checkout, exclude simp_opt
+ checkout_args = ['--exclude', 'simp_opt']
+ checkout_args.extend(self.checkout_args)
+ overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args)
+ checkout_args.append("--status")
+ overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args)
+ self._check_container_component_post_checkout3(overall, tree)
+
  def test_mixed_simple(self):
  """Verify that a mixed use repo can serve as a 'full' container,
  pulling in a set of externals and a seperate set of sub-externals.

manage_externals/test/test_unit_externals_description.py

@@ -342,6 +342,40 @@ def setup_config(self):
  # NOTE(goldy, 2019-03) Should test other possible keywords such as
  # fetchRecurseSubmodules, ignore, and shallow
 
+ @staticmethod
+ def setup_dict_config():
+ """Create the full container dictionary with simple and mixed use
+ externals
+
+ """
+ rdatat = {ExternalsDescription.PROTOCOL: 'git',
+ ExternalsDescription.REPO_URL: 'simple-ext.git',
+ ExternalsDescription.TAG: 'tag1'}
+ rdatab = {ExternalsDescription.PROTOCOL: 'git',
+ ExternalsDescription.REPO_URL: 'simple-ext.git',
+ ExternalsDescription.BRANCH: 'feature2'}
+ rdatam = {ExternalsDescription.PROTOCOL: 'git',
+ ExternalsDescription.REPO_URL: 'mixed-cont-ext.git',
+ ExternalsDescription.BRANCH: 'master'}
+ desc = {'simp_tag': {ExternalsDescription.REQUIRED: True,
+ ExternalsDescription.PATH: 'simp_tag',
+ ExternalsDescription.EXTERNALS: EMPTY_STR,
+ ExternalsDescription.REPO: rdatat},
+ 'simp_branch' : {ExternalsDescription.REQUIRED: True,
+ ExternalsDescription.PATH: 'simp_branch',
+ ExternalsDescription.EXTERNALS: EMPTY_STR,
+ ExternalsDescription.REPO: rdatab},
+ 'simp_opt': {ExternalsDescription.REQUIRED: False,
+ ExternalsDescription.PATH: 'simp_opt',
+ ExternalsDescription.EXTERNALS: EMPTY_STR,
+ ExternalsDescription.REPO: rdatat},
+ 'mixed_req': {ExternalsDescription.REQUIRED: True,
+ ExternalsDescription.PATH: 'mixed_req',
+ ExternalsDescription.EXTERNALS: 'sub-ext.cfg',
+ ExternalsDescription.REPO: rdatam}}
+
+ return desc
+
  def test_cfg_v1_ok(self):
  """Test that a correct cfg v1 object is created by create_externals_description
 
@@ -379,6 +413,49 @@ def test_dict(self):
  ext = create_externals_description(desc, model_format='dict')
  self.assertIsInstance(ext, ExternalsDescriptionDict)
 
+ def test_cfg_component_dict(self):
+ """Verify that create_externals_description works with a dictionary
+ """
+ # create the top level externals file
+ desc = self.setup_dict_config()
+ # Check external with all repos
+ external = create_externals_description(desc, model_format='dict')
+ self.assertIsInstance(external, ExternalsDescriptionDict)
+ self.assertTrue('simp_tag' in external)
+ self.assertTrue('simp_branch' in external)
+ self.assertTrue('simp_opt' in external)
+ self.assertTrue('mixed_req' in external)
+
+ def test_cfg_exclude_component_dict(self):
+ """Verify that exclude component checkout works with a dictionary
+ """
+ # create the top level externals file
+ desc = self.setup_dict_config()
+ # Test an excluded repo
+ external = create_externals_description(desc, model_format='dict',
+ exclude=['simp_tag',
+ 'simp_opt'])
+ self.assertIsInstance(external, ExternalsDescriptionDict)
+ self.assertFalse('simp_tag' in external)
+ self.assertTrue('simp_branch' in external)
+ self.assertFalse('simp_opt' in external)
+ self.assertTrue('mixed_req' in external)
+
+ def test_cfg_opt_component_dict(self):
+ """Verify that exclude component checkout works with a dictionary
+ """
+ # create the top level externals file
+ desc = self.setup_dict_config()
+ # Test an excluded repo
+ external = create_externals_description(desc, model_format='dict',
+ components=['simp_tag',
+ 'simp_opt'])
+ self.assertIsInstance(external, ExternalsDescriptionDict)
+ self.assertTrue('simp_tag' in external)
+ self.assertFalse('simp_branch' in external)
+ self.assertTrue('simp_opt' in external)
+ self.assertFalse('mixed_req' in external)
+
  def test_cfg_unknown_version(self):
  """Test that a runtime error is raised when an unknown file version is
  received