Add migration tools.

migration-tools/README.md

@@ -0,0 +1,71 @@
+Migration Tools
+===============
+
+Utility scripts for migrating packages between rosdistros.
+
+## Environment and setup
+
+It's recommended to run this script within a virtualenv.
+Python 3.6 or newer is required.
+
+In addition to python3 you will also need Git installed.
+If you are not already using a credential helper for https git remotes you will need to set one up:
+https://git-scm.com/docs/gitcredentials
+
+Install the script's dependencies listed in requirements.txt:
+
+ python3 -m pip install -r requirements.txt
+
+Make sure rosdep is initialized if it isn't already.
+
+ sudo rosdep init
+
+Configure a GitHub access token and release repository organization.
+The token only needs public repository access and must belong to a member of the target GitHub organization with repository creation permissions.
+
+```
+GITHUB_TOKEN={token with public repository access}
+export GITHUB_TOKEN
+```
+
+## Script arguments
+
+The migration script has several named arguments, all of which are required.
+* `--dest DEST_ROSDISTRO`: The rosdistro which will receive the newly bloomed repositories.
+* `--source SOURCE_ROSDISTRO`: The rosdistro to take release repositories from for migration.
+* `--source-ref GIT_COMMIT_ID_OR_REF`: The migration may be attempted multiple times. Each attempt must specify ref or commit to start from so that future changes to the source distribution do not unintentionally affect migration. This also enables platform migration without changing the rosdistro name.
+* `--release-org GITHUB_ORG`: A GitHub organization for storing release repositories. If the repository does not exist in this organization a new repository will be created.
+
+## Features
+
+The migration tool currently performs the same operation for all intended use-cases and the different actions are accomplished by setting up the script with a different initial environment.
+
+### Creating a new rosdistro from an existing one
+
+This functionality is intended to support creating new stable ROS distributions from a perennial rolling distribution.
+
+#### Prerequisites
+
+* The local rosdistro index must already have an entry for the target distribution.
+* The distribution.yaml file for the target must exist and should have an empty `repositories` field.
+
+
+### Updating a migrated rosdistro 
+
+This functionality is intended to support re-trying failed bloom releases in a ROS distribution migrated previously.
+
+#### Prerequisites
+
+* The local rosdistro index must already have an entry for the target distribution.
+* The distribution.yaml must exist and may have some repositories already but repositories which failed to release must have no release version. Removing the release version for failed releases is the default behavior of this tool.
+
+
+### Updating the platform of an existing rosdistro
+
+This functionality is intended to support updating the supported platforms of the perennial rolling distribution.
+
+#### Prerequisites
+
+* A git commit id or ref name which has the previous platform and desired repositories in the source distribution.
+* The current distribution.yaml should specify the desired target platforms and should have an empty `repositories` field.
+

migration-tools/migrate-rosdistro.py

@@ -0,0 +1,211 @@
+import argparse
+import copy
+import os
+import os.path
+import subprocess
+import sys
+import tempfile
+
+from bloom.commands.git.patch.common import get_patch_config, set_patch_config
+from bloom.git import inbranch, show
+
+import github
+import yaml
+
+from rosdistro import DistributionFile, get_distribution_cache, get_distribution_file, get_index
+from rosdistro.writer import yaml_from_distribution_file
+
+# These functions are adapted from Bloom's internal 'get_tracks_dict_raw' and
+# 'write_tracks_dict_raw' functions. We cannot use them directly since they
+# make assertions about the release repository that are not true during the
+# manipulation of the release repository for this script.
+def read_tracks_file():
+ return yaml.safe_load(show('master', 'tracks.yaml'))
+
+@inbranch('master')
+def write_tracks_file(tracks, commit_msg=None):
+ if commit_msg is None:
+ commit_msg = f'Update tracks.yaml from {sys.argv[0]}.'
+ with open('tracks.yaml', 'w') as f:
+ f.write(yaml.safe_dump(tracks, indent=2, default_flow_style=False))
+ with open('.git/rosdistromigratecommitmsg', 'w') as f:
+ f.write(commit_msg)
+ subprocess.check_call(['git', 'add', 'tracks.yaml'])
+ subprocess.check_call(['git', 'commit', '-F', '.git/rosdistromigratecommitmsg'])
+
+
+parser = argparse.ArgumentParser(
+ description='Import packages from one rosdistro into another one.'
+)
+parser.add_argument('--source', required=True, help='The source rosdistro name')
+parser.add_argument('--source-ref', required=True, help='The git version for the source. Used to retry failed imports without bumping versions.')
+parser.add_argument('--dest', required=True, help='The destination rosdistro name')
+parser.add_argument('--release-org', required=True, help='The organization containing release repositories')
+
+args = parser.parse_args()
+
+gclient = github.Github(os.environ['GITHUB_TOKEN'])
+release_org = gclient.get_organization(args.release_org)
+org_release_repos = [r.name for r in release_org.get_repos() if r.name]
+
+if not os.path.isfile('index-v4.yaml'):
+ raise RuntimeError('This script must be run from a rosdistro index directory.')
+rosdistro_dir = os.path.abspath(os.getcwd())
+rosdistro_index_url = f'file:https://{rosdistro_dir}/index-v4.yaml'
+
+index = get_index(rosdistro_index_url)
+index_yaml = yaml.safe_load(open('index-v4.yaml', 'r'))
+
+if len(index_yaml['distributions'][args.source]['distribution']) != 1 or \
+ len(index_yaml['distributions'][args.dest]['distribution']) != 1:
+ raise RuntimeError('Both source and destination distributions must have a single distribution file.')
+
+# There is a possibility that the source_ref has different a distribution file
+# layout. Check that they match.
+source_ref_index_yaml = yaml.safe_load(show(args.source_ref, 'index-v4.yaml'))
+if source_ref_index_yaml['distributions'][args.source]['distribution'] != \
+ index_yaml['distributions'][args.source]['distribution']:
+ raise RuntimeError('The distribution file layout has changed between the source ref and now.')
+
+source_distribution_filename = index_yaml['distributions'][args.source]['distribution'][0]
+dest_distribution_filename = index_yaml['distributions'][args.dest]['distribution'][0]
+
+# Fetch the source distribution file from the exact point in the repository history requested.
+source_distfile_data = yaml.safe_load(show(args.source_ref, source_distribution_filename))
+source_distribution = DistributionFile(args.source, source_distfile_data)
+
+# Prepare the destination distribution for new bloom releases from the source distribution.
+dest_distribution = get_distribution_file(index, args.dest)
+new_repositories = []
+repositories_to_retry = []
+for repo_name, repo_data in sorted(source_distribution.repositories.items()):
+ if repo_name not in dest_distribution.repositories:
+ new_repositories.append(repo_name)
+ dest_distribution.repositories[repo_name] = copy.deepcopy(repo_data)
+ elif dest_distribution.repositories[repo_name].release_repository.version is None:
+ dest_distribution.repositories[repo_name].release_repository.version = repo_data.release_repository.version
+ repositories_to_retry.append(repo_name)
+ else:
+ # Nothing to do if the release is there.
+ pass
+
+print(f'Found {len(new_repositories)} new repositories to release:', new_repositories)
+print(f'Found {len(repositories_to_retry)} repositories to retry:', repositories_to_retry)
+
+# Copy out an optimistic destination distribution file to bloom everything
+# against. This obviates the need to bloom packages in a topological order or
+# do any special handling for dependency cycles between repositories as are
+# known to occur in the ros2/launch repository. To allow this we must keep
+# track of repositories that fail to bloom and pull their release in a cleanup
+# step.
+with open(dest_distribution_filename, 'w') as f:
+ f.write(yaml_from_distribution_file(dest_distribution))
+
+repositories_bloomed = []
+repositories_with_errors = []
+
+workdir = tempfile.mkdtemp()
+os.chdir(workdir)
+os.environ['ROSDISTRO_INDEX_URL'] = rosdistro_index_url
+
+for repo_name in sorted(new_repositories + repositories_to_retry):
+ if repo_name != 'gazebo_ros_pkgs':
+ continue
+ try:
+ release_spec = dest_distribution.repositories[repo_name].release_repository
+ print('Adding repo:', repo_name)
+ if release_spec.type != 'git':
+ raise ValueError('This script can only handle git repositories.')
+ remote_url = release_spec.url
+ release_repo = remote_url.split('/')[-1][:-4]
+ subprocess.call(['git', 'clone', remote_url])
+ os.chdir(release_repo)
+ tracks = read_tracks_file()
+
+ if release_repo not in org_release_repos:
+ release_org.create_repo(release_repo)
+ new_release_repo_url = f'https://github.com/{args.release_org}/{release_repo}.git'
+ subprocess.check_call(['git', 'remote', 'rename', 'origin', 'oldorigin'])
+ subprocess.check_call(['git', 'remote', 'set-url', '--push', 'oldorigin', 'no_push'])
+ subprocess.check_call(['git', 'remote', 'add', 'origin', new_release_repo_url])
+
+ if args.source != args.dest:
+ # Copy the source track to the new destination.
+ dest_track = copy.deepcopy(tracks['tracks'][args.source])
+ dest_track['ros_distro'] = args.dest
+ tracks['tracks'][args.dest] = dest_track
+ ls_remote = subprocess.check_output(['git', 'ls-remote', '--heads', 'oldorigin', f'*{args.source}*'], universal_newlines=True)
+ for line in ls_remote.split('\n'):
+ if line == '':
+ continue
+ obj, ref = line.split('\t')
+ ref = ref[11:] # strip 'refs/heads/'
+ newref = ref.replace(args.source, args.dest)
+ subprocess.check_call(['git', 'branch', newref, obj])
+ if newref.startswith('patches/'):
+ # Update parent in patch configs. Without this update the
+ # patches will be rebased out when git-bloom-release is
+ # called because the configured parent won't match the
+ # expected source branch.
+ config = get_patch_config(newref)
+ config['parent'] = config['parent'].replace(args.source, args.dest)
+ set_patch_config(newref, config)
+ write_tracks_file(tracks, 'Copy {args.source} track to {args.dest} with clone.py.')
+ else:
+ dest_track = tracks['tracks'][args.dest]
+
+ # Configure next release to re-release previous version into the
+ # destination. A version value of :{ask} will fail due to
+ # interactivity and :{auto} may result in a previously unreleased tag
+ # on the development branch being released for the first time.
+ if dest_track['version'] in [':{ask}', ':{auto}']:
+ # Override the version for this release to guarantee the same version is released.
+ dest_track['version_saved'] = dest_track['version']
+ dest_track['version'] = dest_track['last_version']
+ write_tracks_file(tracks, f'Update {args.dest} track to release exactly last-released version.')
+
+ if dest_track['release_tag'] == ':{ask}' and 'last_release' in dest_track:
+ # Override the version for this release to guarantee the same version is released.
+ dest_track['release_tag_saved'] = dest_track['release_tag']
+ dest_track['release_tag'] = dest_track['last_release']
+ write_tracks_file(tracks, f'Update {args.dest} track to release exactly last-released tag.')
+
+ # Bloom will not run with multiple remotes.
+ subprocess.check_call(['git', 'remote', 'remove', 'oldorigin'])
+ subprocess.check_call(['git', 'bloom-release', '--non-interactive', '--unsafe', args.dest], env=os.environ)
+ subprocess.check_call(['git', 'push', 'origin', '--all', '--force'])
+ subprocess.check_call(['git', 'push', 'origin', '--tags', '--force'])
+ subprocess.check_call(['git', 'checkout', 'master'])
+
+ # Re-read tracks.yaml after release.
+ tracks = read_tracks_file()
+ dest_track = tracks['tracks'][args.dest]
+ if 'version_saved' in dest_track:
+ dest_track['version'] = dest_track['version_saved']
+ del dest_track['version_saved']
+ write_tracks_file(tracks, f'Restore saved version for {args.dest} track.')
+ if 'release_tag_saved' in dest_track:
+ dest_track['release_tag'] = dest_track['release_tag_saved']
+ del dest_track['release_tag_saved']
+ write_tracks_file(tracks, f'Restore saved version and tag for {args.dest} track.')
+ new_release_track_inc = str(int(tracks['tracks'][args.dest]['release_inc']))
+ release_spec.url = new_release_repo_url
+
+ ver, _inc = release_spec.version.split('-')
+ release_spec.version = '-'.join([ver, new_release_track_inc])
+ repositories_bloomed.append(repo_name)
+ except (subprocess.CalledProcessError, ValueError) as e:
+ repositories_with_errors.append((repo_name, e))
+ os.chdir(workdir)
+
+os.chdir(rosdistro_dir)
+
+for dest_repo in sorted(new_repositories + repositories_to_retry):
+ if dest_repo not in repositories_bloomed:
+ print(f'{dest_repo} was not bloomed! Removing the release version,')
+ dest_distribution.repositories[dest_repo].release_repository.version = None
+
+with open(dest_distribution_filename, 'w') as f:
+ f.write(yaml_from_distribution_file(dest_distribution))
+
+print(f'Had {len(repositories_with_errors)} repositories with errors:', repositories_with_errors)

migration-tools/requirements.txt

@@ -0,0 +1,5 @@
+bloom>=0.9.4
+pygithub>=1.45
+rosdistro>=0.8.0
+rosdep>=0.18.0
+pyyaml>=5.3