svndbadmin

#!/usr/bin/env python
# -*-python-*-
#
# Copyright (C) 2004-2020 The ViewCVS Group. All Rights Reserved.
# Copyright (C) 2004-2007 James Henstridge
#
# By using this file, you agree to the terms and conditions set forth in
# the LICENSE.html file which can be found at the top level of the ViewVC
# distribution or at http:https://viewvc.org/license-1.html.
#
# For more information, visit http:https://viewvc.org/
#
# -----------------------------------------------------------------------
#
# administrative program for loading Subversion revision information
# into the checkin database.  It can be used to add a single revision
# to the database, or rebuild/update all revisions.
#
# To add all the checkins from a Subversion repository to the checkin
# database, run the following:
#    /path/to/svndbadmin rebuild /path/to/repo
#
# This script can also be called from the Subversion post-commit hook,
# something like this:
#    REPOS="$1"
#    REV="$2"
#    /path/to/svndbadmin update "$REPOS" "$REV"
#
# If you allow changes to revision properties in your repository, you
# might also want to set up something similar in the
# post-revprop-change hook using "update" with the --force option to
# keep the checkin database consistent with the repository.
#
# -----------------------------------------------------------------------
#
from __future__ import print_function

#########################################################################
#
# INSTALL-TIME CONFIGURATION
#
# These values will be set during the installation process. During
# development, they will remain None.
#

LIBRARY_DIR = None
CONF_PATHNAME = None

# Adjust sys.path to include our library directory
import sys
import os

if LIBRARY_DIR:
    sys.path.insert(0, LIBRARY_DIR)
else:
    sys.path.insert(0, os.path.abspath(os.path.join(sys.argv[0], "../../lib")))

#########################################################################

import re
import tempfile

import vclib.altsvn._svn as _svn
import vclib.altsvn._svn_repos as _repos
from vclib import _diff_fp

import cvsdb
import viewvc

# as vclib.altsvn._svn* modules and svn swig python binding modules are
# exclusive we must delay import them until loading config.
# here is a flag and placeholders
svn_module = None
_core    = None
_repos   = None
_fs      = None
_delta   = None
_v       = None
_v_repos = None


def import_svn_module(cfg):
    global svn_module
    global _v, _v_repos
    global _core, _repos, _fs, _delta
    if svn_module:
        return
    if getattr(cfg, 'vclib') and getattr(cfg.vclib, 'use_altsvn', None):
        try:
            import vclib.altsvn._svn as _v
            import vclib.altsvn._svn_repos as _v_repos
            svn_module = 'altsvn'
        except ImportError:
            import svn.core as _core
            import svn.repos as _repos
            import svn.fs as _fs
            import svn.delta as _delta
            svn_module = 'svn'
    else:
        import svn.core as _core
        import svn.repos as _repos
        import svn.fs as _fs
        import svn.delta as _delta
        svn_module = 'svn'
    # expose constants
    global SVN_PROP_REVISION_AUTHOR, SVN_PROP_REVISION_DATE, \
           SVN_PROP_REVISION_LOG, svn_node_file, CHANGE_ACTION_DELETE
    if svn_module == 'svn':
        SVN_PROP_REVISION_AUTHOR = _core.SVN_PROP_REVISION_AUTHOR
        SVN_PROP_REVISION_DATE   = _core.SVN_PROP_REVISION_DATE
        SVN_PROP_REVISION_LOG    = _core.SVN_PROP_REVISION_LOG
        svn_node_file            = _core.svn_node_file
        CHANGE_ACTION_DELETE     = _repos.CHANGE_ACTION_DELETE
    else:
        SVN_PROP_REVISION_AUTHOR = _v.SVN_PROP_REVISION_AUTHOR
        SVN_PROP_REVISION_DATE   = _v.SVN_PROP_REVISION_DATE
        SVN_PROP_REVISION_LOG    = _v.SVN_PROP_REVISION_LOG
        svn_node_file            = _v.svn_node_file
        CHANGE_ACTION_DELETE     = _v_repos.svn_fs_path_change_delete

class SvnRepo:
    """Class used to manage a connection to a SVN repository."""
    def __init__(self, path):
        self.path = path
        if svn_module == "svn":
            self.repo       = _repos.svn_repos_open(path)
            self.fs         = _repos.svn_repos_fs(self.repo)
            self.rev_max    = _fs.youngest_rev(self.fs)
        else:
            self.result_pool    = _v.Apr_Pool()
            self.scratch_pool   = _v.Apr_Pool()
            self.repo           = _v_repos.svn_repos_open(
                                        path,
                                        self.result_pool,
                                        self.scratch_pool)
            self.fs             = _v_repos.svn_repos_fs(self.repo)
            self.rev_max        = _v_repos.svn_fs_youngest_rev(
                                        self.fs, self.scratch_pool)
            self.scratch_pool.clear()
    def __getitem__(self, rev):
        if rev is None:
            rev = self.rev_max
        elif rev < 0:
            rev = rev + self.rev_max + 1
        assert 0 <= rev <= self.rev_max
        rev = SvnRev(self, rev)
        return rev

# alternative of svn.fs.FileDiff class. used if svn_module is 'altsvn'
class Diff_fp(_diff_fp):
    def __init__(self, root1, path1, root2, path2, pool=None, diffoptions=[]):
        def get_contents(root, path, scratch_pool):
            fd, temp = tempfile.mkstemp()
            if root and path:
                fp = os.fdopen(fd, "wb")
                try:
                    stream = _v_repos.svn_fs_file_contents(
                                            root, path, scratch_pool)
                    try:
                        while 1:
                            chunk, len = _v.svn_stream_read_full(
                                            stream, _v.SVN_STREAM_CHUNK_SIZE)
                            if not chunk:
                                break
                            fp.write(chunk)
                    finally:
                        _v.svn_stream_close(stream)
                finally:
                    fp.close()
            else:
                os.close(fd)
            return temp

        assert path1 or path2
        temp1 = get_contents(root1, path1, pool)
        temp2 = get_contents(root2, path2, pool)
        _diff_fp.__init__(self, temp1, temp2, info1=None, info2=None,
                        diff_cmd='diff', diff_opts=diffoptions)


_re_diff_change_command = re.compile('(\d+)(?:,(\d+))?([acd])(\d+)(?:,(\d+))?')

def _get_diff_counts(diff_fp):
    """Calculate the plus/minus counts by parsing the output of a
    normal diff.  The reasons for choosing Normal diff format are:
      - the output is short, so should be quicker to parse.
      - only the change commands need be parsed to calculate the counts.
      - All file data is prefixed, so won't be mistaken for a change
        command.
    This code is based on the description of the format found in the
    GNU diff manual."""

    plus, minus = 0, 0
    line = diff_fp.readline()
    while line:
        match = re.match(_re_diff_change_command, line)
        if match:
            # size of first range
            if match.group(2):
                count1 = int(match.group(2)) - int(match.group(1)) + 1
            else:
                count1 = 1
            cmd = match.group(3)
            # size of second range
            if match.group(5):
                count2 = int(match.group(5)) - int(match.group(4)) + 1
            else:
                count2 = 1

            if cmd == 'a':
                # LaR - insert after line L of file1 range R of file2
                plus = plus + count2
            elif cmd == 'c':
                # FcT - replace range F of file1 with range T of file2
                minus = minus + count1
                plus = plus + count2
            elif cmd == 'd':
                # RdL - remove range R of file1, which would have been
                #       at line L of file2
                minus = minus + count1
        line = diff_fp.readline()
    return plus, minus


class SvnRev:
    """Class used to hold information about a particular revision of
    the repository."""
    def __init__(self, repo, rev):
        self.repo = repo
        self.rev = rev
        if svn_module == "svn":
            self.rev_roots = {} # cache of revision roots
        else:
            self.scratch_pool = repo.scratch_pool

        # revision properties ...
        if svn_module == "svn":
            revprops = _fs.svn_fs_revision_proplist(repo.fs, rev)
        else:
            revprops = _v_repos.svn_fs_revision_proplist(repo.fs, rev,
                                                         self.scratch_pool)
        self.author = str(revprops.get(SVN_PROP_REVISION_AUTHOR,''))
        self.date = str(revprops.get(SVN_PROP_REVISION_DATE, ''))
        self.log = str(revprops.get(SVN_PROP_REVISION_LOG, ''))

        # convert the date string to seconds since epoch ...
        if svn_module == "svn":
            try:
                self.date = _core.svn_time_from_cstring(self.date) // 1000000
            except:
                self.date = None
        else:
            self.date = _v.datestr_to_date(self.date, self.scratch_pool)

        # get a root for the current revisions
        fsroot = self._get_root_for_rev(rev)

        # find changes in the revision
        if svn_module == "svn":
            editor = _repos.ChangeCollector(repo.fs, fsroot)
            e_ptr, e_baton = _delta.make_editor(editor)
            _repos.svn_repos_replay(fsroot, e_ptr, e_baton)
            changes = editor.changes
        else:
            changes = _v_repos._get_changed_paths_helper(repo.fs, fsroot,
                                                         self.scratch_pool)

        self.changes = []
        for path, change in changes.items():
            # skip non-file changes
            if change.item_kind != svn_node_file:
                continue

            # deal with the change types we handle
            action = None
            base_root = None
            base_path = change.base_path
            if change.base_path:
                base_root = self._get_root_for_rev(change.base_rev)

            # figure out what kind of change this is, and get a diff
            # object for it.  note that prior to 1.4 Subversion's
            # bindings didn't give us change.action, but that's okay
            # because back then deleted paths always had a change.path
            # of None.
            if hasattr(change, 'action') \
               and change.action == CHANGE_ACTION_DELETE:
                action = 'remove'
            elif not change.path:
                action = 'remove'
            elif change.added:
                action = 'add'
            else:
                action = 'change'

            if svn_module == "svn":
                if action == 'remove':
                    diffobj = _fs.FileDiff(base_root, base_path, None, None)
                else:
                    diffobj = _fs.FileDiff(base_root, base_path,
                                           fsroot, change.path)
                diff_fp = diffobj.get_pipe()
            else:
                if action == 'remove':
                    diff_fp = Diff_fp(base_root, base_path, None, None,
                                      self.scratch_pool)
                else:
                    diff_fp = Diff_fp(base_root, base_path,
                                      fsroot, change.path, self.scratch_pool)

            plus, minus = _get_diff_counts(diff_fp)
            self.changes.append((path, action, plus, minus))

    def _get_root_for_rev(self, rev):
        """Fetch a revision root from a cache of such, or a fresh root
        (which is then cached for later use."""
        if svn_module == "svn":
            if not self.rev_roots.has_key(rev):
                self.rev_roots[rev] = _fs.revision_root(self.repo.fs, rev)
            return self.rev_roots[rev]
        else:
            return self.repo.fs._getroot(rev)


def handle_revision(db, command, repo, rev, verbose, force=0):
    """Adds a particular revision of the repository to the checkin database."""
    revision = repo[rev]
    committed = 0

    if verbose:
        print("Building commit info for revision {0!s}...".format(rev),
              end='')

    if not revision.changes:
        if verbose: print("skipped (no changes).")
        return

    for (path, action, plus, minus) in revision.changes:
        directory, file = os.path.split(path)
        commit = cvsdb.CreateCommit()
        commit.SetRepository(repo.path)
        commit.SetDirectory(directory)
        commit.SetFile(file)
        commit.SetRevision(str(rev))
        commit.SetAuthor(revision.author)
        commit.SetDescription(revision.log)
        commit.SetTime(revision.date)
        commit.SetPlusCount(plus)
        commit.SetMinusCount(minus)
        commit.SetBranch(None)

        if action == 'add':
            commit.SetTypeAdd()
        elif action == 'remove':
            commit.SetTypeRemove()
        elif action == 'change':
            commit.SetTypeChange()

        if command == 'update':
            result = db.CheckCommit(commit)
            if result and not force:
                continue # already recorded

        # commit to database
        db.AddCommit(commit)
        committed = 1

    if verbose:
        if committed:
            print("done.")
        else:
            print("skipped (already recorded).")

def main(command, repository, revs=[], verbose=0, force=0):
    cfg = viewvc.load_config(CONF_PATHNAME)
    import_svn_module(cfg)
    db = cvsdb.ConnectDatabase(cfg)

    # sanitise repository path
    if svn_module == 'svn':
        from vclib.svn import canonicalize_rootpath
        repository = canonicalize_rootpath(repository)
    else:
        repository = _v.canonicalize_rootpath(repository)
    repository = cvsdb.CleanRepository(os.path.abspath(repository))

    # Purge what must be purged.
    if command in ('rebuild', 'purge'):
        if verbose:
            print("Purging commit info "
                  "for repository root `{0!s}'".format(repository))
        try:
            db.PurgeRepository(repository)
        except cvsdb.UnknownRepositoryError as e:
            if command == 'purge':
                sys.stderr.write("ERROR: " + str(e) + "\n")
                sys.exit(1)

    # Record what must be recorded.
    if command in ('rebuild', 'update'):
        if not os.path.exists(repository):
            sys.stderr.write('ERROR: could not find repository %s\n'
                             % (repository))
            sys.exit(1)
        repo = SvnRepo(repository)
        if command == 'rebuild' or (command == 'update' and not revs):
            for rev in range(repo.rev_max+1):
                handle_revision(db, command, repo, rev, verbose)
        elif command == 'update':
            if revs[0] is None:
                revs[0] = repo.rev_max
            if revs[1] is None:
                revs[1] = repo.rev_max
            revs.sort()
            for rev in range(revs[0], revs[1]+1):
                handle_revision(db, command, repo, rev, verbose, force)

def _rev2int(r):
    if r == 'HEAD':
        r = None
    else:
        r = int(r)
        if r < 0:
            raise ValueError("invalid revision '%d'" % (r))
    return r

def usage():
    cmd = os.path.basename(sys.argv[0])
    sys.stderr.write(
"""Administer the ViewVC checkins database data for the Subversion repository
located at REPOS-PATH.

Usage: 1. %s [-v] rebuild REPOS-PATH
       2. %s [-v] update REPOS-PATH [REV[:REV2]] [--force]
       3. %s [-v] purge REPOS-PATH

1.  Rebuild the commit database information for the repository located
    at REPOS-PATH across all revisions, after first purging
    information specific to that repository (if any).

2.  Update the commit database information for the repository located
    at REPOS-PATH across all revisions or, optionally, only for the
    specified revision REV (or revision range REV:REV2).  This is just
    like rebuilding, except that, unless --force is specified, no
    commit information will be stored for commits already present in
    the database.  If a range is specified, the revisions will be
    processed in ascending order, and you may specify "HEAD" to
    indicate "the youngest revision currently in the repository".

3.  Purge information specific to the repository located at REPOS-PATH
    from the database.

Use the -v flag to cause this script to give progress information as it works.

""" % (cmd, cmd, cmd))
    sys.exit(1)

if __name__ == '__main__':
    verbose = 0
    force = 0
    args = sys.argv
    try:
        index = args.index('-v')
        verbose = 1
        del args[index]
    except ValueError:
        pass
    try:
        index = args.index('--force')
        force = 1
        del args[index]
    except ValueError:
        pass

    if len(args) < 3:
        usage()

    command = args[1].lower()
    if command not in ('rebuild', 'update', 'purge'):
        sys.stderr.write('ERROR: unknown command %s\n' % command)
        usage()

    revs = []
    if len(sys.argv) > 3:
        if command == 'rebuild':
            sys.stderr.write('ERROR: rebuild no longer accepts a revision '
                             'number argument.  Usage update --force.')
            usage()
        elif command != 'update':
            usage()
        try:
            revs = map(lambda x: _rev2int(x), sys.argv[3].split(':'))
            if len(revs) > 2:
                raise ValueError("too many revisions in range")
            if len(revs) == 1:
                revs.append(revs[0])
        except ValueError:
            sys.stderr.write('ERROR: invalid revision specification "%s"\n' \
                             % sys.argv[3])
            usage()
    else:
        rev = None

    try:
        # we don't clean args[2] as repository path here, because
        # subversion access modules are not imported yet.
        main(command, args[2], revs, verbose, force)
    except KeyboardInterrupt:
        print()
        print('** break **')
    sys.exit(0)