[ENH] Add Datalad cli interfaces

.circleci/config.yml

@@ -97,7 +97,7 @@ jobs:
  - run:
  name: Download Singularity
  command: |
- if [ ! -f /usr/local/bin/singularity ]
+ if [ ! -f /home/circleci/singularity.tar.gz ]
  then
  sudo apt-get update && sudo apt-get -y install build-essential \
  libssl-dev \
@@ -146,6 +146,9 @@ jobs:
  - restore_cache:
  keys:
  - singularity-v3-{{ .Branch }}-{{ .Revision }}
+ - singularity-v3-{{ .Branch }}-
+ - singularity-v3-master-
+ - singularity-v3-
  - run:
  name: Load Docker image layer cache
  no_output_timeout: 30m
@@ -184,19 +187,32 @@ jobs:
  docker-daemon:https://pennlinc/bond:latest
 
  - run:
- name: Test singularity
+ name: Test bond-group
  command: |
  git config --global user.email "[email protected]"
  git config --global user.name "CircleCI Test"
  export PATH=/tmp/miniconda/bin:$PATH
  source activate bond
  mkdir -p /tmp/bids /tmp/group_testing
- cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/singularity
+ cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/group
  bond-group \
- /tmp/bids/singularity \
- /tmp/group_testing/direct \
+ /tmp/bids/group \
+ /tmp/group_testing/group \
  --container /home/circleci/bond-latest.sif
 
+ - run:
+ name: Test bond-save
+ command: |
+ git config --global user.email "[email protected]"
+ git config --global user.name "CircleCI Test"
+ export PATH=/tmp/miniconda/bin:$PATH
+ source activate bond
+ mkdir -p /tmp/bids
+ cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/singularity
+ bond-datalad-save \
+ /tmp/bids/singularity \
+ -m 'test save' \
+ --container /home/circleci/bond-latest.sif
 
 
  install_and_test:
@@ -231,22 +247,37 @@ jobs:
  command: |
  export PATH=/tmp/miniconda/bin:$PATH
  source activate bond
- pip install .
+ pip install .[all]
+ py.test -sv tests
 
  - run:
- name: Test Docker integration
+ name: Test Docker group
  command: |
  git config --global user.email "[email protected]"
  git config --global user.name "CircleCI Test"
  export PATH=/tmp/miniconda/bin:$PATH
  source activate bond
  mkdir -p /tmp/bids /tmp/group_testing
- cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/docker
+ cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/docker-group
  bond-group \
- /tmp/bids/docker \
+ /tmp/bids/docker-group \
  /tmp/group_testing/docker \
  --container pennlinc/bond:latest
 
+ - run:
+ name: Test Docker save
+ command: |
+ git config --global user.email "[email protected]"
+ git config --global user.name "CircleCI Test"
+ export PATH=/tmp/miniconda/bin:$PATH
+ source activate bond
+ mkdir -p /tmp/bids
+ cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/docker-save
+ bond-datalad-save \
+ /tmp/bids/docker-save \
+ -m 'test docker save!' \
+ --container pennlinc/bond:latest
+
  build_docs:
  docker:
  - image: circleci/python:3.7.4

bond/bond.py

@@ -1,5 +1,6 @@
 """Main module."""
 from collections import defaultdict
+import subprocess
 import bids
 import json
 from pathlib import Path
@@ -41,7 +42,7 @@ def __init__(self, data_root, use_datalad=False):
  if use_datalad:
  self.init_datalad()
 
- def init_datalad(self, save=False, message=None):
+ def init_datalad(self):
  """Initializes a datalad Dataset at self.path.
 
  Parameters:
@@ -57,15 +58,22 @@ def init_datalad(self, save=False, message=None):
  cfg_proc='text2git',
  force=True,
  annex=True)
- if save:
- self.datalad_handle.save(message="Saved by BOnD")
- if not save and not self.is_datalad_clean():
- raise Exception("Unsaved changes in %s" % self.path)
 
  def datalad_save(self, message=None):
- if message is None:
- message = "BOnD Save"
- statuses = self.datalad_handle.save(message=message)
+ """Performs a DataLad Save operation on the BIDS tree.
+
+ Additionally a check for an active datalad handle and that the
+ status of all objects after the save is "ok".
+
+ Parameters:
+ -----------
+ message : str or None
+ Commit message to use with datalad save
+ """
+ if not self.datalad_ready:
+ raise Exception(
+ "DataLad has not been initialized. use datalad_init()")
+ statuses = self.datalad_handle.save(message=message or "BOnD Save")
  saved_status = set([status['status'] for status in statuses])
  if not saved_status == set(["ok"]):
  raise Exception("Failed to save in DataLad")
@@ -79,6 +87,18 @@ def is_datalad_clean(self):
  self.datalad_handle.status()])
  return statuses == set(["clean"])
 
+ def datalad_undo_last_commit(self):
+ """Revert the most recent commit, remove it from history.
+
+ uses git reset --hard
+ """
+ if not self.is_datalad_clean():
+ raise Exception("Untracked changes present. "
+ "Run clear_untracked_changes first")
+ reset_proc = subprocess.run(
+ ["git", "reset", "--hard", "HEAD~1"], cwd=self.path)
+ reset_proc.check_returncode()
+
  def merge_params(self, merge_df, files_df):
  key_param_merge = {}
  for i in range(len(merge_df)):

bond/cli.py

@@ -109,12 +109,87 @@ def bond_apply():
  pass
 
 
-def bond_undo():
+def param_group_merge():
  pass
 
 
-def param_group_merge():
- pass
+def bond_datalad_save():
+ parser = argparse.ArgumentParser(
+ description="bond-datalad-save: perform a DataLad save on a BIDS "
+ "directory",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('bids_dir',
+ type=Path,
+ action='store',
+ help='the root of a BIDS dataset. It should contain '
+ 'sub-X directories and dataset_description.json')
+ parser.add_argument('-m',
+ action='store',
+ help='message for this commit')
+ parser.add_argument('--container',
+ action='store',
+ help='Docker image tag or Singularity image file.')
+ opts = parser.parse_args()
+
+ # Run directly from python using
+ if opts.container is None:
+ bod = BOnD(data_root=str(opts.bids_dir), use_datalad=True)
+ bod.datalad_save(message=opts.m)
+ sys.exit(0)
+
+ # Run it through a container
+ container_type = _get_container_type(opts.container)
+ bids_dir_link = str(opts.bids_dir.absolute()) + ":/bids"
+ if container_type == 'docker':
+ cmd = ['docker', 'run', '--rm', '-v', bids_dir_link,
+ '-v', GIT_CONFIG+":/root/.gitconfig",
+ '--entrypoint', 'bond-datalad-save',
+ opts.container, '/bids', '-m', opts.m]
+ elif container_type == 'singularity':
+ cmd = ['singularity', 'exec', '--cleanenv',
+ '-B', bids_dir_link,
+ opts.container, 'bond-datalad-save',
+ '/bids', '-m', opts.m]
+ print("RUNNING: " + ' '.join(cmd))
+ proc = subprocess.run(cmd)
+ sys.exit(proc.returncode)
+
+
+def bond_undo():
+ parser = argparse.ArgumentParser(
+ description="bond-undo: revert most recent commit",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('bids_dir',
+ type=Path,
+ action='store',
+ help='the root of a BIDS dataset. It should contain '
+ 'sub-X directories and dataset_description.json')
+ parser.add_argument('--container',
+ action='store',
+ help='Docker image tag or Singularity image file.')
+ opts = parser.parse_args()
+
+ # Run directly from python using
+ if opts.container is None:
+ bod = BOnD(data_root=str(opts.bids_dir), use_datalad=True)
+ bod.datalad_undo_last_commit()
+ sys.exit(0)
+
+ # Run it through a container
+ container_type = _get_container_type(opts.container)
+ bids_dir_link = str(opts.bids_dir.absolute()) + ":/bids"
+ if container_type == 'docker':
+ cmd = ['docker', 'run', '--rm', '-v', bids_dir_link,
+ '-v', GIT_CONFIG+":/root/.gitconfig",
+ '--entrypoint', 'bond-undo',
+ opts.container, '/bids']
+ elif container_type == 'singularity':
+ cmd = ['singularity', 'exec', '--cleanenv',
+ '-B', bids_dir_link,
+ opts.container, 'bond-undo', '/bids']
+ print("RUNNING: " + ' '.join(cmd))
+ proc = subprocess.run(cmd)
+ sys.exit(proc.returncode)
 
 
 def _get_container_type(image_name):

setup.cfg

@@ -34,6 +34,7 @@ install_requires =
  datalad>=0.13.5
  wrapt<2,>=1.10
 test_requires =
+ nibabel
  pytest==4.6.5
  pytest-runner==5.1
  pip==19.2.3

setup.py

@@ -30,9 +30,10 @@
  'console_scripts': [
  'bond-group=bond.cli:bond_group',
  'bond-apply=bond.cli:bond_apply',
- 'bond-revert=bond.cli:bond_revert',
+ 'bond-undo=bond.cli:bond_undo',
  'bids-sidecar-merge=bond.cli:param_group_merge',
- 'bond-validate=bond.cli:bond_validate'
+ 'bond-validate=bond.cli:bond_validate',
+ 'bond-datalad-save=bond.cli:bond_datalad_save'
  ],
  },
  license="GNU General Public License v3",

tests/test_bond.py

@@ -3,11 +3,14 @@
 """Tests for `bond` package."""
 import sys
 import shutil
+import hashlib
 import json
 from pkg_resources import resource_filename as pkgrf
 sys.path.append("..")
 import pytest
 from bond import BOnD
+import nibabel as nb
+import numpy as np
 
 TEST_DATA = pkgrf("bond", "testdata")
 
@@ -115,6 +118,26 @@ def _edit_a_json(json_file):
  json.dump(metadata, metadataw)
 
 
+def _edit_a_nifti(nifti_file):
+ img = nb.load(nifti_file)
+ new_img = nb.Nifti1Image(np.random.rand(*img.shape),
+ affine=img.affine,
+ header=img.header)
+ new_img.to_filename(nifti_file)
+
+
+def file_hash(file_name):
+ with open(str(file_name), 'rb') as fcheck:
+ data = fcheck.read()
+ return hashlib.md5(data).hexdigest()
+
+
+def _get_json_string(json_path):
+ with json_path.open("r") as f:
+ content = "".join(f.readlines())
+ return content
+
+
 def test_datalad_integration(tmp_path):
  """Test that datalad works for basic file modification operations.
  """
@@ -129,7 +152,8 @@ def test_datalad_integration(tmp_path):
  uninit_bond.is_datalad_clean()
 
  # initialize the datalad repository and try again
- uninit_bond.init_datalad(save=True)
+ uninit_bond.init_datalad()
+ uninit_bond.datalad_save('Test save')
  assert uninit_bond.is_datalad_clean()
 
  # Now, the datalad repository is initialized and saved.
@@ -140,19 +164,62 @@ def test_datalad_integration(tmp_path):
  assert complete_bod.datalad_ready
  assert complete_bod.is_datalad_clean()
 
- # Edit a file and make sure that it's been detected by datalad
- _edit_a_json(str(data_root / "complete" / "sub-03" / "ses-phdiff" / "func"
- / "sub-03_ses-phdiff_task-rest_bold.json"))
+ # Test clean and revert functionality
+ test_file = data_root / "complete" / "sub-03" / "ses-phdiff" \
+ / "func" / "sub-03_ses-phdiff_task-rest_bold.json"
+ test_binary = data_root / "complete" / "sub-03" / "ses-phdiff" \
+ / "func" / "sub-03_ses-phdiff_task-rest_bold.nii.gz"
+
+ # Try editing a locked file - it should fail
+ with pytest.raises(Exception):
+ _edit_a_nifti(test_binary)
+
+ # Unlock the files so we can access their content
+ complete_bod.datalad_handle.unlock(test_binary)
+ complete_bod.datalad_handle.unlock(test_file)
+
+ # Get the contents of the original files
+ original_content = _get_json_string(test_file)
+ original_binary_content = file_hash(test_binary)
+
+ # Edit the files
+ _edit_a_nifti(test_binary)
+ _edit_a_json(test_file)
+
+ # Get the edited content
+ edited_content = _get_json_string(test_file)
+ edited_binary_content = file_hash(test_binary)
+
+ # Check that the file content has changed
+ assert not original_content == edited_content
+ assert not original_binary_content == edited_binary_content
+
+ # Check that datalad knows something has changed
  assert not uninit_bond.is_datalad_clean()
  assert not complete_bod.is_datalad_clean()
 
- # Make sure you can't initialize a BOnD object on a dirty directory
+ # Attempt to undo a change before checking in changes
  with pytest.raises(Exception):
- BOnD(data_root / "complete", use_datalad=True)
+ uninit_bond.datalad_undo_last_commit()
 
- # Test BOnD.datalad_save()
+ # Perform a save
  uninit_bond.datalad_save(message="TEST SAVE!")
+ assert uninit_bond.is_datalad_clean()
+
+ # Now undo the most recent save
+ uninit_bond.datalad_undo_last_commit()
+
+ # Unlock the restored files so we can access their content
+ complete_bod.datalad_handle.unlock(test_binary)
+ complete_bod.datalad_handle.unlock(test_file)
+
+ # Get the contents of the original files
+ restored_content = _get_json_string(test_file)
+ restored_binary_content = file_hash(test_binary)
 
+ # Check that the file content has returned to its original state
+ assert original_content == restore