NIMS 1.1

apache.conf

@@ -1,28 +1,21 @@
-WSGIDaemonProcess nims processes=4 threads=16 maximum-requests=1000 display-name=%{GROUP}
+WSGIDaemonProcess nims user=nims group=nims processes=2 threads=4 maximum-requests=1000 display-name=%(GROUP)
+
 <Directory /var/local/nims>
  WSGIApplicationGroup %{GLOBAL}
  WSGIProcessGroup nims
+ Require all granted
 </Directory>
 
-Redirect /nimsgears /nims
-WSGIScriptAlias /nims /var/local/nims/nimsgears/public/nims.wsgi
+WSGIScriptAlias /nims /var/local/nims/nimsgears/public/nims.wsgi
 
-Alias /nims/images /var/local/nims/nimsgears/public/images
-Alias /nims/css /var/local/nims/nimsgears/public/css
-Alias /nims/javascript /var/local/nims/nimsgears/public/javascript
-Alias /nims/static /var/local/nims/nimsgears/public/static
+Alias /nims/images /var/local/nims/nimsgears/public/images
+Alias /nims/css /var/local/nims/nimsgears/public/css
+Alias /nims/javascript /var/local/nims/nimsgears/public/javascript
 
 <IfModule mime_module>
  AddType application/octet-stream .7 .bvec .bval .dcm .dat
 </IfModule>
 
-<Directory /run/shm/nimslinks>
- Order allow,deny
- Allow from all
- Options Indexes FollowSymLinks
- AllowOverride AuthConfig
-</Directory>
-
 <Location /nims/auth>
  AuthType WebAuth
  Require valid-user
@@ -31,3 +24,9 @@ Alias /nims/static /var/local/nims/nimsgears/public/static
 <Location /nims/logout_handler>
  WebAuthDoLogout on
 </Location>
+
+<Location /testing/upload>
+ Order deny,allow
+ Deny from all
+ # add allowed hosts
+</Location>

nims.tmux

@@ -0,0 +1,36 @@
+#!/bin/bash
+SESSION="CNI-NIMS"
+
+POSTGRES_USER="nims"
+POSTGRES_PW="nims"
+POSTGRES_HOST="cnifs.stanford.edu"
+POSTGRES_PORT="5432"
+POSTGRES_DB="nims"
+POSTGRES="postgresql:https://${POSTGRES_USER}:${POSTGRES_PW}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
+
+UNSORTABLE_PATH="/scratch/cni/unsortable" # where to place unsortable files
+STAGE_PATH="/scratch/cni/upload" # where uploads are placed, where sorter looks for new files
+NIMS_PATH="/cnifs/nims" # base path where files get sorted
+PHYSIO_PATH="/cnifs/nims/physio" # where physio files are unpacked into
+
+# create
+cd /var/local/nims
+tmux new-session -s "$SESSION" -n bash -d
+
+# sorter
+tmux new-window -t "$SESSION:1" -n "sorter"
+tmux send-keys -t "$SESSION:1" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:1" \
+ "PYTHONPATH=. nimsproc/sorter.py -p ${UNSORTABLE_PATH} ${POSTGRES} ${STAGE_PATH} ${NIMS_PATH}" C-m
+
+# scheduler
+tmux new-window -t "$SESSION:2" -n "scheduler"
+tmux send-keys -t "$SESSION:2" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:2" \
+ "PYTHONPATH=. nimsproc/scheduler.py ${POSTGRES} ${NIMS_PATH}" C-m
+
+# attach to session
+tmux select-window -t "$SESSION:0"
+tmux attach-session -t "$SESSION"

nims.tmux.sample

@@ -0,0 +1,57 @@
+#!/bin/bash
+SESSION="SESSION-NAME"
+
+POSTGRES_USER="db_user"
+POSTGRES_PW="password"
+POSTGRES_HOST="postgres.example.com"
+POSTGRES_PORT="5432"
+POSTGRES_DB="db_name"
+POSTGRES="postgresql:https://${POSTGRES_USER}:${POSTGRES_PW}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
+
+UNSORTABLE_PATH="/scratch/example/unsortable" # where to place unsortable files
+STAGE_PATH="/scratch/example/upload" # where uploads are placed, where sorter looks for new files
+NIMS_PATH="/examplefs/nims" # base path where files get sorted
+PHYSIO_PATH="/examplefs/nims/physio" # where physio files are unpacked into
+
+# create
+cd /var/local/nims
+tmux new-session -s "$SESSION" -n bash -d
+
+# sorter
+tmux new-window -t "$SESSION:1" -n "sorter"
+tmux send-keys -t "$SESSION:1" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:1" \
+ "nimsproc/sorter.py -p ${UNSORTABLE_PATH} ${POSTGRES} ${STAGE_PATH} ${NIMS_PATH}" C-m
+
+# scheduler
+tmux new-window -t "$SESSION:2" -n "scheduler"
+tmux send-keys -t "$SESSION:2" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:2" \
+ "nimsproc/scheduler.py ${POSTGRES} ${NIMS_PATH}" C-m
+
+# processor
+tmux new-window -t "$SESSION:3" -n "processor"
+tmux send-keys -t "$SESSION:3" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:3" \
+ "nimsproc/processor.py -j8 -t /scratch_spinning -e \"~Epoch.psd.contains(u'mux')\" ${POSTGRES} ${NIMS_PATH} ${PHYSIO_PATH}" C-m
+
+# mux processor
+tmux new-window -t "$SESSION:4" -n "mux_proc"
+tmux send-keys -t "$SESSION:4" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:4" \
+ "nimsproc/processor.py -j1 -k32 -t /scratch_spinning -e \"Epoch.psd.contains(u'mux')\" ${POSTGRES} ${NIMS_PATH} ${PHYSIO_PATH}" C-m
+
+# QA
+tmux new-window -t "$SESSION:5" -n "qa"
+tmux send-keys -t "$SESSION:5" \
+ "source /var/local/tg2env/bin/activate" C-m
+tmux send-keys -t "$SESSION:5" \
+ "nimsproc/qa_report.py -j8 ${POSTGRES} ${NIMS_PATH}" C-m
+
+# attach to session
+tmux select-window -t "$SESSION:0"
+tmux attach-session -t "$SESSION"

nimsgears/controllers/root.py

@@ -3,15 +3,17 @@
 
 import os
 import json
+import uuid
+import hashlib
 import datetime
 import tempfile
 import subprocess
 
-from tg import config, expose, flash, lurl, request, redirect, response
+from tg import abort, config, expose, flash, lurl, request, redirect, response
 from tg.i18n import ugettext as _, lazy_ugettext as l_
 import webob.exc
 
-import nimsdata
+import nimsdata.medimg.nimsmontage
 import nimsutil
 from nimsgears.model import *
 
@@ -89,7 +91,7 @@ def pyramid(self, **kwargs):
  ds = Dataset.get(kwargs['dataset_id'])
  if user.has_access_to(ds):
  db_file = os.path.join(store_path, ds.relpath, ds.filenames[0])
- return dict(zip(['dataset_id', 'tile_size', 'x_size', 'y_size'], (ds.id,) + nimsdata.nimsmontage.get_info(db_file)))
+ return dict(zip(['dataset_id', 'tile_size', 'x_size', 'y_size'], (ds.id,) + nimsdata.medimg.nimsmontage.get_info(db_file)))
 
  @expose('nimsgears.templates.qa_report', render_params={'doctype': None})
  def qa_report(self, **kwargs):
@@ -156,7 +158,7 @@ def pyramid_tile(self, *args):
  response.etag = args[0]
  response.cache_control = 'max-age = 86400'
  response.last_modified = ds.updatetime
- return nimsdata.nimsmontage.get_tile(os.path.join(store_path, ds.relpath, ds.filenames[0]), z, x, y)
+ return nimsdata.medimg.nimsmontage.get_tile(os.path.join(store_path, ds.relpath, ds.filenames[0]), z, x, y)
 
  @expose(content_type='application/octet-stream')
  def file(self, **kwargs):
@@ -257,3 +259,20 @@ def download(self, **kwargs):
  tar_proc = subprocess.Popen('tar -chf - -C %s nims; rm -r %s' % (temp_dir, temp_dir), shell=True, stdout=subprocess.PIPE)
  response.content_disposition = 'attachment; filename=%s_%s' % ('nims', datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
  return tar_proc.stdout
+
+ @expose()
+ def upload(self, filename='upload', **kwargs):
+ if 'Content-MD5' not in request.headers:
+ abort(400, 'Request must contain a valid "Content-MD5" header.')
+ stage_path = config.get('upload_path')
+ with nimsutil.TempDir(prefix='.tmp', dir=stage_path) as tempdir_path:
+ hash_ = hashlib.sha1()
+ upload_filepath = os.path.join(tempdir_path, filename)
+ with open(upload_filepath, 'wb') as upload_file:
+ for chunk in iter(lambda: request.body_file.read(2**20), ''):
+ hash_.update(chunk)
+ upload_file.write(chunk)
+ if hash_.hexdigest() != request.headers['Content-MD5']:
+ abort(400, 'Content-MD5 mismatch (or unset).')
+ print 'upload from %s: %s [%s]' % (request.user_agent, os.path.basename(upload_filepath), nimsutil.hrsize(request.content_length))
+ os.rename(upload_filepath, os.path.join(stage_path, str(uuid.uuid1()) + '_' + filename)) # add UUID to prevent clobbering files

nimsgears/controllers/search.py

@@ -10,7 +10,7 @@
 import numpy
 import datetime
 
-import nimsdata
+import nimsdata.medimg.dcm.mr.generic_mr
 from nimsgears.model import *
 from nimsgears.controllers.nims import NimsController
 
@@ -97,7 +97,7 @@ def index(self):
  epoch_columns = [('Group', 'col_sunet'), ('Experiment', 'col_exp'), ('Date & Time', 'col_datetime'),
  ('Exam', 'col_exam'), ('Type Scan', 'col_scantype'), ('Description', 'col_desc')]
  dataset_columns = [('Data Type', 'col_type')]
- scantype_values = [''] + sorted(nimsdata.nimsmrdata.scan_types.all)
+ scantype_values = [''] + sorted(nimsdata.medimg.dcm.mr.generic_mr.scan_types.all)
  psd_names_tuples = DBSession.query(Epoch.psd).distinct(Epoch.psd)
  psd_values = [''] + sorted([elem[0] for elem in psd_names_tuples])
  return dict(page='search',

nimsgears/model/nims.py

@@ -293,7 +293,7 @@ def __unicode__(self):
 class Job(Entity):
 
  timestamp = Field(DateTime, default=datetime.datetime.now)
- status = Field(Enum(u'pending', u'running', u'done', u'failed', u'abandoned', name=u'job_status'))
+ status = Field(Enum(u'pending', u'running', u'done', u'failed', u'abandoned', 'rerun', name=u'job_status'))
  task = Field(Enum(u'find', u'proc', u'find&proc', name=u'job_task'))
  needs_rerun = Field(Boolean, default=False)
  progress = Field(Integer)
@@ -515,7 +515,7 @@ def from_mrfile(cls, mrfile):
  if subj_code:
  subject = query.filter(cls.code==subj_code).first()
  elif mrfile.subj_firstname and mrfile.subj_lastname:
- subject = query.filter(cls.firstname==mrfile.subj_firstname).filter(cls.lastname==mrfile.subj_lastname).filter(cls.dob==mrfile.subj_dob).first()
+ subject = query.filter(cls.firstname==unicode(mrfile.subj_firstname)).filter(cls.lastname==unicode(mrfile.subj_lastname)).filter(cls.dob==mrfile.subj_dob).first()
  else:
  subject = None
  if not subject:
@@ -525,8 +525,8 @@ def from_mrfile(cls, mrfile):
  experiment=experiment,
  person=Person(),
  code=subj_code[:31] or experiment.next_subject_code,
- firstname=mrfile.subj_firstname[:63],
- lastname=mrfile.subj_lastname[:63],
+ firstname=(unicode(mrfile.subj_firstname) or u'')[:63],
+ lastname=(unicode(mrfile.subj_lastname) or u'')[:63],
  dob=mrfile.subj_dob,
  )
  return subject
@@ -605,7 +605,7 @@ def from_mrfile(cls, mrfile):
  # central authority and/or querying the schedule database. But for now,
  # just let the operator be None if the user isn't already in the system.
  operator = User.by_uid(unicode(mrfile.operator), create=False)
- session = Session(uid=uid, exam=mrfile.exam_no, subject=subject, operator=operator)
+ session = Session(uid=uid, exam=int(mrfile.exam_no) if isinstance(mrfile.exam_no, (unicode, int)) else 0, subject=subject, operator=operator)
  return session
 
  @classmethod
@@ -696,6 +696,7 @@ class Epoch(DataContainer):
  phase_encode_undersample = Field(Float)
  slice_encode_undersample = Field(Float)
  acquisition_matrix = Field(Unicode(255))
+ num_mux_cal_cycle = Field(Integer)
 
  session = ManyToOne('Session')
 
@@ -708,13 +709,13 @@ def from_mrfile(cls, mrfile):
  epoch = cls.query.filter_by(uid=uid).filter_by(acq=mrfile.acq_no).first()
  if not epoch:
  session = Session.from_mrfile(mrfile)
- if session.timestamp is None or session.timestamp > mrfile.timestamp:
+ if session.timestamp is None or (mrfile.timestamp is not None and session.timestamp > mrfile.timestamp):
  session.timestamp = mrfile.timestamp
  epoch = cls(
  session = session,
  timestamp = mrfile.timestamp,
- duration = datetime.timedelta(0, mrfile.duration),
- prescribed_duration = datetime.timedelta(0, mrfile.prescribed_duration),
+ duration = datetime.timedelta(0, mrfile.duration or 0),
+ prescribed_duration = datetime.timedelta(0, mrfile.prescribed_duration or 0),
  uid = uid,
  series = mrfile.series_no,
  acq = mrfile.acq_no,
@@ -727,7 +728,7 @@ def from_mrfile(cls, mrfile):
  flip_angle = mrfile.flip_angle,
  pixel_bandwidth = mrfile.pixel_bandwidth,
  num_slices = mrfile.num_slices,
- num_timepoints = mrfile.num_timepoints,
+ num_timepoints = mrfile.num_timepoints or 1,
  num_averages = mrfile.num_averages,
  num_echos = mrfile.num_echos,
  receive_coil_name = unicode(mrfile.receive_coil_name),
@@ -744,6 +745,7 @@ def from_mrfile(cls, mrfile):
  phase_encode_undersample = mrfile.phase_encode_undersample,
  slice_encode_undersample = mrfile.slice_encode_undersample,
  acquisition_matrix = unicode(str(mrfile.acquisition_matrix)),
+ num_mux_cal_cycle = mrfile.num_mux_cal_cycle if mrfile.filetype == u'pfile' else None, # hack for pfile
  qa_status = u'pending',
  # to unpack fov, mm_per_vox, and acquisition_matrix: np.fromstring(str(mm)[1:-1],sep=',')
  )
@@ -758,11 +760,11 @@ def toplevel_query(cls):
 
  @property
  def name(self):
- return '%d_%d_%d' % (self.session.exam, self.series, self.acq)
+ return '%d_%d%s' % (self.session.exam, self.series, '_%d' % self.acq if self.acq is not None else '')
 
  @property
  def dirname(self):
- return '%d_%d_%s' % (self.series, self.acq, self.description)
+ return '%d%s_%s' % (self.series, '_%d' % self.acq if self.acq is not None else '', self.description)
 
  @property
  def contains_trash(self):