Torque launcher works!

example_scripts/TCGA.md

@@ -9,7 +9,7 @@ Run commands from: https://github.com/Christensen-Lab-Dartmouth/PyMethylProcess/
 **Embedding using VAE**
 Run 200 job hyperparameter scan for learning embeddings on torque (remove -t option to run local, same for prediction jobs below): 
 ```
-methylnet-embed launch_hyperparameter_scan -sc disease -t -mc 0.84 -b 1. -g -j 200
+methylnet-embed launch_hyperparameter_scan -sc disease -t -mc 0.84 -b 1. -g -j 200 -a "module load python/3-Anaconda && source activate methylnet_pro2"
 ```
 FINISH BELOW
 Rerun top performing run to get final embeddings:

methylnet/embedding.py

@@ -14,7 +14,7 @@ def embed():
  pass
 
 def embed_vae(train_pkl,output_dir,cuda,n_latent,lr,weight_decay,n_epochs,hidden_layer_encoder_topology, kl_warm_up=0, beta=1., scheduler='null', decay=0.5, t_max=10, eta_min=1e-6, t_mult=2, bce_loss=False, batch_size=50, val_pkl='val_methyl_array.pkl', n_workers=9, convolutional = False, height_kernel_sizes=[], width_kernel_sizes=[], add_validation_set=False, loss_reduction='sum', stratify_column='disease'):
- from methylnet.models import AutoEncoder, TybaltTitusVAE, CVAE
+ from methylnet.models import AutoEncoder, TybaltTitusVAE
  from methylnet.datasets import get_methylation_dataset
  import torch
  from torch.utils.data import DataLoader
@@ -162,13 +162,14 @@ def perform_embedding(train_pkl,output_dir,cuda,n_latent,learning_rate,weight_de
 @click.option('-n', '--n_jobs_relaunch', default=0, help='Relaunch n top jobs from previous run.', show_default=True)
 @click.option('-c', '--crossover_p', default=0., help='Rate of crossover between hyperparameters.', show_default=True)
 @click.option('-v', '--val_loss_column', default='min_val_loss-batchsize_adj', help='Validation loss column.', type=click.Path(exists=False))
-def launch_hyperparameter_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, model_complexity_factor, set_beta,n_jobs, n_jobs_relaunch, crossover_p, val_loss_column):
+@click.option('-a', '--additional_command', default='', help='Additional command to input for torque run.', type=click.Path(exists=False))
+def launch_hyperparameter_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, model_complexity_factor, set_beta,n_jobs, n_jobs_relaunch, crossover_p, val_loss_column, additional_command):
  """Launch randomized grid search of neural network hyperparameters."""
  from methylnet.hyperparameter_scans import coarse_scan, find_top_jobs
  custom_jobs=[]
  if n_jobs_relaunch:
  custom_jobs=find_top_jobs(hyperparameter_input_csv, hyperparameter_output_log,n_jobs_relaunch, crossover_p, val_loss_column)
- coarse_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, mlp=False, model_complexity_factor=model_complexity_factor, set_beta=set_beta,n_jobs=n_jobs, custom_jobs=custom_jobs)
+ coarse_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, mlp=False, model_complexity_factor=model_complexity_factor, set_beta=set_beta,n_jobs=n_jobs, custom_jobs=custom_jobs, additional_command=additional_command)
 
 #################
 

methylnet/hyperparameter_scans.py

@@ -78,7 +78,7 @@ def generate_topology(topology_grid, probability_decay_factor=0.9):
  return ''
  return ''
 
-def coarse_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, mlp=False, custom_jobs=[], model_complexity_factor=0.9, set_beta=-1., n_jobs=4, categorical=True, add_softmax=False):
+def coarse_scan(hyperparameter_input_csv, hyperparameter_output_log, generate_input, job_chunk_size, stratify_column, reset_all, torque, gpu, gpu_node, nohup, mlp=False, custom_jobs=[], model_complexity_factor=0.9, set_beta=-1., n_jobs=4, categorical=True, add_softmax=False, additional_command = ""):
  """Perform randomized hyperparameter grid search
 
  Parameters
@@ -182,7 +182,7 @@ def run(x):
  commands[i] = '{} {} {} {}'.format('CUDA_VISIBLE_DEVICES="{}"'.format(next(gpus)) if not torque else "",'nohup' if nohup else '',commands[i],'&' if nohup else '') # $gpuNum
  if torque:
  for command in commands:
- job = assemble_run_torque(command, use_gpu=True, additions='', queue='gpuq', time=2, ngpu=1, additional_options='' if gpu_node == -1 else ' -l hostlist=g0{}'.format(gpu_node))
+ job = assemble_run_torque(command, use_gpu=True, additions=additional_command, queue='gpuq', time=4, ngpu=1, additional_options='' if gpu_node == -1 else ' -l hostlist=g0{}'.format(gpu_node))
  else:
  if len(commands) == 1:
  subprocess.call(commands[0],shell=True)

methylnet/torque_job_runner.py

@@ -38,7 +38,7 @@ def assemble_replace_dict(command, use_gpu, additions, queue, time, ngpu):
 
  """
  replace_dict = {'COMMAND':"{} {}".format('CUDA_VISIBLE_DEVICES="$gpuNum"' if use_gpu else '',command),
- 'GPU_COMMANDS':"""gpuNum=`cat $PBS_GPUFILE | sed -e 's/.*-gpu//g'`
+ 'GPU_SETUP':"""gpuNum=`cat $PBS_GPUFILE | sed -e 's/.*-gpu//g'`
 unset CUDA_VISIBLE_DEVICES
 export CUDA_DEVICE=$gpuNum""" if use_gpu else '',
  'NGPU':'#PBS -l gpus={}'.format(ngpu) if ngpu else '',
@@ -70,7 +70,7 @@ def run_torque_job_(replace_dict, additional_options=""):
 #PBS -l walltime=TIME:00:00
 #PBS -j oe
 cd $PBS_O_WORKDIR
-GPU_COMMANDS
+GPU_SETUP
 ADDITIONS
 COMMAND"""
  for k,v in replace_dict.items():