Moved plots to plots/{name} and streamlined rule files

workflow/rules/aggregation.smk

@@ -1,12 +1,13 @@
 from snakemake_tools import calculate_memory_resource as mem_res
 
+include: "common.smk"
 
 def aggregate_input(wildcards):
  decode_level = wildcards["decode_level"]
  decomp_level = wildcards["decomp_level"]
  perf_file = "Decoding/decoder/{conditions}/{feature}/{decoder}/perf_df.pkl"
  if decomp_level == "dataset":
- decomp_base_dir = "results/data/{dataset_id}/{parcellation}"
+ decomp_base_dir = f"{DATA_DIR}/{{dataset_id}}/{{parcellation}}"
  if decode_level == "dataset":
  inputs = f"{decomp_base_dir}/{{dataset_id}}/{perf_file}".format(**wildcards)
  elif decode_level == "subsets":
@@ -20,7 +21,7 @@ def aggregate_input(wildcards):
  else:
  raise ValueError
  elif decomp_level == "subsets":
- decomp_base_dir = "results/data/{subset_id}/{parcellation}"
+ decomp_base_dir = f"{DATA_DIR}/{{subset_id}}/{{parcellation}}"
  if decode_level == "subsets":
  inputs = [f"{decomp_base_dir}/{subset_id}/{perf_file}".format(**wildcards, subset_id=subset_id)
  for subset_id in config["dataset_groups"][wildcards["dataset_id"]]
@@ -33,7 +34,7 @@ def aggregate_input(wildcards):
  else:
  raise ValueError
  elif decomp_level == "sessions":
- decomp_base_dir = "results/data/{session_id}/{parcellation}"
+ decomp_base_dir = f"{DATA_DIR}/{{session_id}}/{{parcellation}}"
  if decode_level == "sessions":
  inputs = [f"{decomp_base_dir}/{session_id}/{perf_file}".format(**wildcards, session_id=session_id)
  for session_id in config["dataset_sessions"][wildcards["dataset_id"]]
@@ -48,13 +49,13 @@ rule aggregate_perf:
  input:
  aggregate_input
  output:
- f"results/plots/{{dataset_id}}/{{decode_level}}_in_{{decomp_level}}/{{conditions}}/{{feature}}/{{parcellation}}/{{decoder}}/aggr_perf_df.pkl"
+ f"{PLOTS_DIR}/{{dataset_id}}/{{decode_level}}_in_{{decomp_level}}/{{conditions}}/{{feature}}/{{parcellation}}/{{decoder}}/aggr_perf_df.pkl"
  params:
  decode_level = '{decode_level}',
  decomp_level = '{decomp_level}',
  aggregated_from = '{decode_level} in {decomp_level}'
  log:
- f"results/plots/{{dataset_id}}/{{decode_level}}_in_{{decomp_level}}/{{conditions}}/{{feature}}/{{parcellation}}/{{decoder}}/df_aggregation.log"
+ f"{PLOTS_DIR}/{{dataset_id}}/{{decode_level}}_in_{{decomp_level}}/{{conditions}}/{{feature}}/{{parcellation}}/{{decoder}}/df_aggregation.log"
  conda:
  "../envs/environment.yaml"
  script:

workflow/rules/common.smk

@@ -0,0 +1,11 @@
+from snakemake_tools import alias_dataset, unalias_dataset, temp_if_config
+
+def temp_c(in_file, rule=None):
+ return temp_if_config(in_file, config.get("temporary_outputs",{}), rule)
+
+alias = lambda dataset: alias_dataset(config["dataset_aliases"], dataset)
+unalias = lambda dataset: unalias_dataset(config["dataset_aliases"], dataset)
+
+DATA_DIR = f"results/data"
+PLOTS_DIR = f"results/plots/{config['name']}"
+EXPORTS_DIR = f"results/exports/{config['name']}"

workflow/rules/export.smk

@@ -1,15 +1,15 @@
-from snakemake_tools import calculate_memory_resource as mem_res, unalias_dataset
+from snakemake_tools import calculate_memory_resource as mem_res
 
-unalias = lambda dataset: unalias_dataset(config["dataset_aliases"], dataset)
+include: "common.smk"
 
 rule export_data:
  input:
- lambda w: f"results/data/{unalias(w['decomp_alias'])}/{{parcellation}}/{unalias(w['dataset_alias'])}/data.h5"
+ lambda w: f"{DATA_DIR}/{unalias(w['decomp_alias'])}/{{parcellation}}/{unalias(w['dataset_alias'])}/data.h5"
  output:
- temporals = f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/temporals.{{ext}}",
- spatials = f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/spatials.{{ext}}",
+ temporals = f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/temporals.{{ext}}",
+ spatials = f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/spatials.{{ext}}",
  log:
- f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/export_data.{{ext}}.log"
+ f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/export_data.{{ext}}.log"
  wildcard_constraints:
  #decomp_alias = r"[a-zA-Z\d_.#-]+",
  #dataset_alias = r"[a-zA-Z\d_.#-]+",
@@ -21,9 +21,9 @@ rule export_data:
 
 use rule export_data as export_data_deeper with:
  input:
- lambda w: f"results/data/{unalias(w['decomp_alias'])}/{{parcellation}}/{unalias(w['dataset_alias'])}/Features/{{rest}}/data.h5"
+ lambda w: f"{DATA_DIR}/{unalias(w['decomp_alias'])}/{{parcellation}}/{unalias(w['dataset_alias'])}/Features/{{rest}}/data.h5"
  output:
- temporals = f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/temporals.{{ext}}",
- spatials = f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/spatials.{{ext}}",
+ temporals = f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/temporals.{{ext}}",
+ spatials = f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/spatials.{{ext}}",
  log:
- f"results/exports/{config['name']}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/export_data.{{ext}}.log"
+ f"{EXPORTS_DIR}/{{decomp_alias}}/{{dataset_alias}}/{{parcellation}}/{{rest}}/export_data.{{ext}}.log"

workflow/rules/loading.smk

@@ -1,8 +1,7 @@
-from snakemake_tools import create_parameters, create_conditions, calculate_memory_resource as mem_res, branch_match, hash_config, temp_if_config
 from wildcard_functions import subjects_from_wildcard
+from snakemake_tools import calculate_memory_resource as mem_res
 
-def temp_c(file, rule=None):
- return temp_if_config(file, config.get("temporary_outputs",{}), rule)
+include: "common.smk"
 
 ### Todo unify simon and gerions inout functions (easy)
 
@@ -53,14 +52,14 @@ rule load_GN:
  input:
  unpack(sessions_input_gerion)
  output:
- temp_c("results/data/{session_id}/SVD/{session_id}/data.h5", rule="load"),
- align_plot = report("results/data/{session_id}/SVD/{session_id}/alignment.pdf", caption="../report/alignment.rst", category="1 Brain Alignment", labels={"Dataset": "GN", "Subjects":"{session_id}"}),
- config = "results/data/{session_id}/SVD/{session_id}/conf.yaml",
- stim_side = report("results/data/{session_id}/SVD/{session_id}/stim_side.pdf", caption="../report/alignment.rst", category="0 Loading", labels={"Dataset": "GN", "Subjects":"{session_id}"})
+ temp_c(f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/data.h5", rule="load"),
+ align_plot = report(f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/alignment.pdf", caption="../report/alignment.rst", category="1 Brain Alignment", labels={"Dataset": "GN", "Subjects":"{{session_id}}"}),
+ config = f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/conf.yaml",
+ stim_side = report(f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/stim_side.pdf", caption="../report/alignment.rst", category="0 Loading", labels={"Dataset": "GN", "Subjects":"{{session_id}}"})
  wildcard_constraints:
  session_id = r"GN[\w_.\-]*"
  log:
- "results/data/{session_id}/SVD/{session_id}/pipeline_entry.log"
+ f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/pipeline_entry.log"
  conda:
  "../envs/environment.yaml"
  resources:
@@ -76,13 +75,13 @@ rule load_mSM:
  input:
  unpack(sessions_input_simon)
  output:
- temp_c("results/data/{session_id}/SVD/{session_id}/data.h5", rule="load"),
- align_plot = report("results/data/{session_id}/SVD/{session_id}/alignment.pdf", caption="../report/alignment.rst", category="1 Brain Alignment", labels={"Dataset": "mSM", "Subjects":"{session_id}"}),
- config = "results/data/{session_id}/SVD/{session_id}/conf.yaml",
+ temp_c(f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/data.h5", rule="load"),
+ align_plot = report(f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/alignment.pdf", caption="../report/alignment.rst", category="1 Brain Alignment", labels={"Dataset": "mSM", "Subjects":"{{session_id}}"}),
+ config = f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/conf.yaml",
  wildcard_constraints:
  session_id = r"mSM[\w_.\-]*"
  log:
- "results/data/{session_id}/SVD/{session_id}/pipeline_entry.log"
+ f"{DATA_DIR}/{{session_id}}/SVD/{{session_id}}/pipeline_entry.log"
  conda:
  "../envs/environment.yaml"
  resources:
@@ -95,12 +94,12 @@ def input_unification(wildcards):
  # if dataset_id is a defined alias, replace it with the canonical (hash) id, else leave it be
  if dataset_id in config['dataset_aliases']:
  dataset_id = config['dataset_aliases'].get(dataset_id)
- input = [ f"results/data/{dataset_id}/SVD/{dataset_id}/data.h5" ]
+ input = [ f"{DATA_DIR}/{dataset_id}/SVD/{dataset_id}/data.h5" ]
  else:
  digest_lng = config.get('hash_digest_length', 8)
  matched_ids = [ id for id in config['datasets'].keys() if dataset_id[:digest_lng] == id[:digest_lng] ]
  assert (len(matched_ids) == 1), f"Did not match exactly one dataset for {dataset_id=}, but instead {matched_ids=}"
- input = [ f"results/data/{subj}-{date}/SVD/{subj}-{date}/data.h5" for subj, date in config['datasets'].get(matched_ids[0])]
+ input = [ f"{DATA_DIR}/{subj}-{date}/SVD/{subj}-{date}/data.h5" for subj, date in config['datasets'].get(matched_ids[0])]
  return input
 
 rule unify:
@@ -110,10 +109,10 @@ rule unify:
  input:
  unpack(input_unification)
  output:
- temp_c("results/data/{dataset_id}/SVD/{dataset_id}/data.h5", rule="unify"),
- config = "results/data/{dataset_id}/SVD/{dataset_id}/conf.yaml",
+ temp_c(f"{DATA_DIR}/{{dataset_id}}/SVD/{{dataset_id}}/data.h5", rule="unify"),
+ config = f"{DATA_DIR}/{{dataset_id}}/SVD/{{dataset_id}}/conf.yaml",
  log:
- "results/data/{dataset_id}/SVD/{dataset_id}/unify.log"
+ f"{DATA_DIR}/{{dataset_id}}/SVD/{{dataset_id}}/unify.log"
  params:
  alias=lambda wildcards: wildcards['dataset_id'] in config['dataset_aliases']
  conda: