in the module general_tools, the functionality for searching existing…

… reconstructions was modified. The procedure now finds the existing experiments, the existing reconstructions and matches both. It creates a dataframe with for columns: experiments, number of + path to reconstructions (if any), and a flag if a reconstruction was moved into a '_finished_reconstructions' folder to mark them as finished and ease a user's life. Non matched recons are put in the category 'others'.

alrecon/components/general_tools.py

@@ -7,6 +7,12 @@
 
 import re
 
+from pathlib import Path
+
+
+def get_project_root() -> Path:
+ return str(Path(__file__).parent.parent)
+
 
 def sorted_alphanum(listOfStrings):
  """Sorts the given iterable in the way that is expected. Converts each given list to a list of strings
@@ -27,7 +33,7 @@ def alphanum_key(key):
  return sorted(listOfStrings, key=alphanum_key)
 
 
-def collect_recon_paths(root_dir):
+def collect_recon_paths(root_dir, search_string):
  """
  check for .Trash or empty dirs?
  could be improved for speed, I have the impression that "it walks too much"
@@ -37,7 +43,7 @@ def collect_recon_paths(root_dir):
 
  for dirpath, dirnames, filenames in os.walk(root_dir):
  # Temporarily store directories that match 'recon'
- temp_recon_dirs = [d for d in dirnames if d.startswith("recon")]
+ temp_recon_dirs = [d for d in dirnames if d.startswith(search_string)]
 
  # For each directory that starts with 'recon', find the first parent that doesn't
  for dirname in temp_recon_dirs:
@@ -54,26 +60,103 @@ def collect_recon_paths(root_dir):
 
  # Ensure we still traverse into subdirectories of 'recon'
  # reference to dirnames
- dirnames[:] = [d for d in dirnames if d in temp_recon_dirs or not d.startswith("recon")]
+ dirnames[:] = [d for d in dirnames if d in temp_recon_dirs or not d.startswith(search_string)]
  # print("dirnames", dirnames)
 
- # sorting keys alphanumerically:
+ return dict_recon_paths
+
+
+def get_dict_files_with_ending(source_dir, file_ending):
+ found_files = [os.path.join(d, x) for d, dirs, files in os.walk(source_dir) for x in files if x.endswith(file_ending)]
+ dict_found_files = {os.path.splitext(os.path.basename(file))[0]: file for file in found_files}
+ return dict_found_files
+
+
+def message_if_list_empty(key, should_be_a_list):
+ len_of_list = len(should_be_a_list)
+ if len_of_list > 0:
+ keys_corresponding_to_path = [""] * len_of_list
+ keys_corresponding_to_path[0] = key
+ return [keys_corresponding_to_path, should_be_a_list]
+ else:
+ return [[key], ["no reconstructions so far"]]
+ # return [[key], [" "]]
+
+
+def return_number_of_files_in_folder(path):
+ if os.path.isdir(path):
+ return len(os.listdir(path))
+ return ""
+
+
+def does_its_path_contain(list_of_strings, path):
+ """
+ list_of_strings must be a list!!!!
+ """
+
+ for string_ in list_of_strings:
+ if string_ in path:
+ return "completed"
+ return " "
+
+
+def provide_unified_data_frame_from_dicts(dict_exp_files, dict_recon_paths, list_of_marker_for_completed_recons):
+ """
+ should contain:
+ exp name1 - recon name - elements in folder - paths
+ recon name - elements in folder - paths
+ recon name - elements in folder - paths
+ exp name2 - recon name - elements in folder - paths
+ recon name - elements in folder - paths
+
+ # für jeden key der experimente brauchen wir mindestens einen recon gegenwert, ansonsten schreiben wir ein "missing oä". alle anderen kommen in den key "others"
+ """
+
+ keys_exp = sorted_alphanum(dict_exp_files.keys())
+
+ df_rows = [
+ (key_corresponding_to_path, return_number_of_files_in_folder(path), path, does_its_path_contain(list_of_marker_for_completed_recons, path))
+ for key in keys_exp
+ for key_corresponding_to_path, path in zip(*message_if_list_empty(key, sorted_alphanum(dict_recon_paths.pop(key, []))))
+ ]
+
+ remaining_datasets = dict_recon_paths.values() # unclassified
+
+ def flatten_concatenation(list_of_lists):
+ flat_list = []
+ for row in list_of_lists:
+ flat_list += row
+ return flat_list
 
- keys = sorted_alphanum(dict_recon_paths.keys())
+ remaining_datasets = flatten_concatenation(remaining_datasets)
 
- df_rows = [(key, path) for key in keys for path in sorted_alphanum(dict_recon_paths.get(key))]
+ df_rows_remaining = [
+ (key_corresponding_to_path, return_number_of_files_in_folder(path), path, does_its_path_contain(list_of_marker_for_completed_recons, path))
+ for key_corresponding_to_path, path in zip(*message_if_list_empty("others", sorted_alphanum(remaining_datasets)))
+ ]
 
- # df_rows = [(key, path) for key, paths in dict_recon_paths.items() for path in paths]
- df = pd.DataFrame(df_rows, columns=["Parent Directory", "recon_dir"])
+ df_rows += df_rows_remaining
+
+ df = pd.DataFrame(df_rows, columns=["exp dataset", "number_of_files", "recon_dir", "recon complete?"])
 
  return df
 
 
-from pathlib import Path
+def create_dataoverview_BEATS(exp_dir, recon_dir, list_of_marker_for_completed_recons=["_finished_reconstructions"]):
+ dict_exp_files = get_dict_files_with_ending(source_dir=exp_dir, file_ending=".h5")
 
+ dict_recon_paths = collect_recon_paths(root_dir=recon_dir, search_string="recon")
 
-def get_project_root() -> Path:
- return str(Path(__file__).parent.parent)
+ df = provide_unified_data_frame_from_dicts(dict_exp_files, dict_recon_paths, list_of_marker_for_completed_recons)
+
+ return df
+
+
+# ==== # ==== # ====
+# ==== # ==== # ====
+# ==== # ==== # ====
+# ==== # ==== # ====
+# ==== # ==== # ====
 
 
 def settings_file():

alrecon/pages/__init__.py

@@ -13,7 +13,9 @@
 matplotlib.rc("font", **font)
 
 from alrecon.components import alrecon, viewers
-from alrecon.components.general_tools import collect_recon_paths
+
+# from alrecon.components.general_tools import collect_recon_paths
+from alrecon.components.general_tools import create_dataoverview_BEATS
 
 ar = alrecon.alrecon()
 view = viewers.viewers()
@@ -105,6 +107,7 @@ def ProcessExtendedFOVScan():
  image_path = Path("./docs/pictures/overlap_optimization.png").as_posix()
  solara.Image(image_path) # , width='1500px'
 
+
 @solara.component
 def OverlapInspect():
  with solara.Card(title="Find the sinogram overlap", margin=0, classes=["my-2"], style={"min-width": "900px"}):
@@ -689,7 +692,6 @@ def on_action_cell(column, row_index):
 # solara.DataFrame(master_local, items_per_page=50, cell_actions=cell_actions) # , scrollable=True
 
 
-
 @solara.component
 def ReconList(master_updated, pandastable=True):
  """
@@ -709,18 +711,27 @@ def read_gspread_and_update_state():
  def search_for_recons():
  """
  We update here the ar.master, which is self.master in read_gspread_master, directly.
+ The function "create_dataoverview_BEATS" takes the experiment directory andr the recon directory
+ and returns an overview of experiments vs reconstructions vs "unrelated existing objects"
+
+ ATTENTION: The following is somehow a "hack"
+
+ search_root_recons = os.path.dirname(os.path.dirname(ar.recon_dir.value))
+
+ since this is only possible because we know that at BEATS the 'root of recon' is two levels up.
+ But in reality we must fix that by a seperate variable! e.g. by having ar RECON_ROOT
  """
 
- # this is also only possible because we know at BEATS that the 'root' is two levels up.
- # But in reality we must fix that by a seperate variable! e.g. by having a RECON_ROOT
- search_root = os.path.dirname(os.path.dirname(ar.recon_dir.value))
- ar.master = collect_recon_paths(search_root)
+ search_root_experiments = ar.experiment_dir.value
+ search_root_recons = os.path.dirname(os.path.dirname(ar.recon_dir.value))
+ # ar.master = collect_recon_paths(search_root)
+ ar.master = create_dataoverview_BEATS(exp_dir=search_root_experiments, recon_dir=search_root_recons)
  master_updated.value += 1
 
  def load_sheet_or_crawl():
  pass
 
- #selected_button = "dirs"
+ # selected_button = "dirs"
 
  ### perhaps a function that lets us select the right opener?