Fail early on crashes; purge cache directory on rerun (Fixes #11)

google-research · Apr 15, 2022 · d584d0c · d584d0c
1 parent 8a172b0
commit d584d0c
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/scripts/deduplicate_single_file.sh b/scripts/deduplicate_single_file.sh
@@ -1,3 +1,4 @@
+set -e
 python3 scripts/make_suffix_array.py $1
 cargo run self-similar --data-file $1 --length-threshold $3 --cache-dir /tmp/cache --num-threads $4
 cargo run collect --data-file $1 --cache-dir /tmp/cache --length-threshold $3 > /tmp/drop_tokens_file

diff --git a/scripts/make_suffix_array.py b/scripts/make_suffix_array.py
@@ -65,7 +65,6 @@
  for x,(s,e) in zip(files,started):
  size_data = os.path.getsize(x)
  FACT = np.ceil(np.log(size_data)/np.log(2)/8)
- print("FACT", FACT)
  size_table = os.path.getsize(x+".table.bin")
  if not os.path.exists(x) or not os.path.exists(x+".table.bin") or size_table == 0 or size_data*FACT != size_table:
  cmd = "./target/debug/dedup_dataset make-part --data-file %s --start-byte %d --end-byte %d"%(sys.argv[1], s, e)
@@ -80,13 +79,21 @@
 
 print("Merging suffix trees")
 
+os.popen("rm tmp/out.table.bin.*").read()
+
 torun = " --suffix-path ".join(files)
 print("./target/debug/dedup_dataset merge --output-file %s --suffix-path %s --num-threads %d"%("tmp/out.table.bin", torun, mp.cpu_count()))
 os.popen("./target/debug/dedup_dataset merge --output-file %s --suffix-path %s --num-threads %d"%("tmp/out.table.bin", torun, mp.cpu_count())).read()
 #exit(0)
 print("Now merging individual tables")
 os.popen("cat tmp/out.table.bin.* > tmp/out.table.bin").read()
 print("Cleaning up")
-#os.popen("rm tmp/out.table.bin.*").read()
 os.popen("mv tmp/out.table.bin %s.table.bin"%sys.argv[1]).read()
 
+if os.path.exists(sys.argv[1]+".table.bin"):
+ if os.path.getsize(sys.argv[1]+".table.bin")%os.path.getsize(sys.argv[1]) != 0:
+ print("File size is wrong")
+ exit(1)
+else:
+ print("Failed to create table")
+ exit(1)
diff --git a/src/main.rs b/src/main.rs
@@ -511,8 +511,12 @@ fn cmd_self_similar(data_file: &String, length_threshold: &usize, frequency_thre
  let mut first = true;
 
  loop {
- cur_location = get_next_pointer_from_table(&mut table);
+ cur_location = get_next_pointer_from_table_canfail(&mut table);
  i += 1;
+ if cur_location == std::u64::MAX {
+ // The last two items in the file matched
+ break;
+ }
 
  let suf2 = &text[cur_location as usize..];
  let does_match = suf2.len() >= length_threshold && suf1.len() >= length_threshold && suf1[..length_threshold] == suf2[..length_threshold];