Dedup single file will abort early if merge fails (fixes google-resea…

…rch#12)
norabelrose · Apr 15, 2022 · 3c209d0 · 3c209d0
1 parent d584d0c
commit 3c209d0
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/scripts/make_suffix_array.py b/scripts/make_suffix_array.py
@@ -83,7 +83,12 @@
 
 torun = " --suffix-path ".join(files)
 print("./target/debug/dedup_dataset merge --output-file %s --suffix-path %s --num-threads %d"%("tmp/out.table.bin", torun, mp.cpu_count()))
-os.popen("./target/debug/dedup_dataset merge --output-file %s --suffix-path %s --num-threads %d"%("tmp/out.table.bin", torun, mp.cpu_count())).read()
+pipe = os.popen("./target/debug/dedup_dataset merge --output-file %s --suffix-path %s --num-threads %d"%("tmp/out.table.bin", torun, mp.cpu_count()))
+output = pipe.read()
+if pipe.close() is not None:
+ print("Something went wrong with merging.")
+ print("Please check that you ran with ulimit -Sn 100000")
+ exit(1)
 #exit(0)
 print("Now merging individual tables")
 os.popen("cat tmp/out.table.bin.* > tmp/out.table.bin").read()