Skip to content

Commit

Permalink
Fix drop table during iceberg cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
holdenk committed Jul 7, 2022
1 parent 856e483 commit faba6da
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions pipelinecompare/domagic.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ async def run_pipelines():
# currently no git like branching buuuut we can hack something "close enough"
magic = f"magic_cmp_{mytestid}"
tbl_id = 0
tbl_prefix = args.table_prefix

def run_spark_sql_query(query):
cmd = spark_sql_command.copy()
Expand All @@ -198,7 +199,6 @@ def snapshot_ish(table_name):
def make_tbl_like(table_name):
global tbl_id
tbl_id = tbl_id + 1
tbl_prefix = args.table_prefix
new_table_name = f"{tbl_prefix}{tbl_id}{magic}"
run_spark_sql_query(f"CREATE TABLE {new_table_name} LIKE {table_name}")
return new_table_name
Expand Down Expand Up @@ -239,7 +239,7 @@ async def run_pipelines():
finally:
if not args.no_cleanup:
for tid in range(0, tbl_id):
table_name = f"{tid}{magic}"
table_name = f"{tbl_prefix}{tbl_id}{magic}"
proc = run_spark_sql_query(f"DROP TABLE {table_name}")


Expand Down

0 comments on commit faba6da

Please sign in to comment.