Skip to content

Commit

Permalink
fix: prevent athena.to_iceberg overwrite to delete table in order to …
Browse files Browse the repository at this point in the history
…preserve Iceberg transactions history (#2776)
  • Loading branch information
erwan-simon authored Apr 22, 2024
1 parent 60beb95 commit 6dd28e4
Showing 1 changed file with 14 additions and 5 deletions.
19 changes: 14 additions & 5 deletions awswrangler/athena/_write_iceberg.py
Original file line number Diff line number Diff line change
Expand Up @@ -383,11 +383,6 @@ def to_iceberg(
glue_table_settings if glue_table_settings else {},
)

if mode == "overwrite":
catalog.delete_table_if_exists(
database=database, table=table, catalog_id=catalog_id, boto3_session=boto3_session
)

try:
# Create Iceberg table if it doesn't exist
if not catalog.does_table_exist(
Expand Down Expand Up @@ -469,6 +464,20 @@ def to_iceberg(
s3_additional_kwargs=s3_additional_kwargs,
catalog_id=catalog_id,
)
# if mode == "overwrite", delete whole data from table (but not table itself)
elif mode == "overwrite":
delete_sql_statement = f"DELETE FROM {table}"
delete_query_execution_id: str = _start_query_execution(
sql=delete_sql_statement,
workgroup=workgroup,
wg_config=wg_config,
database=database,
data_source=data_source,
encryption=encryption,
kms_key=kms_key,
boto3_session=boto3_session,
)
wait_query(query_execution_id=delete_query_execution_id, boto3_session=boto3_session)

# Create temporary external table, write the results
s3.to_parquet(
Expand Down

0 comments on commit 6dd28e4

Please sign in to comment.