From f68b250d6549abcdb09821ba3076a8a40aef5b8d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 11 Dec 2024 12:00:55 +0000 Subject: [PATCH] fix iterating through resource table with seek method by recreating the table with rowid sorted --- src/ota_metadata/legacy/metadata.py | 41 ++++------------------ src/ota_metadata/utils/sqlite3_helper.py | 31 ++++++++++++++++ src/otaclient/create_standby/_delta_gen.py | 11 ++++-- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/ota_metadata/legacy/metadata.py b/src/ota_metadata/legacy/metadata.py index d3e5caedd..9c0cbf125 100644 --- a/src/ota_metadata/legacy/metadata.py +++ b/src/ota_metadata/legacy/metadata.py @@ -61,6 +61,7 @@ from ota_metadata.file_table._table import FileTableRegularFiles from ota_metadata.utils import DownloadInfo from ota_metadata.utils.cert_store import CAChainStore +from ota_metadata.utils.sqlite3_helper import sort_and_place from otaclient_common.common import urljoin_ensure_base from otaclient_common.typing import StrOrPath @@ -83,39 +84,6 @@ DB_TIMEOUT = 16 # seconds -def _sort_ft_regular_in_place(_orm: FileTableRegularFilesORM) -> None: - """Sort the ft_regular table by digest, and then replace the old table - with the sorted one. - - This is required for later otaclient applies update to standby slot. - """ - SORTED_TABLE_NAME = "ft_regular_sorted" - ORIGINAL_TABLE_NAME = FileTableRegularFiles.table_name - - _table_create_stmt = _orm.orm_table_spec.table_create_stmt(SORTED_TABLE_NAME) - _dump_sorted = ( - f"INSERT INTO {SORTED_TABLE_NAME} SELECT * FROM " - f"{ORIGINAL_TABLE_NAME} ORDER BY digest;" - ) - - conn = _orm.orm_con - with conn as conn: - conn.executescript( - "\n".join( - [ - "BEGIN;", - _table_create_stmt, - _dump_sorted, - f"DROP TABLE {ORIGINAL_TABLE_NAME};", - f"ALTER TABLE {SORTED_TABLE_NAME} RENAME TO {ORIGINAL_TABLE_NAME};", - ] - ) - ) - - with conn as conn: - conn.execute("VACUUM;") - - class OTAMetadata: """ workdir layout: @@ -274,7 +242,12 @@ def download_metafiles( _orm=_ft_regular_orm, _orm_rs=_rs_orm, ) - _sort_ft_regular_in_place(_ft_regular_orm) + # NOTE: also check file_table definition at ota_metadata.file_table._table + sort_and_place( + _ft_regular_orm, + FileTableRegularFiles.table_name, + order_by_col="digest", + ) except Exception as e: _err_msg = f"failed to parse CSV metafiles: {e!r}" logger.error(_err_msg) diff --git a/src/ota_metadata/utils/sqlite3_helper.py b/src/ota_metadata/utils/sqlite3_helper.py index ecf24f099..53e7109a5 100644 --- a/src/ota_metadata/utils/sqlite3_helper.py +++ b/src/ota_metadata/utils/sqlite3_helper.py @@ -30,6 +30,7 @@ def iter_all( """Iter all entries with seek method by rowid. NOTE: the target table must has rowid defined! + NOTE: the rowid MUST BE continues without any holes! """ _pagination_stmt = self.orm_table_spec.table_select_stmt( select_from=self.orm_table_name, @@ -55,6 +56,7 @@ def iter_all_with_shuffle( """Iter all entries with seek method by rowid, shuffle each batch before yield. NOTE: the target table must has rowid defined! + NOTE: the rowid MUST BE continues without any holes! """ _pagination_stmt = self.orm_table_spec.table_select_stmt( select_from=self.orm_table_name, @@ -75,3 +77,32 @@ def iter_all_with_shuffle( random.shuffle(_batch) yield from _batch + + +def sort_and_place(_orm: ORMBase, table_name: str, *, order_by_col: str) -> None: + """Sort the table, and then replace the old table with the sorted one.""" + ORIGINAL_TABLE_NAME = table_name + SORTED_TABLE_NAME = f"{table_name}_sorted" + _table_spec = _orm.orm_table_spec + + _table_create_stmt = _table_spec.table_create_stmt(SORTED_TABLE_NAME) + _dump_sorted = ( + f"INSERT INTO {SORTED_TABLE_NAME} SELECT * FROM " + f"{ORIGINAL_TABLE_NAME} ORDER BY {order_by_col};" + ) + + conn = _orm.orm_con + with conn as conn: + conn.executescript( + "\n".join( + [ + "BEGIN;", + _table_create_stmt, + _dump_sorted, + f"DROP TABLE {ORIGINAL_TABLE_NAME};", + f"ALTER TABLE {SORTED_TABLE_NAME} RENAME TO {ORIGINAL_TABLE_NAME};", + ] + ) + ) + with conn as conn: + conn.execute("VACUUM;") diff --git a/src/otaclient/create_standby/_delta_gen.py b/src/otaclient/create_standby/_delta_gen.py index bb362b0b8..22c1315e2 100644 --- a/src/otaclient/create_standby/_delta_gen.py +++ b/src/otaclient/create_standby/_delta_gen.py @@ -33,8 +33,10 @@ ) from ota_metadata.legacy.metadata import OTAMetadata from ota_metadata.legacy.rs_table import ( + ResourceTable, RSTableORMThreadPool, ) +from ota_metadata.utils.sqlite3_helper import sort_and_place from otaclient._status_monitor import StatusReport, UpdateProgressReport from otaclient.configs.cfg import cfg from otaclient_common.common import create_tmp_fname @@ -258,8 +260,13 @@ def calculate_delta(self) -> None: thread_local=thread_local, ).add_done_callback(self._task_done_callback) - # NOTE: fill up the holes created by DELETE. - self._rstable_orm.orm_execute("VACUUM;") + # NOTE: fill up the holes created by DELETE, and make + # the rowid continues again. + sort_and_place( + self._rstable_orm, + ResourceTable.table_name, + order_by_col="rowid", + ) finally: pool.shutdown(wait=True) self._ft_regular_orm._con.close()