From d2d3d7079cf5bc3ebc7bbdab45a10b92b0321639 Mon Sep 17 00:00:00 2001 From: Dimitris Papagiannis Date: Wed, 30 Aug 2023 16:33:32 +0200 Subject: [PATCH] More cleanup and refactoring --- db.py | 23 ++++++++++------------- dqmsquare_cfg.py | 3 --- grabber.py | 2 +- tests/test_1.py | 2 -- tests/test_2.py | 4 +--- 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/db.py b/db.py index 2e3da25..b8e8a1d 100644 --- a/db.py +++ b/db.py @@ -364,7 +364,6 @@ def get(self, run_start, run_end, bad_only=False, with_ls_only=False): def make_mirror_entry(self, data): answer = [] - # values = (id , client , run , rev , hostname , exit_code , events_total , events_rate , cmssw_run , cmssw_lumi , client_path , runkey , fi_state, timestamp, VmRSS, stdlog_start, stdlog_end ) ( id, client, @@ -542,12 +541,11 @@ def get_clients(self, run_start: int, run_end: int) -> list: # self.log.debug( "return " + str(answer) ) return answer - # update metadata table with info about min and max run number in runs table for fast fetch def update_min_max(self, new_min: int, new_max: int): + """update metadata table with info about min and max run number in runs table for fast fetch""" with self.engine.connect() as cur: session = self.Session(bind=cur) try: - # cur.execute("INSERT OR REPLACE INTO " + self.TB_NAME_META + " " + self.TB_DESCRIPTION_META_SHORT + " VALUES('min_max_runs', '[" + str(new_min) + "," + str(new_max) + "]')" ) session.execute( f"DELETE FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';" ) @@ -584,24 +582,24 @@ def get_info(self) -> list: return answer - def get_rev(self, machine: str) -> int: + def get_latest_revision(self, host: str) -> int: """ Search the runs table in the DB for get latest rev for a given dqm machine """ self.log.debug("DQM2MirrorDB.get_rev()") - if ".cms" in machine: - machine = machine[: -len(".cms")] + if ".cms" in host: + host = host[: -len(".cms")] with self.engine.connect() as cur: - if "fu" in machine: + if "fu" in host: answer = cur.execute( - f"SELECT MAX(rev) FROM {self.TB_NAME_RUNS} WHERE hostname = '{machine}';" + f"SELECT MAX(rev) FROM {self.TB_NAME_RUNS} WHERE hostname = '{host}';" ).all() answer = list(answer[0]) return answer[0] else: answer = cur.execute( - f"SELECT MAX(rev) FROM {self.TB_NAME_GRAPHS} WHERE hostname = '{machine}';" + f"SELECT MAX(rev) FROM {self.TB_NAME_GRAPHS} WHERE hostname = '{host}';" ).all() answer = list(answer[0]) return answer[0] @@ -618,16 +616,15 @@ def get_logs(self, client_id: int) -> list: answer = answer[0] return answer - # get next run and prev run, unordered def get_runs_around(self, run: int) -> list: + """ + get next run and prev run, unordered + """ answer = [] self.log.debug("DQM2MirrorDB.get_runs_around()") with self.engine.connect() as cur: answer = cur.execute( f"SELECT min(run) FROM {self.TB_NAME_RUNS} WHERE run > {run} union SELECT max(run) FROM {self.TB_NAME_RUNS} WHERE run < {run};" ).all() - # answer1 = cur.execute( "SELECT min(run) from " + self.TB_NAME_RUNS + " where run > " + str(run) + ";" ).all() - # answer2 = cur.execute( "SELECT max(run) FROM " + self.TB_NAME_RUNS + " WHERE run < " + str(run) + ";" ).all() - # print( run, answer, answer1, answer2 ) answer = [item[0] for item in answer] return answer diff --git a/dqmsquare_cfg.py b/dqmsquare_cfg.py index 4a35b57..7d3f726 100644 --- a/dqmsquare_cfg.py +++ b/dqmsquare_cfg.py @@ -11,7 +11,6 @@ def format_db_uri( - env: str = "production", username: str = "postgres", password: str = "postgres", host: str = "postgres", @@ -129,7 +128,6 @@ def load_cfg() -> dict: cfg["GRABBER_DEBUG"] = os.environ.get("GRABBER_DEBUG", False) cfg["DB_PLAYBACK_URI"] = format_db_uri( - env=cfg["ENV"], username=os.environ.get("POSTGRES_USERNAME", "postgres"), password=os.environ.get("POSTGRES_PASSWORD", "postgres"), host=os.environ.get("POSTGRES_HOST", "127.0.0.1"), @@ -137,7 +135,6 @@ def load_cfg() -> dict: db_name=os.environ.get("POSTGRES_PLAYBACK_DB_NAME", "postgres"), ) cfg["DB_PRODUCTION_URI"] = format_db_uri( - env=cfg["ENV"], username=os.environ.get("POSTGRES_USERNAME", "postgres"), password=os.environ.get("POSTGRES_PASSWORD", "postgres"), host=os.environ.get("POSTGRES_HOST", "127.0.0.1"), diff --git a/grabber.py b/grabber.py index 78a6549..c442df1 100644 --- a/grabber.py +++ b/grabber.py @@ -145,7 +145,7 @@ def get_latest_info_from_hosts(hosts: list[str], db: DQM2MirrorDB) -> None: """ for host in hosts: log.debug(f"Getting latest rev for {host} from DB.") - rev = db.get_rev(host) + rev = db.get_latest_revision(host) get_latest_info_from_host(host=host, rev=rev, db=db) diff --git a/tests/test_1.py b/tests/test_1.py index 6a01a55..00fe599 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -30,9 +30,7 @@ def format_entry_to_db_entry(graph_entry: list, datetime_cols: list): @pytest.fixture def testing_database(): - cfg = load_cfg() db_uri = format_db_uri( - env=cfg["ENV"], username=os.environ.get("POSTGRES_USERNAME", "postgres"), password=os.environ.get("POSTGRES_PASSWORD", "postgres"), host=os.environ.get("POSTGRES_HOST", "127.0.0.1"), diff --git a/tests/test_2.py b/tests/test_2.py index c3cb915..bffbe0c 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -31,9 +31,8 @@ def cfg(): @pytest.fixture -def testing_databases(cfg): +def testing_databases(): db_uri_prod = format_db_uri( - env=cfg["ENV"], username=os.environ.get("POSTGRES_USERNAME", "postgres"), password=os.environ.get("POSTGRES_PASSWORD", "postgres"), host=os.environ.get("POSTGRES_HOST", "127.0.0.1"), @@ -41,7 +40,6 @@ def testing_databases(cfg): db_name=os.environ.get("POSTGRES_PRODUCTION_DB_NAME") + "_test", ) db_uri_playback = format_db_uri( - env=cfg["ENV"], username=os.environ.get("POSTGRES_USERNAME", "postgres"), password=os.environ.get("POSTGRES_PASSWORD", "postgres"), host=os.environ.get("POSTGRES_HOST", "127.0.0.1"),