Skip to content

Commit

Permalink
asdfa
Browse files Browse the repository at this point in the history
  • Loading branch information
@nothingface0
nothingface0 committed Sep 6, 2023
1 parent a0fd213 commit 286e148
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 51 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- name: Test with pytest
run: |
pytest tests -s -v -k 7
pytest tests -s -v
env:
# Env vars for pytest
POSTGRES_USERNAME: postgres
Expand Down
106 changes: 70 additions & 36 deletions db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
import psycopg2
import sqlalchemy
from sqlalchemy import text
from sqlalchemy_utils import database_exists
from sqlalchemy.orm import sessionmaker
from collections import defaultdict
Expand All @@ -12,7 +13,7 @@

sys.path.append(os.path.join(os.path.dirname(__file__), "."))

from dqmsquare_cfg import TZ
from dqmsquare_cfg import TZ, TIMEZONE

DEFAULT_DATETIME = TZ.localize(datetime(2012, 3, 3, 10, 10, 10, 0))

Expand Down Expand Up @@ -89,7 +90,9 @@ def __init__(self, log: logging.Logger, db_uri: str = None, server: bool = False
)

self.Session = sessionmaker(bind=self.engine)

with self.engine.connect() as cur:
session = self.Session(bind=cur)
session.execute(text(f"SET TIMEZONE = '{TIMEZONE}';"))
if not server:
self.create_tables()
self.db_meta = sqlalchemy.MetaData(bind=self.engine)
Expand All @@ -104,23 +107,29 @@ def create_tables(self):
session = self.Session(bind=cur)
try:
session.execute(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_RUNS
+ " "
+ self.TB_DESCRIPTION_RUNS
text(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_RUNS
+ " "
+ self.TB_DESCRIPTION_RUNS
)
)
session.execute(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_GRAPHS
+ " "
+ self.TB_DESCRIPTION_GRAPHS
text(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_GRAPHS
+ " "
+ self.TB_DESCRIPTION_GRAPHS
)
)
session.execute("DROP TABLE IF EXISTS " + self.TB_NAME_META)
session.execute(text(f"DROP TABLE IF EXISTS {self.TB_NAME_META};"))
session.execute(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_META
+ " "
+ self.TB_DESCRIPTION_META
text(
"CREATE TABLE IF NOT EXISTS "
+ self.TB_NAME_META
+ " "
+ self.TB_DESCRIPTION_META
)
)
session.commit()
except psycopg2.IntegrityError as e:
Expand Down Expand Up @@ -210,7 +219,7 @@ def fill_graph(self, header: dict, document: dict) -> int:
session = self.Session(bind=cur)
try:
session.execute(
f"DELETE FROM {self.TB_NAME_GRAPHS} WHERE id = '{str(id)}';"
text(f"DELETE FROM {self.TB_NAME_GRAPHS} WHERE id = '{str(id)}';")
)
session.execute(
sqlalchemy.insert(self.db_meta.tables[self.TB_NAME_GRAPHS]).values(
Expand All @@ -232,7 +241,9 @@ def get_graphs_data(self, run: int) -> list:
self.log.debug("DQM2MirrorDB.get_graphs_data() - " + str(run))
with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT * FROM {self.TB_NAME_GRAPHS} WHERE CAST(run as INTEGER) = {str(run)};"
text(
f"SELECT * FROM {self.TB_NAME_GRAPHS} WHERE CAST(run as INTEGER) = {str(run)};"
)
).all()
if not len(answer):
return []
Expand Down Expand Up @@ -315,8 +326,9 @@ def fill_run(self, header: dict, document: dict) -> int:
with self.engine.connect() as cur:
session = self.Session(bind=cur)
try:
# cur.execute("INSERT OR REPLACE INTO " + self.TB_NAME_RUNS + " " + self.TB_DESCRIPTION_RUNS_SHORT + " VALUES " + template, values)
session.execute(f"DELETE FROM {self.TB_NAME_RUNS} WHERE id = '{id}';")
session.execute(
text(f"DELETE FROM {self.TB_NAME_RUNS} WHERE id = '{id}';")
)
session.execute(
sqlalchemy.insert(self.db_meta.tables[self.TB_NAME_RUNS]).values(
values_dic
Expand All @@ -337,13 +349,15 @@ def fill_run(self, header: dict, document: dict) -> int:
old_min_max = [999999999, -1]
with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT data FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
text(
f"SELECT data FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
)
).all()
if answer:
old_min_max = eval(answer[0][0])
else:
answer = cur.execute(
f"SELECT MIN(run), MAX(run) FROM {self.TB_NAME_RUNS};"
text(f"SELECT MIN(run), MAX(run) FROM {self.TB_NAME_RUNS};")
).all()
if answer:
old_min_max = answer[0]
Expand Down Expand Up @@ -375,13 +389,17 @@ def get(
postfix = " AND cmssw_lumi > 0 " + postfix
if run_start == run_end:
answer = cur.execute(
f"SELECT {self.TB_DESCRIPTION_RUNS_SHORT_NOLOGS} FROM {self.TB_NAME_RUNS} "
+ f"WHERE run = {run_start} {postfix} ORDER BY client, id;"
text(
f"SELECT {self.TB_DESCRIPTION_RUNS_SHORT_NOLOGS} FROM {self.TB_NAME_RUNS} "
+ f"WHERE run = {run_start} {postfix} ORDER BY client, id;"
)
).all()
else:
answer = cur.execute(
f"SELECT {self.TB_DESCRIPTION_RUNS_SHORT_NOLOGS} FROM {self.TB_NAME_RUNS} "
+ f"WHERE run BETWEEN {run_start} AND {run_end} {postfix}"
text(
f"SELECT {self.TB_DESCRIPTION_RUNS_SHORT_NOLOGS} FROM {self.TB_NAME_RUNS} "
+ f"WHERE run BETWEEN {run_start} AND {run_end} {postfix};"
)
).all()
self.log.debug(f"Read DB for runs {run_start}-{run_end}: {answer}")
for _ in answer:
Expand Down Expand Up @@ -562,8 +580,10 @@ def get_clients(self, run_start: int, run_end: int) -> list:
self.log.debug("DQM2MirrorDB.get_clients()")
with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT DISTINCT client FROM {self.TB_NAME_RUNS} "
+ f"WHERE run BETWEEN {run_start} AND {run_end} ORDER BY client;"
text(
f"SELECT DISTINCT client FROM {self.TB_NAME_RUNS} "
+ f"WHERE run BETWEEN {run_start} AND {run_end} ORDER BY client;"
)
).all()
answer = [
self.get_short_client_name(name[0])
Expand All @@ -581,11 +601,15 @@ def update_min_max(self, new_min: int, new_max: int):
session = self.Session(bind=cur)
try:
session.execute(
f"DELETE FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
text(
f"DELETE FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
)
)
session.execute(
f"INSERT INTO {self.TB_NAME_META} {self.TB_DESCRIPTION_META_SHORT} "
+ f"VALUES('min_max_runs', '[{new_min},{new_max}]');"
text(
f"INSERT INTO {self.TB_NAME_META} {self.TB_DESCRIPTION_META_SHORT} "
+ f"VALUES('min_max_runs', '[{new_min},{new_max}]');"
)
)
session.commit()
except Exception as e:
Expand All @@ -603,14 +627,16 @@ def get_info(self) -> list:

with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT data FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
text(
f"SELECT data FROM {self.TB_NAME_META} WHERE name = 'min_max_runs';"
)
).all()

if answer:
return eval(answer[0][0])

answer = cur.execute(
f"SELECT MIN(run), MAX(run) FROM {self.TB_NAME_RUNS};"
text(f"SELECT MIN(run), MAX(run) FROM {self.TB_NAME_RUNS};")
).all()
if not answer:
return [-1, -1]
Expand All @@ -631,13 +657,17 @@ def get_latest_revision(self, host: str) -> int:
with self.engine.connect() as cur:
if "fu" in host:
answer = cur.execute(
f"SELECT MAX(rev) FROM {self.TB_NAME_RUNS} WHERE hostname = '{host}';"
text(
f"SELECT MAX(rev) FROM {self.TB_NAME_RUNS} WHERE hostname = '{host}';"
)
).all()
answer = list(answer[0])
return answer[0]
else:
answer = cur.execute(
f"SELECT MAX(rev) FROM {self.TB_NAME_GRAPHS} WHERE hostname = '{host}';"
text(
f"SELECT MAX(rev) FROM {self.TB_NAME_GRAPHS} WHERE hostname = '{host}';"
)
).all()
answer = list(answer[0])
return answer[0]
Expand All @@ -646,7 +676,9 @@ def get_logs(self, client_id: int) -> list:
self.log.debug("DQM2MirrorDB.get_logs()")
with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT stdlog_start, stdlog_end FROM {self.TB_NAME_RUNS} WHERE id = '{client_id}';"
text(
f"SELECT stdlog_start, stdlog_end FROM {self.TB_NAME_RUNS} WHERE id = '{client_id}';"
)
).all()
if not answer:
answer = ["None", "None"]
Expand All @@ -662,7 +694,9 @@ def get_runs_around(self, run: int) -> list:
self.log.debug("DQM2MirrorDB.get_runs_around()")
with self.engine.connect() as cur:
answer = cur.execute(
f"SELECT min(run) FROM {self.TB_NAME_RUNS} WHERE run > {run} union SELECT max(run) FROM {self.TB_NAME_RUNS} WHERE run < {run};"
text(
f"SELECT min(run) FROM {self.TB_NAME_RUNS} WHERE run > {run} union SELECT max(run) FROM {self.TB_NAME_RUNS} WHERE run < {run};"
)
).all()
answer = [item[0] for item in answer]
return answer
5 changes: 3 additions & 2 deletions dqmsquare_cfg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
# Important for converting datetime objects (from the database)
# to timestamps. Github actions, for example, run in different timezones,
# leading to different timestamps and failing tests.
TZ = pytz.timezone("Europe/Zurich")
TIMEZONE = "Europe/Zurich"
TZ = pytz.timezone(TIMEZONE)


def format_db_uri(
Expand Down Expand Up @@ -147,7 +148,7 @@ def load_cfg() -> dict:
port=os.environ.get("POSTGRES_PORT", 5432),
db_name=os.environ.get("POSTGRES_PRODUCTION_DB_NAME", "postgres_production"),
)

cfg["TIMEZONE"] = TIMEZONE
return cfg


Expand Down
1 change: 1 addition & 0 deletions server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def greet():
PREFIX=os.path.join("/", cfg["SERVER_URL_PREFIX"] + "/"),
FRONTEND_API_QUERY_INTERVAL=cfg["FRONTEND_API_QUERY_INTERVAL"],
VERSION=cfg["VERSION"],
TIMEZONE=cfg["TIMEZONE"],
)

@app.route(os.path.join("/", cfg["SERVER_URL_PREFIX"], "static/<path:filename>"))
Expand Down
2 changes: 1 addition & 1 deletion templates/runs.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
for (let client_data of clients_data) {
tr = document.createElement("tr");
// Timestamp that client was running
client_data[0] = new Date(client_data[0]).toLocaleString('en-GB', { hour12: false, timeZone: 'Europe/Zurich' });
client_data[0] = new Date(client_data[0]).toLocaleString('en-GB', { hour12: false, timeZone: '{{ TIMEZONE }}' });
// console.log(client_data)
if (client_data[9] < 0) client_data[9] = 0
// Event process speed
Expand Down
22 changes: 14 additions & 8 deletions tests/test_1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from truth_values import TEST_DB_7_TRUTH, TEST_DB_9_TRUTH
from datetime import datetime
from db import DQM2MirrorDB, DEFAULT_DATETIME
from sqlalchemy import create_engine, insert
from sqlalchemy import create_engine, text
from sqlalchemy_utils import create_database, database_exists, drop_database
from custom_logger import dummy_log
from dqmsquare_cfg import format_db_uri, TZ
Expand Down Expand Up @@ -64,7 +64,9 @@ def testing_database() -> DQM2MirrorDB:
for run in runs:
try:
session.execute(
f"""INSERT into runs ({str(db.TB_DESCRIPTION_RUNS_SHORT).replace("[", "").replace("]", "").replace("'", "")}) VALUES ({format_entry_to_db_entry(run, [13])})"""
text(
f"""INSERT into runs ({str(db.TB_DESCRIPTION_RUNS_SHORT).replace("[", "").replace("]", "").replace("'", "")}) VALUES ({format_entry_to_db_entry(run, [13])})"""
)
)
session.commit()
except Exception as e:
Expand All @@ -74,7 +76,9 @@ def testing_database() -> DQM2MirrorDB:
for graph in graphs:
try:
session.execute(
f"""INSERT into graphs ({str(db.TB_DESCRIPTION_GRAPHS_SHORT).replace("[", "").replace("]", "").replace("'", "")}) VALUES ({format_entry_to_db_entry(graph, [3, 4])})"""
text(
f"""INSERT into graphs ({str(db.TB_DESCRIPTION_GRAPHS_SHORT).replace("[", "").replace("]", "").replace("'", "")}) VALUES ({format_entry_to_db_entry(graph, [3, 4])})"""
)
)
session.commit()
except Exception as e:
Expand Down Expand Up @@ -282,11 +286,13 @@ def test_db_11(testing_database: DQM2MirrorDB):
with testing_database.engine.connect() as cur:
session = testing_database.Session(bind=cur)
session.execute(
"DELETE FROM "
+ testing_database.TB_NAME_RUNS
+ " WHERE run = "
+ str(123456)
+ ""
text(
"DELETE FROM "
+ testing_database.TB_NAME_RUNS
+ " WHERE run = "
+ str(123456)
+ ";"
)
)
session.commit()

Expand Down
6 changes: 3 additions & 3 deletions tests/test_2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ def testing_databases():


@pytest.fixture
def app(cfg, testing_databases):
def app(cfg, testing_databases: list[DQM2MirrorDB]):
cfg_test = cfg
# Override databases for the test
cfg_test["DB_PRODUCTION_URI"] = testing_databases[0].db_str
cfg_test["DB_PLAYBACK_URI"] = testing_databases[1].db_str
cfg_test["DB_PRODUCTION_URI"] = testing_databases[0].db_uri
cfg_test["DB_PLAYBACK_URI"] = testing_databases[1].db_uri
print(cfg_test["DB_PRODUCTION_URI"], cfg_test["DB_PLAYBACK_URI"])

app = server.create_app(cfg_test)
Expand Down

0 comments on commit 286e148

Please sign in to comment.