From 7ec3b1e7b23f4baaecd995993766adc5d66c2be4 Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Wed, 1 Nov 2023 00:22:06 -0400 Subject: [PATCH 1/3] OD-1731: Remove unused imports --- efiction/chapters.py | 2 +- efiction/metadata.py | 1 - efiction/original.py | 1 - efiction/simplified.py | 1 - efiction/tests/test_chapters.py | 4 +--- efiction/tests/test_metadata.py | 6 +----- efiction/tests/test_original.py | 3 --- opendoors/utils.py | 3 +-- steps/tests/test_step_02.py | 3 +-- 9 files changed, 5 insertions(+), 19 deletions(-) diff --git a/efiction/chapters.py b/efiction/chapters.py index c93ae8a..8645977 100644 --- a/efiction/chapters.py +++ b/efiction/chapters.py @@ -96,7 +96,7 @@ def load_chapter_text_into_db(self, chapter_paths: List[dict]): # check if encoding is valid ''.encode(encoding_text) encoding = encoding_text - except: + except LookupError: print(f"{encoding_text} is not a valid encoding, try again") for old_chapter in old_chapters: chapid = old_chapter['chapid'] diff --git a/efiction/metadata.py b/efiction/metadata.py index 7b323d5..b9c4521 100644 --- a/efiction/metadata.py +++ b/efiction/metadata.py @@ -1,4 +1,3 @@ -import os import re from configparser import ConfigParser from logging import Logger diff --git a/efiction/original.py b/efiction/original.py index 6861908..d2fff47 100644 --- a/efiction/original.py +++ b/efiction/original.py @@ -1,7 +1,6 @@ """ Step 01 """ -import os from configparser import ConfigParser from logging import Logger diff --git a/efiction/simplified.py b/efiction/simplified.py index b979797..d3b0d53 100644 --- a/efiction/simplified.py +++ b/efiction/simplified.py @@ -1,4 +1,3 @@ -import os import re from configparser import ConfigParser from logging import Logger diff --git a/efiction/tests/test_chapters.py b/efiction/tests/test_chapters.py index 9deabf5..48cd202 100644 --- a/efiction/tests/test_chapters.py +++ b/efiction/tests/test_chapters.py @@ -5,10 +5,8 @@ from efiction.chapters import EFictionChapters -from opendoors.big_insert import BigInsert from opendoors.config import ArchiveConfig -from opendoors.mysql import SqlDb -from opendoors.utils import get_full_path, normalize, remove_output_files +from opendoors.utils import get_full_path, normalize def get_data(): diff --git a/efiction/tests/test_metadata.py b/efiction/tests/test_metadata.py index 26c87a6..d6e7f30 100644 --- a/efiction/tests/test_metadata.py +++ b/efiction/tests/test_metadata.py @@ -1,12 +1,8 @@ import datetime from unittest import TestCase -from unittest.mock import MagicMock -from efiction.metadata import EFictionMetadata from efiction.tests.test_utils import load_fixtures, create_efiction_converter -from opendoors.config import ArchiveConfig -from opendoors.mysql import SqlDb -from opendoors.utils import get_full_path, remove_output_files +from opendoors.utils import remove_output_files class TestEFictionConverter(TestCase): diff --git a/efiction/tests/test_original.py b/efiction/tests/test_original.py index f1a3f6a..761228b 100644 --- a/efiction/tests/test_original.py +++ b/efiction/tests/test_original.py @@ -1,6 +1,3 @@ -import glob -import os -import re from unittest import TestCase from unittest.mock import MagicMock, patch diff --git a/opendoors/utils.py b/opendoors/utils.py index 8fdf637..d569fe6 100644 --- a/opendoors/utils.py +++ b/opendoors/utils.py @@ -6,7 +6,6 @@ import os import re import shutil -import sys from typing import Mapping from pathlib import Path @@ -161,4 +160,4 @@ def get_prefixed_path(step: str, path: str, filename: str=""): if filename: return os.path.join(path, f"{prefix}-{filename}") else: - return os.path.join(path, prefix) \ No newline at end of file + return os.path.join(path, prefix) diff --git a/steps/tests/test_step_02.py b/steps/tests/test_step_02.py index eae9e4b..76e7470 100644 --- a/steps/tests/test_step_02.py +++ b/steps/tests/test_step_02.py @@ -4,11 +4,10 @@ import shutil from pathlib import Path from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from opendoors.config import ArchiveConfig from opendoors.step_base import StepInfo -from steps.step_01 import Step01 from steps.step_02 import Step02 test_logger = MagicMock() From 94aa44ea57c55836875673f31e15f5b3151d8c16 Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Wed, 1 Nov 2023 00:23:54 -0400 Subject: [PATCH 2/3] OD-1731: Fix f-strings without placeholders --- efiction/metadata.py | 6 +++--- efiction/tag_converter.py | 2 +- opendoors/progress.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/efiction/metadata.py b/efiction/metadata.py index b9c4521..58ccdbd 100644 --- a/efiction/metadata.py +++ b/efiction/metadata.py @@ -151,7 +151,7 @@ def _convert_story_tags(self, old_story): def _convert_tags_join(self, new_story, tags, sql=None): # Support using non-default sql connection for multithreaded workloads sql = self.sql if sql is None else sql - full_query = f"INSERT INTO item_tags (item_id, item_type, tag_id) VALUES " + full_query = "INSERT INTO item_tags (item_id, item_type, tag_id) VALUES " tag_query = [] for tag_list in tags.values(): for tag in tag_list: @@ -218,12 +218,12 @@ def story_processor(old_story): """ sql.execute(self.working_open_doors, query) - self.logger.debug(f" tags...") + self.logger.debug(" tags...") tags = self._convert_story_tags(old_story) # pass the new sql to be used instead of the main one self._convert_tags_join(new_story, tags, sql) - self.logger.debug(f" authors...") + self.logger.debug(" authors...") self._convert_author_join(new_story, old_story['uid'], sql) # Find if there are any coauthors for the work coauthors = self.fetch_coauthors(new_story, sql) diff --git a/efiction/tag_converter.py b/efiction/tag_converter.py index b265a12..593e478 100644 --- a/efiction/tag_converter.py +++ b/efiction/tag_converter.py @@ -29,7 +29,7 @@ def check_for_nonstandard_tag_tables(self) -> bool: if tag_table_name == 'rating': # Only one rating per story, so story rating should be single number # that exactly matches rating id - query = f"SELECT count(*) as cnt FROM stories WHERE rid NOT IN (SELECT rid FROM ratings);" + query = "SELECT count(*) as cnt FROM stories WHERE rid NOT IN (SELECT rid FROM ratings);" count: List[Dict[str, int]] = self.sql.execute_and_fetchall(self.working_original, query) tag_tables['rating'] = bool(count and count[0]['cnt'] > 0) else: diff --git a/opendoors/progress.py b/opendoors/progress.py index 5146577..c6a5f15 100644 --- a/opendoors/progress.py +++ b/opendoors/progress.py @@ -29,7 +29,7 @@ def continue_from_last(config: ConfigParser, logger: Logger, sql: SqlDb, steps: next_step = config['Processing']['next_step'] = step.next_step update_done_steps(config, done_steps, step_to_run) else: - restart_yn = input(f"All steps have been completed for this archive. Do you want to\n" + restart_yn = input("All steps have been completed for this archive. Do you want to\n" "1. Restart from step 1\n" "2. Exit (default - press Enter)\n>> ") if restart_yn == "1": From 564f4530446415c0168e221207daf93f0c18ae16 Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Wed, 1 Nov 2023 00:24:50 -0400 Subject: [PATCH 3/3] OD-1731: Don't store exceptions if we're not going to use them --- efiction/metadata.py | 2 +- efiction/tag_converter.py | 2 +- opendoors/mysql.py | 2 +- opendoors/progress.py | 2 +- opendoors/utils.py | 2 +- steps/tests/test_step_01.py | 2 +- steps/tests/test_step_02.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/efiction/metadata.py b/efiction/metadata.py index 58ccdbd..ae277d3 100644 --- a/efiction/metadata.py +++ b/efiction/metadata.py @@ -175,7 +175,7 @@ def fetch_coauthors(self, new_story, sql=None): # get a dict of coauthor IDs for the story try: authors = sql.execute_and_fetchall(self.working_original, full_query) - except Exception as e: + except Exception: authors = None self.logger.info("No coauthors table...") # We only try to operate on this result if it is not None diff --git a/efiction/tag_converter.py b/efiction/tag_converter.py index 593e478..e328ce8 100644 --- a/efiction/tag_converter.py +++ b/efiction/tag_converter.py @@ -53,7 +53,7 @@ def check_for_nonstandard_tag_tables(self) -> bool: tags = list(map(lambda story_tags: story_tags[id_name].replace(',', ''), tags)) int(''.join(tags)) tag_tables[tag_table_name] = False - except Exception as e: + except Exception: # Non-integer in identifier tag_tables[tag_table_name] = True except Exception as e: diff --git a/opendoors/mysql.py b/opendoors/mysql.py index 193ef99..ed67c1c 100644 --- a/opendoors/mysql.py +++ b/opendoors/mysql.py @@ -73,7 +73,7 @@ def read_table_to_dict(self, database: str, tablename: str): try: cursor.execute(f"SELECT * FROM {database}.{tablename};") return cursor.fetchall() - except Exception as e: + except Exception: self.logger.info(f"No table {tablename} in {database}...") return [] diff --git a/opendoors/progress.py b/opendoors/progress.py index c6a5f15..15daa08 100644 --- a/opendoors/progress.py +++ b/opendoors/progress.py @@ -36,7 +36,7 @@ def continue_from_last(config: ConfigParser, logger: Logger, sql: SqlDb, steps: next_step = "01" else: run_next = False - except Exception as e: + except Exception: logger.error(traceback.format_exc()) diff --git a/opendoors/utils.py b/opendoors/utils.py index d569fe6..a950f1c 100644 --- a/opendoors/utils.py +++ b/opendoors/utils.py @@ -135,7 +135,7 @@ def remove_output_files(path: str): shutil.rmtree(file) else: os.remove(file) - except PermissionError as pe: + except PermissionError: # We don't necessarily care that much continue diff --git a/steps/tests/test_step_01.py b/steps/tests/test_step_01.py index b59d843..40ea1fb 100644 --- a/steps/tests/test_step_01.py +++ b/steps/tests/test_step_01.py @@ -25,7 +25,7 @@ def tearDown(self) -> None: shutil.rmtree(file) else: os.remove(file) - except PermissionError as pe: + except PermissionError: # We don't necessarily care that much continue diff --git a/steps/tests/test_step_02.py b/steps/tests/test_step_02.py index 76e7470..b51064a 100644 --- a/steps/tests/test_step_02.py +++ b/steps/tests/test_step_02.py @@ -25,7 +25,7 @@ def tearDown(self) -> None: shutil.rmtree(file) else: os.remove(file) - except PermissionError as pe: + except PermissionError: # We don't necessarily care that much continue