From 887b4e38ceb8be8b25b99f986b60a755d18f25a9 Mon Sep 17 00:00:00 2001 From: Ali Sheikhi Date: Wed, 5 Feb 2025 17:22:55 +0100 Subject: [PATCH 1/2] DD-1763 Voeg --age parameter toe aan deposit-clean-data --- src/datastation/deposit_clean_data.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/datastation/deposit_clean_data.py b/src/datastation/deposit_clean_data.py index 2cbfc47..8594a78 100644 --- a/src/datastation/deposit_clean_data.py +++ b/src/datastation/deposit_clean_data.py @@ -1,4 +1,6 @@ import argparse +from datetime import date, timedelta + from datastation.managedeposit.manage_deposit import ManageDeposit from datastation.common.config import init @@ -12,12 +14,19 @@ def clean_manage_deposit_data(server_url, args): def main(): config = init() parser = argparse.ArgumentParser(prog='deposit_data_cleaner', description='Clean up dd-manage-deposit database') - parser.add_argument('-e', '--enddate', dest='enddate', help='Filter until the record creation of this date') parser.add_argument('-s', '--startdate', dest='startdate', help='Filter from the record creation of this date') - parser.add_argument('-t', '--state', dest='state', help='The state of the deposit') - parser.add_argument('-u', '--user', dest='user', help='The depositor name') + + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-a', '--age', dest='age', type=int, help='Filter from record creation older than a number of days before today') + group.add_argument('-e', '--enddate', dest='enddate', help='Filter until the record creation of this date') + + parser.add_argument('-t', '--state', dest='state', help='The state of the deposit (repeatable)', action='append') + parser.add_argument('-u', '--user', dest='user', help='The depositor name (repeatable)', action='append') args = parser.parse_args() + if args.age is not None: # Note: args is a Namespace object + vars(args)['enddate'] = (date.today() + timedelta(days=-args.age)).strftime('%Y-%m-%d') + server_url = config['manage_deposit']['service_baseurl'] + '/delete-deposit' clean_manage_deposit_data(server_url, args) From 4f4dc79b0699aa457ca56fafef03d42b750a077d Mon Sep 17 00:00:00 2001 From: Ali Sheikhi Date: Tue, 4 Mar 2025 16:23:06 +0100 Subject: [PATCH 2/2] DD-1763 add age parameter to the script deposit-clean-data --- src/datastation/deposit_clean_data.py | 47 +++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/src/datastation/deposit_clean_data.py b/src/datastation/deposit_clean_data.py index 8594a78..37e9e2c 100644 --- a/src/datastation/deposit_clean_data.py +++ b/src/datastation/deposit_clean_data.py @@ -1,23 +1,56 @@ import argparse +import csv from datetime import date, timedelta +import io +import os +import shutil from datastation.managedeposit.manage_deposit import ManageDeposit from datastation.common.config import init -def clean_manage_deposit_data(server_url, args): - result = ManageDeposit(args).clean_data(server_url) - if result is not None: - print(result) +class CleanHandler: + def __init__(self, server_url, cmd_args): + self.__deposit_folder_index_in_csv = 1 + self.__deposit_location_index_in_csv = 7 + self.__server_url = server_url + self.__command_line_args = cmd_args + + def handle_request(self): + report = ManageDeposit(self.__command_line_args).create_report(self.__server_url) + + if report is not None and len(report.split('\n')) > 1: + self.remove_folders(self.collect_paths(report)) + else: + print("deposit_clean_data: report is empty.") + + def collect_paths(self, csv_data): + paths = [] + csv_mem = io.StringIO(csv_data) + csv_reader = csv.reader(csv_mem) + for row in csv_reader: + if len(row) > max(self.__deposit_folder_index_in_csv, self.__deposit_location_index_in_csv): + paths.append(row[self.__deposit_location_index_in_csv] + "/" + row[self.__deposit_folder_index_in_csv]) + + return paths + + def remove_folders(self, paths): + for path in paths: + if os.path.exists(path) and os.path.isdir(path): + shutil.rmtree(path) + print(f"Folder '{path}' has been removed successfully.") + else: + print(f"Folder '{path}' does not exist.") def main(): config = init() parser = argparse.ArgumentParser(prog='deposit_data_cleaner', description='Clean up dd-manage-deposit database') + parser.add_argument('-f', '--format', dest='file_format', default='text/csv', help='Output data format') parser.add_argument('-s', '--startdate', dest='startdate', help='Filter from the record creation of this date') group = parser.add_mutually_exclusive_group(required=False) - group.add_argument('-a', '--age', dest='age', type=int, help='Filter from record creation older than a number of days before today') + group.add_argument('-a', '--age', dest='age', type=int, help='Filter records older than a number of days before today') group.add_argument('-e', '--enddate', dest='enddate', help='Filter until the record creation of this date') parser.add_argument('-t', '--state', dest='state', help='The state of the deposit (repeatable)', action='append') @@ -27,9 +60,9 @@ def main(): if args.age is not None: # Note: args is a Namespace object vars(args)['enddate'] = (date.today() + timedelta(days=-args.age)).strftime('%Y-%m-%d') - server_url = config['manage_deposit']['service_baseurl'] + '/delete-deposit' + server_url = config['manage_deposit']['service_baseurl'] + '/report' - clean_manage_deposit_data(server_url, args) + CleanHandler(server_url, args).handle_request() if __name__ == '__main__':