diff --git a/.gitignore b/.gitignore index 66cbf978..a230e760 100644 --- a/.gitignore +++ b/.gitignore @@ -41,8 +41,4 @@ firebase-debug.log yarn.lock .env .env.* -firebase_to_xml/build -output - - -*.log +*.log \ No newline at end of file diff --git a/firebase_to_xml/firebase_to_xml/__main__.py b/firebase_to_xml/firebase_to_xml/__main__.py index 6861a6a4..aae5768e 100644 --- a/firebase_to_xml/firebase_to_xml/__main__.py +++ b/firebase_to_xml/firebase_to_xml/__main__.py @@ -6,18 +6,23 @@ import argparse import traceback from pathlib import Path -import os import yaml from dotenv import load_dotenv from metadata_xml.template_functions import metadata_to_xml -from loguru import logger -from tqdm import tqdm -from get_records_from_firebase import get_records_from_firebase -from record_json_to_yaml import record_json_to_yaml +from firebase_to_xml.get_records_from_firebase import get_records_from_firebase +from firebase_to_xml.record_json_to_yaml import record_json_to_yaml +def parse_status(status: str): + """Return a list version fo their status selection""" + + if "," in status: + return status.split(",") + + return [status] + def get_filename(record): """Creates a filename by combinig the title and UUID """ @@ -58,46 +63,66 @@ def main(): ], ) parser.add_argument("--record_url", required=False) - parser.add_argument("--database_url", default=os.getenv("DATABASE_URL"), required=False, help="Firebase database URL (default: %(default)s)") + args = vars(parser.parse_args()) + region = args["region"] + record_status = parse_status(args["status"]) + record_url = args["record_url"] + + firebase_auth_key_file = args["key"] also_save_yaml = args["yaml"] # get list of records from Firebase record_list = get_records_from_firebase( - args["region"], args["key"], record_url, args["status"].split(','), args["database_url"] + region, firebase_auth_key_file, record_url, record_status ) # translate each record to YAML and then to XML - for record in tqdm(record_list, desc="Convert records", unit="record"): + for record in record_list: # if single record it uses std out, hide info + if not record_url: + print( + "Processing", + f"'{record['title']['en']}'", + f"'{record['title']['fr']}'", + record["identifier"], + record["recordID"], + "\n", + ) + try: record_yaml = record_json_to_yaml(record) organization = record.get("organization", "") - filename = record.get("filename") or get_filename(record) + name = record.get("filename") or get_filename(record) + + xml_directory = args["out"] - output_directory = Path(args["out"]) / organization - output_directory.mkdir(parents=True, exist_ok=True) + xml_directory = "/".join([args["out"], organization]) + + Path(xml_directory).mkdir(parents=True, exist_ok=True) # output yaml if also_save_yaml: - yaml_file = output_directory / f"{filename}.yaml" - yaml_file.write_text(yaml.dump(record_yaml, allow_unicode=True, sort_keys=False), encoding="utf-8") + filename = f"{xml_directory}/{name}.yaml" + file = open(filename, "w") + file.write(yaml.dump(record_yaml, allow_unicode=True, sort_keys=False)) # render xml template and write to file xml = metadata_to_xml(record_yaml) if record_url: - logger.info(xml) - continue - - xml_file = output_directory / f"{filename}.xml" - xml_file.write_text(xml, encoding="utf-8") + print(xml) + else: + filename = f"{xml_directory}/{name}.xml" + file = open(filename, "w") + file.write(xml) + print("Wrote " + file.name) except Exception: - logger.error(traceback.format_exc()) + print(traceback.format_exc()) if __name__ == "__main__": diff --git a/firebase_to_xml/firebase_to_xml/get_records_from_firebase.py b/firebase_to_xml/firebase_to_xml/get_records_from_firebase.py index 3aa26a73..a11ee807 100644 --- a/firebase_to_xml/firebase_to_xml/get_records_from_firebase.py +++ b/firebase_to_xml/firebase_to_xml/get_records_from_firebase.py @@ -5,13 +5,15 @@ """ import json +import pprint import sys from google.auth.transport.requests import AuthorizedSession from google.oauth2 import service_account + def get_records_from_firebase( - region:str, firebase_auth_key_file:str, record_url:str, record_status:list, database_url:str, firebase_auth_key_json:str=None + region, firebase_auth_key_file, record_url, record_status, firebase_auth_key_json=None ): """ Returns list of records from firebase for this region, @@ -42,7 +44,7 @@ def get_records_from_firebase( if record_url: response = authed_session.get( - f"{database_url}{record_url}.json" + f"https://cioos-metadata-form.firebaseio.com/{record_url}.json" ) body = json.loads(response.text) records.append(body) @@ -50,12 +52,12 @@ def get_records_from_firebase( else: response = authed_session.get( - f"{database_url}{region}/users.json" + f"https://cioos-metadata-form.firebaseio.com/{region}/users.json" ) body = json.loads(response.text) # Parse response - if not body or not isinstance(body, dict) : + if not body or type(body) != dict : print("Region",region,"not found?") # print(response.content) sys.exit() diff --git a/firebase_to_xml/pyproject.toml b/firebase_to_xml/pyproject.toml deleted file mode 100644 index 2262d7ca..00000000 --- a/firebase_to_xml/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "firebase-to-xml" -version = "0.1.0" -description = "Python module translates metadata records from Firebase into XML" -readme = "README.md" -requires-python = ">=3.12" -dependencies = [ - "firebase-to-xml", - "google-auth>=2.35.0", - "google-oauth>=1.0.1", - "loguru>=0.7.2", - "metadata-xml", - "python-dotenv>=1.0.1", - "tqdm>=4.66.5", -] - -[tool.uv.sources] -metadata-xml = { git = "https://github.com/cioos-siooc/metadata-xml.git" } -firebase-to-xml = { workspace = true }