From c85165e6f7272722f0cc8ccb5bed8ca3760135ab Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Mon, 19 Feb 2024 09:36:37 +0100 Subject: [PATCH 01/16] replaced pymongo with mongomock --- fireworks/core/launchpad.py | 6 +++++- fireworks/core/tests/test_launchpad.py | 2 +- fireworks/utilities/filepad.py | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index 6eda6c18d..829b75802 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -15,8 +15,10 @@ from bson import ObjectId from monty.os.path import zpath from monty.serialization import loadfn -from pymongo import ASCENDING, DESCENDING, MongoClient +from pymongo import ASCENDING, DESCENDING from pymongo.errors import DocumentTooLarge +from mongomock import MongoClient +import mongomock.gridfs from tqdm import tqdm from fireworks.core.firework import Firework, FWAction, Launch, Tracker, Workflow @@ -43,6 +45,8 @@ # TODO: lots of duplication reduction and cleanup possible +mongomock.gridfs.enable_gridfs_integration() + def sort_aggregation(sort): """Build sorting aggregation pipeline. diff --git a/fireworks/core/tests/test_launchpad.py b/fireworks/core/tests/test_launchpad.py index c6c614a39..d5ddc96db 100644 --- a/fireworks/core/tests/test_launchpad.py +++ b/fireworks/core/tests/test_launchpad.py @@ -15,7 +15,7 @@ import pytest from monty.os import cd -from pymongo import MongoClient +from mongomock import MongoClient from pymongo import __version__ as PYMONGO_VERSION from pymongo.errors import OperationFailure diff --git a/fireworks/utilities/filepad.py b/fireworks/utilities/filepad.py index e63e8ad85..1c32a65b4 100644 --- a/fireworks/utilities/filepad.py +++ b/fireworks/utilities/filepad.py @@ -10,7 +10,9 @@ import pymongo from monty.json import MSONable from monty.serialization import loadfn -from pymongo import MongoClient + +import mongomock.gridfs +from mongomock import MongoClient from fireworks.fw_config import LAUNCHPAD_LOC, MONGO_SOCKET_TIMEOUT_MS from fireworks.utilities.fw_utilities import get_fw_logger @@ -19,6 +21,8 @@ __email__ = "kmathew@lbl.gov" __credits__ = "Anubhav Jain" +mongomock.gridfs.enable_gridfs_integration() + class FilePad(MSONable): def __init__( From a178e9fa50a038d1851d08ddef2e66e1ad6cf7a5 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Tue, 20 Feb 2024 09:35:22 +0100 Subject: [PATCH 02/16] added mongomock@persistence to install_requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 309074f13..23b9acd86 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ "tqdm>=4.8.4", "importlib-metadata>=4.8.2; python_version<'3.8'", "typing-extensions; python_version<'3.8'", + "mongomock @ git+https://github.com/ikondov/mongomock.git@persistence", ], extras_require={ "rtransfer": ["paramiko>=2.4.2"], From a4af47cce8adaaf10d5f3647a9fa4a0acb61c766 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Tue, 20 Feb 2024 13:49:00 +0100 Subject: [PATCH 03/16] made use of mongomock conditional on a configuration variable --- docs_rst/config_tutorial.rst | 1 + fireworks/core/launchpad.py | 18 +++++++++++++----- fireworks/fw_config.py | 3 +++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs_rst/config_tutorial.rst b/docs_rst/config_tutorial.rst index 291aed717..366a326fa 100644 --- a/docs_rst/config_tutorial.rst +++ b/docs_rst/config_tutorial.rst @@ -76,6 +76,7 @@ A few basic parameters that can be tweaked are: * ``WEBSERVER_HOST: 127.0.0.1`` - the default host on which to run the web server * ``WEBSERVER_PORT: 5000`` - the default port on which to run the web server * ``QUEUE_JOBNAME_MAXLEN: 20`` - the max length of the job name to send to the queuing system (some queuing systems limit the size of job names) +* ``MONGOMOCK_SERVERSTORE_FILE`` - path to a non-empty JSON file, if set then mongomock will be used instead of MongoDB; this file should be initialized with '{}' Parameters that you probably shouldn't change --------------------------------------------- diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index 829b75802..02e00deab 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -15,9 +15,10 @@ from bson import ObjectId from monty.os.path import zpath from monty.serialization import loadfn +import pymongo from pymongo import ASCENDING, DESCENDING from pymongo.errors import DocumentTooLarge -from mongomock import MongoClient +import mongomock import mongomock.gridfs from tqdm import tqdm @@ -27,6 +28,7 @@ LAUNCHPAD_LOC, MAINTAIN_INTERVAL, MONGO_SOCKET_TIMEOUT_MS, + MONGOMOCK_SERVERSTORE_FILE, RESERVATION_EXPIRATION_SECS, RUN_EXPIRATION_SECS, SORT_FWS, @@ -45,8 +47,6 @@ # TODO: lots of duplication reduction and cleanup possible -mongomock.gridfs.enable_gridfs_integration() - def sort_aggregation(sort): """Build sorting aggregation pipeline. @@ -199,14 +199,22 @@ def __init__( self.user_indices = user_indices if user_indices else [] self.wf_user_indices = wf_user_indices if wf_user_indices else [] + if MONGOMOCK_SERVERSTORE_FILE: + os.environ['MONGOMOCK_SERVERSTORE_FILE'] = MONGOMOCK_SERVERSTORE_FILE + mongoclient_cls = getattr(mongomock, 'MongoClient') + if GRIDFS_FALLBACK_COLLECTION: + mongomock.gridfs.enable_gridfs_integration() + else: + mongoclient_cls = getattr(pymongo, 'MongoClient') + # get connection if uri_mode: - self.connection = MongoClient(host, **self.mongoclient_kwargs) + self.connection = mongoclient_cls(host, **self.mongoclient_kwargs) if self.name is None: raise ValueError("Must specify a database name when using a MongoDB URI string.") self.db = self.connection[self.name] else: - self.connection = MongoClient( + self.connection = mongoclient_cls( self.host, self.port, socketTimeoutMS=MONGO_SOCKET_TIMEOUT_MS, diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py index f5f243660..768c3f14e 100644 --- a/fireworks/fw_config.py +++ b/fireworks/fw_config.py @@ -102,6 +102,9 @@ # a dynamically generated document exceeds the 16MB limit. Functionality disabled if None. GRIDFS_FALLBACK_COLLECTION = "fw_gridfs" +# path to a database file to use with mongomock, do not use mongomock if None +MONGOMOCK_SERVERSTORE_FILE = None + def override_user_settings() -> None: module_dir = os.path.dirname(os.path.abspath(__file__)) From c56dad022c4055b0fcef1b70fc5a006e967276f6 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Tue, 20 Feb 2024 16:11:13 +0100 Subject: [PATCH 04/16] moved MongoClient to fw_config.py; pylint & flakes --- fireworks/core/launchpad.py | 19 ++++--------------- fireworks/core/tests/test_launchpad.py | 3 +-- fireworks/fw_config.py | 15 +++++++++++++++ .../user_objects/firetasks/filepad_tasks.py | 16 ++++++---------- fireworks/utilities/filepad.py | 13 ++++--------- 5 files changed, 30 insertions(+), 36 deletions(-) diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index 02e00deab..b08cc5f76 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -15,20 +15,17 @@ from bson import ObjectId from monty.os.path import zpath from monty.serialization import loadfn -import pymongo from pymongo import ASCENDING, DESCENDING from pymongo.errors import DocumentTooLarge -import mongomock -import mongomock.gridfs from tqdm import tqdm from fireworks.core.firework import Firework, FWAction, Launch, Tracker, Workflow +from fireworks.fw_config import MongoClient from fireworks.fw_config import ( GRIDFS_FALLBACK_COLLECTION, LAUNCHPAD_LOC, MAINTAIN_INTERVAL, MONGO_SOCKET_TIMEOUT_MS, - MONGOMOCK_SERVERSTORE_FILE, RESERVATION_EXPIRATION_SECS, RUN_EXPIRATION_SECS, SORT_FWS, @@ -199,22 +196,14 @@ def __init__( self.user_indices = user_indices if user_indices else [] self.wf_user_indices = wf_user_indices if wf_user_indices else [] - if MONGOMOCK_SERVERSTORE_FILE: - os.environ['MONGOMOCK_SERVERSTORE_FILE'] = MONGOMOCK_SERVERSTORE_FILE - mongoclient_cls = getattr(mongomock, 'MongoClient') - if GRIDFS_FALLBACK_COLLECTION: - mongomock.gridfs.enable_gridfs_integration() - else: - mongoclient_cls = getattr(pymongo, 'MongoClient') - # get connection if uri_mode: - self.connection = mongoclient_cls(host, **self.mongoclient_kwargs) + self.connection = MongoClient(host, **self.mongoclient_kwargs) if self.name is None: raise ValueError("Must specify a database name when using a MongoDB URI string.") self.db = self.connection[self.name] else: - self.connection = mongoclient_cls( + self.connection = MongoClient( self.host, self.port, socketTimeoutMS=MONGO_SOCKET_TIMEOUT_MS, @@ -423,7 +412,7 @@ def bulk_add_wfs(self, wfs): """ # Make all fireworks workflows - wfs = [Workflow.from_firework(wf) if isinstance(wf, Firework) else wf for wf in wfs] + wfs = [Workflow.from_Firework(wf) if isinstance(wf, Firework) else wf for wf in wfs] # Initialize new firework counter, starting from the next fw id total_num_fws = sum(len(wf) for wf in wfs) diff --git a/fireworks/core/tests/test_launchpad.py b/fireworks/core/tests/test_launchpad.py index d5ddc96db..78161e2b9 100644 --- a/fireworks/core/tests/test_launchpad.py +++ b/fireworks/core/tests/test_launchpad.py @@ -15,7 +15,6 @@ import pytest from monty.os import cd -from mongomock import MongoClient from pymongo import __version__ as PYMONGO_VERSION from pymongo.errors import OperationFailure @@ -43,7 +42,7 @@ class AuthenticationTest(unittest.TestCase): @classmethod def setUpClass(cls): try: - client = MongoClient() + client = fireworks.fw_config.MongoClient() client.not_the_admin_db.command("createUser", "myuser", pwd="mypassword", roles=["dbOwner"]) except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py index 768c3f14e..ebd77d59d 100644 --- a/fireworks/fw_config.py +++ b/fireworks/fw_config.py @@ -5,6 +5,9 @@ from monty.design_patterns import singleton from monty.serialization import dumpfn, loadfn +import pymongo +import mongomock +import mongomock.gridfs __author__ = "Anubhav Jain" __copyright__ = "Copyright 2012, The Materials Project" @@ -105,6 +108,9 @@ # path to a database file to use with mongomock, do not use mongomock if None MONGOMOCK_SERVERSTORE_FILE = None +# default mongoclient class +MongoClient = pymongo.MongoClient + def override_user_settings() -> None: module_dir = os.path.dirname(os.path.abspath(__file__)) @@ -156,6 +162,15 @@ def override_user_settings() -> None: if len(m_paths) > 0: globals()[k] = m_paths[0] + if 'MONGOMOCK_SERVERSTORE_FILE' in os.environ: + globals()['MONGOMOCK_SERVERSTORE_FILE'] = os.environ['MONGOMOCK_SERVERSTORE_FILE'] + if globals()['MONGOMOCK_SERVERSTORE_FILE']: + if not os.environ.get('MONGOMOCK_SERVERSTORE_FILE'): + os.environ['MONGOMOCK_SERVERSTORE_FILE'] = globals()['MONGOMOCK_SERVERSTORE_FILE'] + globals()['MongoClient'] = getattr(mongomock, 'MongoClient') + if globals()['GRIDFS_FALLBACK_COLLECTION']: + mongomock.gridfs.enable_gridfs_integration() + override_user_settings() diff --git a/fireworks/user_objects/firetasks/filepad_tasks.py b/fireworks/user_objects/firetasks/filepad_tasks.py index b4bec5d5b..55ec172cf 100644 --- a/fireworks/user_objects/firetasks/filepad_tasks.py +++ b/fireworks/user_objects/firetasks/filepad_tasks.py @@ -1,7 +1,11 @@ import os - +import json +from glob import glob +from pymongo import DESCENDING +from ruamel.yaml import YAML from fireworks.core.firework import FiretaskBase from fireworks.utilities.filepad import FilePad +from fireworks.utilities.dict_mods import arrow_to_dot __author__ = "Kiran Mathew, Johannes Hoermann" __email__ = "kmathew@lbl.gov, johannes.hoermann@imtek.uni-freiburg.de" @@ -28,7 +32,6 @@ class AddFilesTask(FiretaskBase): optional_params = ["identifiers", "directory", "filepad_file", "compress", "metadata"] def run_task(self, fw_spec): - from glob import glob directory = os.path.abspath(self.get("directory", ".")) @@ -143,19 +146,12 @@ class GetFilesByQueryTask(FiretaskBase): ] def run_task(self, fw_spec): - import json - - import pymongo - from ruamel.yaml import YAML - - from fireworks.utilities.dict_mods import arrow_to_dot - fpad = get_fpad(self.get("filepad_file", None)) dest_dir = self.get("dest_dir", os.path.abspath(".")) new_file_names = self.get("new_file_names", []) query = self.get("query", {}) sort_key = self.get("sort_key", None) - sort_direction = self.get("sort_direction", pymongo.DESCENDING) + sort_direction = self.get("sort_direction", DESCENDING) limit = self.get("limit", None) fizzle_empty_result = self.get("fizzle_empty_result", True) fizzle_degenerate_file_name = self.get("fizzle_degenerate_file_name", True) diff --git a/fireworks/utilities/filepad.py b/fireworks/utilities/filepad.py index 1c32a65b4..c60a539e3 100644 --- a/fireworks/utilities/filepad.py +++ b/fireworks/utilities/filepad.py @@ -7,13 +7,12 @@ import zlib import gridfs -import pymongo +from pymongo import DESCENDING from monty.json import MSONable from monty.serialization import loadfn +from bson.objectid import ObjectId -import mongomock.gridfs -from mongomock import MongoClient - +from fireworks.fw_config import MongoClient from fireworks.fw_config import LAUNCHPAD_LOC, MONGO_SOCKET_TIMEOUT_MS from fireworks.utilities.fw_utilities import get_fw_logger @@ -21,8 +20,6 @@ __email__ = "kmathew@lbl.gov" __credits__ = "Anubhav Jain" -mongomock.gridfs.enable_gridfs_integration() - class FilePad(MSONable): def __init__( @@ -180,7 +177,7 @@ def get_file_by_id(self, gfs_id): doc = self.filepad.find_one({"gfs_id": gfs_id}) return self._get_file_contents(doc) - def get_file_by_query(self, query, sort_key=None, sort_direction=pymongo.DESCENDING): + def get_file_by_query(self, query, sort_key=None, sort_direction=DESCENDING): """ Args: @@ -293,8 +290,6 @@ def _get_file_contents(self, doc): Returns: (str, dict): the file content as a string, document dictionary """ - from bson.objectid import ObjectId - if doc: gfs_id = doc["gfs_id"] file_contents = self.gridfs.get(ObjectId(gfs_id)).read() From de7bd6703fbab5a3d1b7fbfc2d306fa998e1975e Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 24 May 2024 11:46:13 +0200 Subject: [PATCH 05/16] mark two tests for testing only with mongodb --- fireworks/core/tests/test_launchpad.py | 3 ++- pyproject.toml | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fireworks/core/tests/test_launchpad.py b/fireworks/core/tests/test_launchpad.py index 2945e5853..bc46b120b 100644 --- a/fireworks/core/tests/test_launchpad.py +++ b/fireworks/core/tests/test_launchpad.py @@ -1309,7 +1309,7 @@ def test_recover_errors(self) -> None: assert fw.state == "FIZZLED" - +@pytest.mark.mongodb class GridfsStoredDataTest(unittest.TestCase): """ Tests concerning the storage of data in Gridfs when the size of the @@ -1343,6 +1343,7 @@ def tearDown(self) -> None: for ldir in glob.glob(os.path.join(MODULE_DIR, "launcher_*")): shutil.rmtree(ldir) + def test_many_detours(self) -> None: task = DetoursTask(n_detours=2000, data_per_detour=["a" * 100] * 100) fw = Firework([task]) diff --git a/pyproject.toml b/pyproject.toml index 28a4cdfb3..207c0236f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,3 +64,8 @@ isort.split-on-trailing-comma = false "tests/**" = ["D"] "tasks.py" = ["D"] "fw_tutorials/**" = ["D"] + +[tool.pytest.ini_options] +markers = [ + "mongodb: marks tests that need mongodb (deselect with '-m \"not mongodb\"')", +] From 378b05545e1809539c42dc6030e69fc18119cb84 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 24 May 2024 14:43:44 +0200 Subject: [PATCH 06/16] mark tests for testing only with mongodb; fix duplicate names --- fireworks/scripts/tests/test_lpad_run.py | 1 + .../firetasks/tests/test_filepad_tasks.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fireworks/scripts/tests/test_lpad_run.py b/fireworks/scripts/tests/test_lpad_run.py index d5244c676..73e573e09 100644 --- a/fireworks/scripts/tests/test_lpad_run.py +++ b/fireworks/scripts/tests/test_lpad_run.py @@ -19,6 +19,7 @@ def lp(capsys): lp.reset(password=None, require_password=False) +@pytest.mark.mongodb @pytest.mark.parametrize(("detail", "expected_1", "expected_2"), [("count", "0\n", "1\n"), ("ids", "[]\n", "1\n")]) def test_lpad_get_fws(capsys, lp, detail, expected_1, expected_2) -> None: """Test lpad CLI get_fws command.""" diff --git a/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py b/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py index 7b807a681..d403930ae 100644 --- a/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py +++ b/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py @@ -23,6 +23,7 @@ def setUp(self) -> None: self.identifiers = ["write", "delete"] self.fp = FilePad.auto_load() + @pytest.mark.mongodb def test_addfilestask_run(self) -> None: t = AddFilesTask(paths=self.paths, identifiers=self.identifiers) t.run_task({}) @@ -43,6 +44,7 @@ def test_deletefilestask_run(self) -> None: assert file_contents is None assert doc is None + @pytest.mark.mongodb def test_getfilestask_run(self) -> None: t = AddFilesTask(paths=self.paths, identifiers=self.identifiers) t.run_task({}) @@ -56,6 +58,7 @@ def test_getfilestask_run(self) -> None: assert write_file_contents == f.read().encode() os.remove(os.path.join(dest_dir, new_file_names[0])) + @pytest.mark.mongodb def test_getfilesbyquerytask_run(self) -> None: """Tests querying objects from FilePad by metadata.""" t = AddFilesTask(paths=self.paths, identifiers=self.identifiers, metadata={"key": "value"}) @@ -69,7 +72,8 @@ def test_getfilesbyquerytask_run(self) -> None: assert test_file_contents == file.read().encode() os.remove(os.path.join(dest_dir, new_file_names[0])) - def test_getfilesbyquerytask_run(self) -> None: + @pytest.mark.mongodb + def test_getfilesbyquerytask_run_some_identifier(self) -> None: """Tests querying objects from FilePad by metadata.""" with open("original_test_file.txt", "w") as f: f.write("Some file with some content") @@ -87,6 +91,7 @@ def test_getfilesbyquerytask_run(self) -> None: assert test_file_contents == f.read().encode() os.remove(os.path.join(dest_dir, "queried_test_file.txt")) + @pytest.mark.mongodb def test_getfilesbyquerytask_metafile_run(self) -> None: """Tests writing metadata to a yaml file.""" with open("original_test_file.txt", "w") as f: @@ -138,6 +143,7 @@ def test_getfilesbyquerytask_raise_empty_result_run(self) -> None: t.run_task({}) # test successful if exception raised + @pytest.mark.mongodb def test_getfilesbyquerytask_ignore_degenerate_file_name(self) -> None: """Tests on ignoring degenerate file name in result from FilePad query.""" with open("degenerate_file.txt", "w") as f: @@ -179,6 +185,7 @@ def test_getfilesbyquerytask_raise_degenerate_file_name(self) -> None: t.run_task({}) # test successful if exception raised + @pytest.mark.mongodb def test_getfilesbyquerytask_sort_ascending_name_run(self) -> None: """Tests on sorting queried files in ascending order.""" file_contents = ["Some file with some content", "Some other file with some other content"] @@ -209,6 +216,7 @@ def test_getfilesbyquerytask_sort_ascending_name_run(self) -> None: with open("degenerate_file.txt") as f: assert file_contents[-1] == f.read() + @pytest.mark.mongodb def test_getfilesbyquerytask_sort_descending_name_run(self) -> None: """Tests on sorting queried files in descending order.""" file_contents = ["Some file with some content", "Some other file with some other content"] @@ -244,6 +252,7 @@ def test_getfilesbyquerytask_sort_descending_name_run(self) -> None: os.remove("degenerate_file.txt") + @pytest.mark.mongodb def test_addfilesfrompatterntask_run(self) -> None: t = AddFilesTask(paths="*.yaml", directory=module_dir) t.run_task({}) From 78355b7841fc63745d35226a18a6bff3d608e8ae Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 24 May 2024 14:57:52 +0200 Subject: [PATCH 07/16] skip failing test that has been masked by a test with identical name --- fireworks/user_objects/firetasks/tests/test_filepad_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py b/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py index d403930ae..e1372cbcc 100644 --- a/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py +++ b/fireworks/user_objects/firetasks/tests/test_filepad_tasks.py @@ -59,6 +59,7 @@ def test_getfilestask_run(self) -> None: os.remove(os.path.join(dest_dir, new_file_names[0])) @pytest.mark.mongodb + @pytest.mark.skip(reason='fails after fixing the identical names with the next test') def test_getfilesbyquerytask_run(self) -> None: """Tests querying objects from FilePad by metadata.""" t = AddFilesTask(paths=self.paths, identifiers=self.identifiers, metadata={"key": "value"}) From 605a009958e8203023ff569188e2859b83a68a89 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 24 May 2024 15:18:01 +0200 Subject: [PATCH 08/16] add a CI job for testing with mongomock --- .circleci/config.yml | 34 ++++++++++++++++++++++++++++++++-- requirements.txt | 1 + 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a53874990..9659eb18b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - pytest: + pytest_mongodb: working_directory: ~/fireworks docker: - image: continuumio/miniconda3:4.6.14 @@ -26,7 +26,37 @@ jobs: pip install .[workflow-checks,graph-plotting,flask-plotting] pytest fireworks + pytest_mongomock: + working_directory: ~/fireworks + docker: + - image: continuumio/miniconda3:4.6.14 + steps: + - checkout + - run: + command: | + export PATH=$HOME/miniconda3/bin:$PATH + conda config --set always_yes yes --set changeps1 no + conda update -q conda + conda info -a + conda create -q -n test-environment python=3.8 + source activate test-environment + conda update --quiet --all + pip install --quiet --ignore-installed -r requirements.txt -r requirements-ci.txt + - run: + name: Run fireworks tests + command: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate test-environment + pip install .[workflow-checks,graph-plotting,flask-plotting] + server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json + echo "{}" > $server_store_file + export MONGOMOCK_SERVERSTORE_FILE=$server_store_file + pytest -m "not mongodb" fireworks + rm $server_store_file + workflows: version: 2 build_and_test: - jobs: [pytest] + jobs: + - pytest_mongodb + - pytest_mongomock diff --git a/requirements.txt b/requirements.txt index 66f62c17b..856490548 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ flask==2.2.5 flask-paginate==0.5.5 gunicorn==20.0.4 tqdm==4.41.0 +mongomock @ git+https://github.com/ikondov/mongomock.git@persistence From 8b27956d7aaffea8a5eabacf0cc7aaf0d6938317 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 24 May 2024 16:06:52 +0200 Subject: [PATCH 09/16] switch from mongomock persistence branch to mongomock-persistence package --- fireworks/fw_config.py | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py index 7b2b964b6..390556eb0 100644 --- a/fireworks/fw_config.py +++ b/fireworks/fw_config.py @@ -8,7 +8,7 @@ from monty.design_patterns import singleton from monty.serialization import dumpfn, loadfn import pymongo -import mongomock +import mongomock_persistence import mongomock.gridfs __author__ = "Anubhav Jain" @@ -169,7 +169,7 @@ def override_user_settings() -> None: if globals()['MONGOMOCK_SERVERSTORE_FILE']: if not os.environ.get('MONGOMOCK_SERVERSTORE_FILE'): os.environ['MONGOMOCK_SERVERSTORE_FILE'] = globals()['MONGOMOCK_SERVERSTORE_FILE'] - globals()['MongoClient'] = getattr(mongomock, 'MongoClient') + globals()['MongoClient'] = getattr(mongomock_persistence, 'MongoClient') if globals()['GRIDFS_FALLBACK_COLLECTION']: mongomock.gridfs.enable_gridfs_integration() diff --git a/requirements.txt b/requirements.txt index 856490548..27b5a67bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ flask==2.2.5 flask-paginate==0.5.5 gunicorn==20.0.4 tqdm==4.41.0 -mongomock @ git+https://github.com/ikondov/mongomock.git@persistence +mongomock-persistence==0.0.3 diff --git a/setup.py b/setup.py index 83c18152e..d8ae1fdd0 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "tqdm>=4.8.4", "importlib-metadata>=4.8.2; python_version<'3.8'", "typing-extensions; python_version<'3.8'", - "mongomock @ git+https://github.com/ikondov/mongomock.git@persistence", + "mongomock-persistence>=0.0.3", ], extras_require={ "rtransfer": ["paramiko>=2.4.2"], From a8ad587a600049b1d32742eeb49d5515ae6968f2 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 7 Jun 2024 14:56:49 +0200 Subject: [PATCH 10/16] remove mongomock dependency in main package; add an extra mongomock --- .circleci/config.yml | 2 +- requirements.txt | 1 - setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9659eb18b..9b4f6b6ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: command: | export PATH=$HOME/miniconda3/bin:$PATH source activate test-environment - pip install .[workflow-checks,graph-plotting,flask-plotting] + pip install .[workflow-checks,graph-plotting,flask-plotting,mongomock] server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json echo "{}" > $server_store_file export MONGOMOCK_SERVERSTORE_FILE=$server_store_file diff --git a/requirements.txt b/requirements.txt index 27b5a67bb..66f62c17b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,3 @@ flask==2.2.5 flask-paginate==0.5.5 gunicorn==20.0.4 tqdm==4.41.0 -mongomock-persistence==0.0.3 diff --git a/setup.py b/setup.py index d8ae1fdd0..554530e9e 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,6 @@ "tqdm>=4.8.4", "importlib-metadata>=4.8.2; python_version<'3.8'", "typing-extensions; python_version<'3.8'", - "mongomock-persistence>=0.0.3", ], extras_require={ "rtransfer": ["paramiko>=2.4.2"], @@ -48,6 +47,7 @@ "flask-plotting": ["matplotlib>=2.0.1"], "workflow-checks": ["igraph>=0.7.1"], "graph-plotting": ["graphviz"], + "mongomock": ["mongomock-persistence>=0.0.3"], }, classifiers=[ "Programming Language :: Python", From 66216150724f761d7d71ccbe5f7ced5d96e1183d Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 7 Jun 2024 15:35:36 +0200 Subject: [PATCH 11/16] fix import of mongomock-persistence and mongomock --- fireworks/fw_config.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py index 390556eb0..d4b7f701d 100644 --- a/fireworks/fw_config.py +++ b/fireworks/fw_config.py @@ -3,13 +3,12 @@ from __future__ import annotations import os +import importlib from typing import Any from monty.design_patterns import singleton from monty.serialization import dumpfn, loadfn import pymongo -import mongomock_persistence -import mongomock.gridfs __author__ = "Anubhav Jain" __copyright__ = "Copyright 2012, The Materials Project" @@ -167,11 +166,18 @@ def override_user_settings() -> None: if 'MONGOMOCK_SERVERSTORE_FILE' in os.environ: globals()['MONGOMOCK_SERVERSTORE_FILE'] = os.environ['MONGOMOCK_SERVERSTORE_FILE'] if globals()['MONGOMOCK_SERVERSTORE_FILE']: + try: + mongomock_persistence = importlib.import_module('mongomock_persistence') + mongomock_gridfs = importlib.import_module('mongomock.gridfs') + except (ModuleNotFoundError, ImportError) as err: + msg = ('\nTo use mongomock instead of mongodb the extra mongomock must' + ' be installed, for example like this:\npip install fireworks[mongomock]') + raise RuntimeError(msg) from err if not os.environ.get('MONGOMOCK_SERVERSTORE_FILE'): os.environ['MONGOMOCK_SERVERSTORE_FILE'] = globals()['MONGOMOCK_SERVERSTORE_FILE'] globals()['MongoClient'] = getattr(mongomock_persistence, 'MongoClient') if globals()['GRIDFS_FALLBACK_COLLECTION']: - mongomock.gridfs.enable_gridfs_integration() + mongomock_gridfs.enable_gridfs_integration() override_user_settings() From d6abee8045c22532daffb1800e017a4de126b8ea Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 7 Jun 2024 17:32:16 +0200 Subject: [PATCH 12/16] add docs on new way to test and run tutorial --- docs_rst/index.rst | 8 ++++ docs_rst/quickstart_tutorial.rst | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 docs_rst/quickstart_tutorial.rst diff --git a/docs_rst/index.rst b/docs_rst/index.rst index afdfcd192..cba5f9619 100644 --- a/docs_rst/index.rst +++ b/docs_rst/index.rst @@ -115,6 +115,14 @@ To get a first glimpse of FireWorks, we suggest that you follow our installation installation quickstart +Quickstart (tutorial mode) +========================== + +.. toctree:: + :maxdepth: 1 + + quickstart_tutorial + Basic usage =========== diff --git a/docs_rst/quickstart_tutorial.rst b/docs_rst/quickstart_tutorial.rst new file mode 100644 index 000000000..0780db05f --- /dev/null +++ b/docs_rst/quickstart_tutorial.rst @@ -0,0 +1,76 @@ +============================================= +Two-minute installation, setup and quickstart +============================================= + +Install and setup +================= + +Supposed you have a :doc:`virtual environment ` with the `pip`` package installed. Then simply type: + + pip install fireworks[mongomock] + mkdir -p ~/.fireworks + echo MONGOMOCK_SERVERSTORE_FILE: $HOME/.fireworks/mongomock.json > ~/.fireworks/FW_config.yaml + echo '{}' > ~/.fireworks/mongomock.json + lpad reset --password="$(date +%Y-%m-%d)" + +See that the database contains no workflows: + + lpad get_wflows + +*Output*:: + + [] + + +Add and display a workflow +========================== + +Add a script that prints the date as a single firework in a workflow: + + lpad add_scripts 'date' -n date_printer_firework -w date_printer_workflow + +Let us display the workflow just added: + + lpad get_wflows -d more + +*Output*:: + + { + "state": "READY", + "name": "date_printer_workflow--1", + "created_on": "2024-06-07T15:05:02.096000", + "updated_on": "2024-06-07T15:05:02.096000", + "states": { + "date_printer_firework--1": "READY" + }, + "launch_dirs": { + "date_printer_firework--1": [] + } + } + +We have only one workflow with only one firework on the database. + +Run a workflow +============== + +Now we can run the firework in our workflow locally with this simple command: + + rlaunch singleshot + +*Output*:: + 2024-06-07 17:15:08,515 INFO Hostname/IP lookup (this will take a few seconds) + 2024-06-07 17:15:08,517 INFO Launching Rocket + 2024-06-07 17:15:08,608 INFO RUNNING fw_id: 1 in directory: /home/ubuntu + 2024-06-07 17:15:08,610 INFO Task started: ScriptTask. + Fri Jun 7 17:15:08 CEST 2024 + 2024-06-07 17:15:08,612 INFO Task completed: ScriptTask + 2024-06-07 17:15:08,616 INFO Rocket finished + +Further steps +============= + +This setup uses a JSON file on the local computer as a database instead of MongoDB. You can carry out many of the following tutorials +and do local testing by using this setting. + +If you want to complete the more advanced tutorials, such as the :doc:`queue tutorial `, or use FireWorks productively on +a computing cluster, then you should consider :doc:`installing and setting up FireWorks ` with a MongoDB server. \ No newline at end of file From 13f37ffdbbb0a300ce8ff7a41e98d2f36d9fb93f Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 7 Jun 2024 17:40:32 +0200 Subject: [PATCH 13/16] fix rst formatting --- docs_rst/quickstart_tutorial.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs_rst/quickstart_tutorial.rst b/docs_rst/quickstart_tutorial.rst index 0780db05f..fcaf2660a 100644 --- a/docs_rst/quickstart_tutorial.rst +++ b/docs_rst/quickstart_tutorial.rst @@ -5,7 +5,7 @@ Two-minute installation, setup and quickstart Install and setup ================= -Supposed you have a :doc:`virtual environment ` with the `pip`` package installed. Then simply type: +Supposed you have a :doc:`virtual environment ` with the `pip`` package installed. Then simply type:: pip install fireworks[mongomock] mkdir -p ~/.fireworks @@ -13,7 +13,7 @@ Supposed you have a :doc:`virtual environment ` with the ` echo '{}' > ~/.fireworks/mongomock.json lpad reset --password="$(date +%Y-%m-%d)" -See that the database contains no workflows: +See that the database contains no workflows:: lpad get_wflows @@ -21,15 +21,14 @@ See that the database contains no workflows: [] - Add and display a workflow ========================== -Add a script that prints the date as a single firework in a workflow: +Add a script that prints the date as a single firework in a workflow:: lpad add_scripts 'date' -n date_printer_firework -w date_printer_workflow -Let us display the workflow just added: +Let us display the workflow just added:: lpad get_wflows -d more @@ -53,7 +52,7 @@ We have only one workflow with only one firework on the database. Run a workflow ============== -Now we can run the firework in our workflow locally with this simple command: +Now we can run the firework in our workflow locally with this simple command:: rlaunch singleshot @@ -73,4 +72,4 @@ This setup uses a JSON file on the local computer as a database instead of Mongo and do local testing by using this setting. If you want to complete the more advanced tutorials, such as the :doc:`queue tutorial `, or use FireWorks productively on -a computing cluster, then you should consider :doc:`installing and setting up FireWorks ` with a MongoDB server. \ No newline at end of file +a computing cluster, then you should consider :doc:`installing and setting up FireWorks ` with a MongoDB server. From 244e53430d0af7e41dda2f6bb8f6ee9eb39f1873 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Fri, 7 Jun 2024 17:45:33 +0200 Subject: [PATCH 14/16] fix more rst formatting --- docs_rst/quickstart_tutorial.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs_rst/quickstart_tutorial.rst b/docs_rst/quickstart_tutorial.rst index fcaf2660a..8d6937908 100644 --- a/docs_rst/quickstart_tutorial.rst +++ b/docs_rst/quickstart_tutorial.rst @@ -5,7 +5,7 @@ Two-minute installation, setup and quickstart Install and setup ================= -Supposed you have a :doc:`virtual environment ` with the `pip`` package installed. Then simply type:: +Supposed you have a :doc:`virtual environment ` with the `pip` package installed. Then simply type:: pip install fireworks[mongomock] mkdir -p ~/.fireworks @@ -57,6 +57,7 @@ Now we can run the firework in our workflow locally with this simple command:: rlaunch singleshot *Output*:: + 2024-06-07 17:15:08,515 INFO Hostname/IP lookup (this will take a few seconds) 2024-06-07 17:15:08,517 INFO Launching Rocket 2024-06-07 17:15:08,608 INFO RUNNING fw_id: 1 in directory: /home/ubuntu @@ -68,8 +69,7 @@ Now we can run the firework in our workflow locally with this simple command:: Further steps ============= -This setup uses a JSON file on the local computer as a database instead of MongoDB. You can carry out many of the following tutorials -and do local testing by using this setting. - -If you want to complete the more advanced tutorials, such as the :doc:`queue tutorial `, or use FireWorks productively on -a computing cluster, then you should consider :doc:`installing and setting up FireWorks ` with a MongoDB server. +This setup uses a JSON file on the local computer as a database instead of MongoDB. You can continue with the other tutorials +and do local testing by using this setting. If you want to complete the more advanced tutorials, such as the +:doc:`queue tutorial `, or use FireWorks productively on a computing cluster, then you should consider +:doc:`installing and setting up FireWorks ` with a MongoDB server. From 2a1ffa0d67d90df41435a2f99ee87e8bae6f5a78 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Mon, 10 Jun 2024 14:02:28 +0200 Subject: [PATCH 15/16] fix github workflow for mongomock testing --- .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89ba28317..78d276fc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,3 +32,31 @@ jobs: - name: Run fireworks tests run: pytest fireworks + + pytest_mongomock: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + pip install -r requirements.txt -r requirements-ci.txt + pip install '.[workflow-checks,graph-plotting,flask-plotting,mongomock]' + + - name: Setup mongomock server store + run: | + server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json + echo "{}" > $server_store_file + export MONGOMOCK_SERVERSTORE_FILE=$server_store_file + + - name: Run fireworks tests with mongomock + run: | + pytest -m "not mongodb" fireworks + rm -f $server_store_file \ No newline at end of file From 7520962ed17b3d15deb1014478147d4e81e709f6 Mon Sep 17 00:00:00 2001 From: Ivan Kondov Date: Mon, 10 Jun 2024 14:59:39 +0200 Subject: [PATCH 16/16] put setup and pytest in one stanza --- .github/workflows/test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78d276fc7..606425e6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,13 +50,10 @@ jobs: pip install -r requirements.txt -r requirements-ci.txt pip install '.[workflow-checks,graph-plotting,flask-plotting,mongomock]' - - name: Setup mongomock server store + - name: Setup mongomock server store and run pytest run: | server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json echo "{}" > $server_store_file export MONGOMOCK_SERVERSTORE_FILE=$server_store_file - - - name: Run fireworks tests with mongomock - run: | pytest -m "not mongodb" fireworks rm -f $server_store_file \ No newline at end of file