PyMongo 4 is too advanced for us (#142)

* PyMongo 4 is too advanced for us
* Motor 3 is too advanced for us

file_catalog/mongo.py

@@ -7,8 +7,8 @@
 from typing import Any, Dict, List, Optional, Union, cast
 
 from motor.motor_tornado import MotorClient, MotorCursor  # type: ignore[import]
-import pymongo
-from pymongo.results import InsertOneResult
+import pymongo  # type: ignore[import]
+from pymongo.results import InsertOneResult  # type: ignore[import]
 from wipac_telemetry import tracing_tools as wtt
 
 from .schema.types import Metadata

requirements-dev.txt

@@ -38,8 +38,6 @@ cryptography==40.0.0
     # via pyjwt
 deprecated==1.2.13
     # via opentelemetry-api
-dnspython==2.3.0
-    # via pymongo
 exceptiongroup==1.1.1
     # via pytest
 flake8==6.0.0
@@ -70,7 +68,7 @@ mccabe==0.7.0
     # via flake8
 more-itertools==9.1.0
     # via wipac-file-catalog (setup.py)
-motor==3.1.1
+motor==2.5.1
     # via wipac-file-catalog (setup.py)
 multidict==6.0.4
     # via
@@ -124,7 +122,7 @@ pyflakes==3.0.1
     # via flake8
 pyjwt[crypto]==2.5.0
     # via wipac-rest-tools
-pymongo==4.3.3
+pymongo==3.13.0
     # via
     #   motor
     #   wipac-file-catalog (setup.py)

requirements.txt

@@ -22,8 +22,6 @@ cryptography==40.0.0
     # via pyjwt
 deprecated==1.2.13
     # via opentelemetry-api
-dnspython==2.3.0
-    # via pymongo
 googleapis-common-protos==1.56.2
     # via
     #   opentelemetry-exporter-jaeger-proto-grpc
@@ -38,7 +36,7 @@ importlib-metadata==6.0.1
     # via opentelemetry-api
 ldap3==2.9.1
     # via wipac-file-catalog (setup.py)
-motor==3.1.1
+motor==2.5.1
     # via wipac-file-catalog (setup.py)
 opentelemetry-api==1.17.0
     # via
@@ -76,7 +74,7 @@ pycparser==2.21
     # via cffi
 pyjwt[crypto]==2.5.0
     # via wipac-rest-tools
-pymongo==4.3.3
+pymongo==3.13.0
     # via
     #   motor
     #   wipac-file-catalog (setup.py)

resources/enable_profiling.py

@@ -1,21 +1,11 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 # fmt:off
 
 import os
-from typing import Any, cast, Dict
 
+import pymongo  # type: ignore[import]
 from pymongo import MongoClient
-from pymongo.database import Database
-
-# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4)
-# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL
-# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile
-OFF = 0
-SLOW_ONLY = 1
-ALL = 2
-
-FCDoc = Dict[str, Any]
 
 env = {
     'TEST_DATABASE_HOST': 'localhost',
@@ -30,10 +20,6 @@
         else:
             env[k] = os.environ[k]
 
-test_database_host = str(env['TEST_DATABASE_HOST'])
-test_database_port = int(str(env['TEST_DATABASE_PORT']))
-db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog)
-# db.set_profiling_level(pymongo.OFF)
-# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed
-db.command('profile', ALL, filter={'op': 'query'})
+db = MongoClient(host=env['TEST_DATABASE_HOST'], port=env['TEST_DATABASE_PORT']).file_catalog
+db.set_profiling_level(pymongo.ALL)
 print('MongoDB profiling enabled')

resources/enable_profiling_future.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# fmt:off
+
+import os
+from typing import Any, cast, Dict
+
+from pymongo import MongoClient  # type: ignore[import]
+from pymongo.database import Database  # type: ignore[import]
+
+# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4)
+# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL
+# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile
+OFF = 0
+SLOW_ONLY = 1
+ALL = 2
+
+FCDoc = Dict[str, Any]
+
+env = {
+    'TEST_DATABASE_HOST': 'localhost',
+    'TEST_DATABASE_PORT': 27017,
+}
+for k in env:
+    if k in os.environ:
+        if isinstance(env[k], int):
+            env[k] = int(os.environ[k])
+        elif isinstance(env[k], float):
+            env[k] = float(os.environ[k])
+        else:
+            env[k] = os.environ[k]
+
+test_database_host = str(env['TEST_DATABASE_HOST'])
+test_database_port = int(str(env['TEST_DATABASE_PORT']))
+db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog)
+# db.set_profiling_level(pymongo.OFF)
+# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed
+db.command('profile', ALL, filter={'op': 'query'})
+print('MongoDB profiling enabled')

resources/profile_queries.py

@@ -1,21 +1,12 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 # fmt:off
+# flake8:noqa
 
 import os
-from typing import Any, cast, Dict
 
+import pymongo  # type: ignore[import]
 from pymongo import MongoClient
-from pymongo.database import Database
-
-# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4)
-# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL
-# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile
-OFF = 0
-SLOW_ONLY = 1
-ALL = 2
-
-FCDoc = Dict[str, Any]
 
 env = {
     'TEST_DATABASE_HOST': 'localhost',
@@ -30,47 +21,39 @@
         else:
             env[k] = os.environ[k]
 
-test_database_host = str(env['TEST_DATABASE_HOST'])
-test_database_port = int(str(env['TEST_DATABASE_PORT']))
-db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog)
-
-# level = db.profiling_level()
-# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-profiling-level-is-removed
-profile: Dict[str, Any] = db.command('profile', -1)
-level = profile['was']
-if level != ALL:
+db = MongoClient(host=env['TEST_DATABASE_HOST'], port=env['TEST_DATABASE_PORT']).file_catalog
+ret = db.profiling_level()
+if ret != pymongo.ALL:
     raise Exception('profiling disabled')
-# db.set_profiling_level(pymongo.OFF)
-# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed
-db.command('profile', ALL, filter={'op': 'query'})
+db.set_profiling_level(pymongo.OFF)
 
 unrealistic_queries = [
     {'locations.archive': True},
     {'locations.archive': False},
     {'locations.archive': None},
     {'locations.archive': None, 'run.first_event': {'$lte': 400}, 'run.last_event': {'$gte': 400}},
-    # the query to get the queries that we're profiling doesn't count!
-    {'op': {'$nin': ['command', 'insert']}},
 ]
 
 bad_queries = []
-ret = db.system.profile.find({'op': {'$nin': ['command', 'insert']}})
+ret = db.system.profile.find({ 'op': { '$nin' : ['command', 'insert'] } })
 for query in ret:
     try:
+        if 'find' in query['command'] and query['command']['find'] == 'collections':
+            continue
         # exclude unrealistic test queries
         if 'filter' in query['command'] and query['command']['filter'] in unrealistic_queries:
             continue
         if 'planSummary' not in query:
             print(query)
             continue
         if 'IXSCAN' not in query['planSummary']:
-            bad_queries.append((query['command'], query['planSummary']))
+            bad_queries.append((query['command'],query['planSummary']))
     except Exception:
         print(query)
         raise
 
 if bad_queries:
-    for q, p in bad_queries:
+    for q,p in bad_queries:
         print(q)
         print(p)
         print('---')

resources/profile_queries_future.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+# fmt:off
+
+import os
+from typing import Any, cast, Dict
+
+from pymongo import MongoClient  # type: ignore[import]
+from pymongo.database import Database  # type: ignore[import]
+
+# PyMongo profiling level constants from PyMongo 3 (removed in PyMongo 4)
+# See: https://api.mongodb.com/python/3.0.3/api/pymongo/database.html#pymongo.ALL
+# See: https://www.mongodb.com/docs/manual/reference/command/profile/#mongodb-dbcommand-dbcmd.profile
+OFF = 0
+SLOW_ONLY = 1
+ALL = 2
+
+FCDoc = Dict[str, Any]
+
+env = {
+    'TEST_DATABASE_HOST': 'localhost',
+    'TEST_DATABASE_PORT': 27017,
+}
+for k in env:
+    if k in os.environ:
+        if isinstance(env[k], int):
+            env[k] = int(os.environ[k])
+        elif isinstance(env[k], float):
+            env[k] = float(os.environ[k])
+        else:
+            env[k] = os.environ[k]
+
+test_database_host = str(env['TEST_DATABASE_HOST'])
+test_database_port = int(str(env['TEST_DATABASE_PORT']))
+db: Database[FCDoc] = cast(Database[FCDoc], MongoClient(host=test_database_host, port=test_database_port).file_catalog)
+
+# level = db.profiling_level()
+# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-profiling-level-is-removed
+profile: Dict[str, Any] = db.command('profile', -1)
+level = profile['was']
+if level != ALL:
+    raise Exception('profiling disabled')
+# db.set_profiling_level(pymongo.OFF)
+# See: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html#database-set-profiling-level-is-removed
+db.command('profile', ALL, filter={'op': 'query'})
+
+unrealistic_queries = [
+    {'locations.archive': True},
+    {'locations.archive': False},
+    {'locations.archive': None},
+    {'locations.archive': None, 'run.first_event': {'$lte': 400}, 'run.last_event': {'$gte': 400}},
+    # the query to get the queries that we're profiling doesn't count!
+    {'op': {'$nin': ['command', 'insert']}},
+]
+
+bad_queries = []
+ret = db.system.profile.find({'op': {'$nin': ['command', 'insert']}})
+for query in ret:
+    try:
+        # exclude unrealistic test queries
+        if 'filter' in query['command'] and query['command']['filter'] in unrealistic_queries:
+            continue
+        if 'planSummary' not in query:
+            print(query)
+            continue
+        if 'IXSCAN' not in query['planSummary']:
+            bad_queries.append((query['command'], query['planSummary']))
+    except Exception:
+        print(query)
+        raise
+
+if bad_queries:
+    for q, p in bad_queries:
+        print(q)
+        print(p)
+        print('---')
+    raise Exception('Non-indexed queries')
+
+print('MongoDB profiling OK')

setup.cfg

@@ -57,8 +57,8 @@ packages = find:
 install_requires =
 	coloredlogs
 	ldap3
-	motor
-	pymongo
+	motor<3
+	pymongo<4
 	requests
 	requests-futures
 	requests-toolbelt

tests/conftest.py

@@ -12,8 +12,8 @@
 import socket
 from typing import Any, AsyncGenerator, cast, Dict
 
-from pymongo import MongoClient
-from pymongo.errors import ServerSelectionTimeoutError
+from pymongo import MongoClient  # type: ignore[import]
+from pymongo.errors import ServerSelectionTimeoutError  # type: ignore[import]
 import pytest
 from pytest import MonkeyPatch
 import pytest_asyncio

tests/test_mongo.py

@@ -10,7 +10,7 @@
 
 from file_catalog.mongo import AllKeys, Mongo
 from motor import MotorCollection  # type: ignore[import]
-from pymongo.errors import DuplicateKeyError
+from pymongo.errors import DuplicateKeyError  # type: ignore[import]
 
 logger = logging.getLogger(__name__)