From e863046bcef9a7c909d177a93b933e7574317e0c Mon Sep 17 00:00:00 2001 From: Pratyus Patnaik Date: Wed, 18 Oct 2017 13:40:47 -0700 Subject: [PATCH 1/3] Added ability to access mongo oplog via mongos router/proxy --- mongo_connector/connector.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mongo_connector/connector.py b/mongo_connector/connector.py index 7465b6d4..ebdf10c9 100644 --- a/mongo_connector/connector.py +++ b/mongo_connector/connector.py @@ -124,6 +124,9 @@ def __init__(self, mongo_address, doc_managers=None, **kwargs): # Timezone awareness self.tz_aware = kwargs.get('tz_aware', False) + # If connecting to mongos router for oplog access + self.is_oplog_proxy= kwargs.get('is_oplog_proxy', False) + # SSL keyword arguments to MongoClient. ssl_certfile = kwargs.pop('ssl_certfile', None) ssl_ca_certs = kwargs.pop('ssl_ca_certs', None) @@ -215,7 +218,8 @@ def from_config(cls, config): ssl_keyfile=config['ssl.sslKeyfile'], ssl_ca_certs=config['ssl.sslCACerts'], ssl_cert_reqs=config['ssl.sslCertificatePolicy'], - tz_aware=config['timezoneAware'] + tz_aware=config['timezoneAware'], + is_oplog_proxy=config['isOplogProxy'] ) return connector @@ -374,12 +378,12 @@ def run(self): ) return - # Establish a connection to the replica set as a whole - self.main_conn.close() - self.main_conn = self.create_authed_client( - replicaSet=is_master['setName']) - - self.update_version_from_client(self.main_conn) + if not self.is_oplog_proxy: + # Establish a connection to the replica set as a whole + self.main_conn.close() + self.main_conn = self.create_authed_client( + replicaSet=is_master['setName']) + self.update_version_from_client(self.main_conn) # non sharded configuration oplog = OplogThread( @@ -486,6 +490,16 @@ def add_option(*args, **kwargs): " would be a valid argument to `-m`. Don't use" " quotes around the address.") + is_oplog_proxy = add_option( + config_key="isOplogProxy", + default=False, + type=bool) + + is_oplog_proxy.add_cli( + "--is_oplog_proxy", dest="is_oplog_proxy", help= + "True if passing uri to mongos router/proxy with access" + "to mongo oplog.rs.") + oplog_file = add_option( config_key="oplogFile", default="oplog.timestamp", From da7bb64f7bd3055d8278188d7409207dcdac1bd0 Mon Sep 17 00:00:00 2001 From: Pratyus Patnaik Date: Wed, 18 Oct 2017 16:01:49 -0700 Subject: [PATCH 2/3] Added check to verify local.oplog.rs is readable --- mongo_connector/connector.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mongo_connector/connector.py b/mongo_connector/connector.py index ebdf10c9..6fdf8019 100644 --- a/mongo_connector/connector.py +++ b/mongo_connector/connector.py @@ -384,6 +384,13 @@ def run(self): self.main_conn = self.create_authed_client( replicaSet=is_master['setName']) self.update_version_from_client(self.main_conn) + else: + try: + # Check if local.oplog.rs is readable + self.main_conn.local.oplog.rs.find_one() + except pymongo.errors.OperationFailure: + LOG.error('Could not read local.oplog.rs!') + sys.exit(1) # non sharded configuration oplog = OplogThread( From 50f02cb5906cb6dbcc4c2e04a0b5c9ee5c0faaca Mon Sep 17 00:00:00 2001 From: Pratyus Patnaik Date: Thu, 19 Oct 2017 13:37:46 -0700 Subject: [PATCH 3/3] Updated comment wording --- mongo_connector/connector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongo_connector/connector.py b/mongo_connector/connector.py index 6fdf8019..bb516598 100644 --- a/mongo_connector/connector.py +++ b/mongo_connector/connector.py @@ -124,7 +124,7 @@ def __init__(self, mongo_address, doc_managers=None, **kwargs): # Timezone awareness self.tz_aware = kwargs.get('tz_aware', False) - # If connecting to mongos router for oplog access + # If oplog access is via the proxy self.is_oplog_proxy= kwargs.get('is_oplog_proxy', False) # SSL keyword arguments to MongoClient. @@ -504,8 +504,8 @@ def add_option(*args, **kwargs): is_oplog_proxy.add_cli( "--is_oplog_proxy", dest="is_oplog_proxy", help= - "True if passing uri to mongos router/proxy with access" - "to mongo oplog.rs.") + "True if passed uri is a proxy with access to mongo" + "oplog.rs.") oplog_file = add_option( config_key="oplogFile",