19794 update colin/lear sync job (bcgov#2834)

jobs/update-colin-filings/config.py

@@ -47,12 +47,13 @@ class _Config(object):  # pylint: disable=too-few-public-methods
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
     COLIN_URL = os.getenv('COLIN_URL', '')
-    LEGAL_URL = os.getenv('LEGAL_URL', '')
+    LEGAL_API_URL = os.getenv('LEGAL_API_URL', '')
     SENTRY_DSN = os.getenv('SENTRY_DSN', '')
 
     ACCOUNT_SVC_AUTH_URL = os.getenv('ACCOUNT_SVC_AUTH_URL', None)
     ACCOUNT_SVC_CLIENT_ID = os.getenv('ACCOUNT_SVC_CLIENT_ID', None)
     ACCOUNT_SVC_CLIENT_SECRET = os.getenv('ACCOUNT_SVC_CLIENT_SECRET', None)
+    ACCOUNT_SVC_TIMEOUT = os.getenv('ACCOUNT_SVC_TIMEOUT', 20)
 
     SECRET_KEY = 'a secret'
 
@@ -74,7 +75,7 @@ class TestConfig(_Config):  # pylint: disable=too-few-public-methods
     TESTING = True
 
     COLIN_URL = os.getenv('COLIN_URL_TEST', '')
-    LEGAL_URL = os.getenv('LEGAL_URL_TEST', '')
+    LEGAL_API_URL = os.getenv('LEGAL_API_URL_TEST', '')
 
 
 class ProdConfig(_Config):  # pylint: disable=too-few-public-methods

jobs/update-colin-filings/k8s/templates/cronjob.yaml

@@ -52,10 +52,10 @@ objects:
                       secretKeyRef:
                         key: SENTRY_DSN
                         name: ${NAME}-${TAG}-secret
-                  - name: LEGAL_URL
+                  - name: LEGAL_API_URL
                     valueFrom:
                       secretKeyRef:
-                        key: LEGAL_URL
+                        key: LEGAL_API_URL
                         name: ${NAME}-${TAG}-secret
                   - name: ACCOUNT_SVC_AUTH_URL
                     valueFrom:

jobs/update-colin-filings/update_colin_filings.py

@@ -21,7 +21,6 @@
 import requests
 import sentry_sdk  # noqa: I001; pylint: disable=ungrouped-imports; conflicts with Flake8
 from flask import Flask
-from legal_api.models import Business
 from legal_api.services.bootstrap import AccountService
 from sentry_sdk.integrations.logging import LoggingIntegration  # noqa: I001
 
@@ -47,6 +46,10 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')):
             integrations=[SENTRY_LOGGING]
         )
 
+    # Static class load the variables while importing the class for the first time,
+    # By then config is not loaded, so it never get the config value
+    AccountService.timeout = int(app.config.get('ACCOUNT_SVC_TIMEOUT'))
+
     register_shellcontext(app)
 
     return app
@@ -61,9 +64,10 @@ def shell_context():
     app.shell_context_processor(shell_context)
 
 
-def get_filings(app: Flask = None):
+def get_filings(app: Flask, token):
     """Get a filing with filing_id."""
-    req = requests.get(f'{app.config["LEGAL_URL"]}/internal/filings',
+    req = requests.get(f'{app.config["LEGAL_API_URL"]}/internal/filings',
+                       headers={'Authorization': AccountService.BEARER + token},
                        timeout=AccountService.timeout)
     if not req or req.status_code != 200:
         app.logger.error(f'Failed to collect filings from legal-api. {req} {req.json()} {req.status_code}')
@@ -78,10 +82,7 @@ def send_filing(app: Flask = None, filing: dict = None, filing_id: str = None):
 
     filing_type = filing['filing']['header'].get('name', None)
     identifier = filing['filing']['business'].get('identifier', None)
-    if identifier[:2] == Business.LegalTypes.COOP.value:
-        legal_type = Business.LegalTypes.COOP.value
-    else:
-        legal_type = filing['filing']['business'].get('legalType', Business.LegalTypes.BCOMP.value)
+    legal_type = filing['filing']['business'].get('legalType', None)
 
     req = None
     if legal_type and identifier and filing_type:
@@ -102,8 +103,8 @@ def send_filing(app: Flask = None, filing: dict = None, filing_id: str = None):
 def update_colin_id(app: Flask = None, filing_id: str = None, colin_ids: list = None, token: dict = None):
     """Update the colin_id in the filings table."""
     req = requests.patch(
-        f'{app.config["LEGAL_URL"]}/internal/filings/{filing_id}',
-        headers={'Authorization': f'Bearer {token}'},
+        f'{app.config["LEGAL_API_URL"]}/internal/filings/{filing_id}',
+        headers={'Authorization': AccountService.BEARER + token},
         json={'colinIds': colin_ids},
         timeout=AccountService.timeout
     )
@@ -132,7 +133,7 @@ def run():
             # get updater-job token
             token = AccountService.get_bearer_token()
 
-            filings = get_filings(app=application)
+            filings = get_filings(application, token)
             if not filings:
                 # pylint: disable=no-member; false positive
                 application.logger.debug('No completed filings to send to colin.')

jobs/update-legal-filings/config.py

@@ -47,12 +47,13 @@ class _Config(object):  # pylint: disable=too-few-public-methods
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
     COLIN_URL = os.getenv('COLIN_URL', '')
-    LEGAL_URL = os.getenv('LEGAL_URL', '')
+    LEGAL_API_URL = os.getenv('LEGAL_API_URL', '')
     SENTRY_DSN = os.getenv('SENTRY_DSN', '')
 
     ACCOUNT_SVC_AUTH_URL = os.getenv('ACCOUNT_SVC_AUTH_URL', None)
     ACCOUNT_SVC_CLIENT_ID = os.getenv('ACCOUNT_SVC_CLIENT_ID', None)
     ACCOUNT_SVC_CLIENT_SECRET = os.getenv('ACCOUNT_SVC_CLIENT_SECRET', None)
+    ACCOUNT_SVC_TIMEOUT = os.getenv('ACCOUNT_SVC_TIMEOUT', 20)
 
     NATS_SERVERS = os.getenv('NATS_SERVERS', None)
     NATS_CLUSTER_ID = os.getenv('NATS_CLUSTER_ID', None)
@@ -84,7 +85,7 @@ class TestConfig(_Config):  # pylint: disable=too-few-public-methods
     TESTING = True
 
     COLIN_URL = os.getenv('COLIN_URL_TEST', '')
-    LEGAL_URL = os.getenv('LEGAL_URL_TEST', '')
+    LEGAL_API_URL = os.getenv('LEGAL_API_URL_TEST', '')
 
     # POSTGRESQL
     DB_USER = os.getenv('DATABASE_TEST_USERNAME', '')

jobs/update-legal-filings/k8s/templates/cronjob.yaml

@@ -52,11 +52,11 @@ objects:
                       secretKeyRef:
                         name: ${NAME}-${TAG}-secret
                         key: SENTRY_DSN
-                  - name: LEGAL_URL
+                  - name: LEGAL_API_URL
                     valueFrom:
                       secretKeyRef:
                         name: ${NAME}-${TAG}-secret
-                        key: LEGAL_URL
+                        key: LEGAL_API_URL
                   - name: ACCOUNT_SVC_AUTH_URL
                     valueFrom:
                       secretKeyRef:

jobs/update-legal-filings/requirements.txt

@@ -23,6 +23,10 @@ rsa==4.7.2
 sentry-sdk==1.20.0
 six==1.15.0
 urllib3==1.26.11
+asyncio-nats-client==0.11.4
+asyncio-nats-streaming==0.4.0
+nest_asyncio
+protobuf==3.15.8
 git+https://github.com/bcgov/lear.git#subdirectory=colin-api
 git+https://github.com/bcgov/lear.git#egg=legal_api&subdirectory=legal-api
 git+https://github.com/bcgov/[email protected]#egg=registry_schemas

jobs/update-legal-filings/requirements/prod.txt

@@ -7,3 +7,6 @@ flask-jwt-oidc>=0.1.5
 python-dotenv
 sentry-sdk[flask]
 werkzeug
+asyncio-nats-client==0.11.4
+asyncio-nats-streaming==0.4.0
+nest_asyncio

jobs/update-legal-filings/update_legal_filings.py

@@ -58,6 +58,10 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')):
 
     register_shellcontext(app)
 
+    # Static class load the variables while importing the class for the first time,
+    # By then config is not loaded, so it never get the config value
+    AccountService.timeout = int(app.config.get('ACCOUNT_SVC_TIMEOUT'))
+
     return app
 
 
@@ -75,15 +79,17 @@ def check_for_manual_filings(application: Flask = None, token: dict = None):
     """Check for colin filings in oracle."""
     id_list = []
     colin_events = None
-    legal_url = application.config['LEGAL_URL']
+    legal_url = application.config['LEGAL_API_URL']
     colin_url = application.config['COLIN_URL']
     corp_types = [Business.TypeCodes.COOP.value, Business.TypeCodes.BC_COMP.value,
                   Business.TypeCodes.ULC_COMP.value, Business.TypeCodes.CCC_COMP.value]
     no_corp_num_prefix_in_colin = [Business.TypeCodes.BC_COMP.value, Business.TypeCodes.ULC_COMP.value,
                                    Business.TypeCodes.CCC_COMP.value]
 
     # get max colin event_id from legal
-    response = requests.get(f'{legal_url}/internal/filings/colin_id', timeout=AccountService.timeout)
+    response = requests.get(f'{legal_url}/internal/filings/colin_id',
+                            headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
+                            timeout=AccountService.timeout)
     if response.status_code not in [200, 404]:
         application.logger.error(f'Error getting last updated colin id from \
             legal: {response.status_code} {response.json()}')
@@ -139,8 +145,10 @@ def check_for_manual_filings(application: Flask = None, token: dict = None):
                 )
                 if response.status_code == 200:
                     # check legal table
-                    response = requests.get(f'{legal_url}/internal/filings/colin_id/{info["event_id"]}',
-                                            timeout=AccountService.timeout)
+                    response = requests.get(
+                        f'{legal_url}/internal/filings/colin_id/{info["event_id"]}',
+                        headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
+                        timeout=AccountService.timeout)
                     if response.status_code == 404:
                         id_list.append(info)
                     elif response.status_code != 200:
@@ -209,7 +217,7 @@ def update_filings(application):  # pylint: disable=redefined-outer-name, too-ma
                     # call legal api with filing
                     application.logger.debug(f'sending filing with event info: {event_info} to legal api.')
                     response = requests.post(
-                        f'{application.config["LEGAL_URL"]}/{event_info["corp_num"]}/filings',
+                        f'{application.config["LEGAL_API_URL"]}/{event_info["corp_num"]}/filings',
                         json=filing,
                         headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
                         timeout=AccountService.timeout
@@ -247,7 +255,7 @@ def update_filings(application):  # pylint: disable=redefined-outer-name, too-ma
             # update max_event_id in legal_db
             application.logger.debug(f'setting last_event_id in legal_db to {max_event_id}')
             response = requests.post(
-                f'{application.config["LEGAL_URL"]}/internal/filings/colin_id/{max_event_id}',
+                f'{application.config["LEGAL_API_URL"]}/internal/filings/colin_id/{max_event_id}',
                 headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
                 timeout=AccountService.timeout
             )
@@ -308,7 +316,7 @@ async def update_business_nos(application):  # pylint: disable=redefined-outer-n
         # get identifiers with outstanding tax_ids
         application.logger.debug('Getting businesses with outstanding tax ids from legal api...')
         response = requests.get(
-            application.config['LEGAL_URL'] + '/internal/tax_ids',
+            application.config['LEGAL_API_URL'] + '/internal/tax_ids',
             headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
             timeout=AccountService.timeout
         )
@@ -334,7 +342,7 @@ async def update_business_nos(application):  # pylint: disable=redefined-outer-n
                 # update lear with new tax ids from colin
                 application.logger.debug(f'Updating tax ids for {tax_ids.keys()} in lear...')
                 response = requests.post(
-                    application.config['LEGAL_URL'] + '/internal/tax_ids',
+                    application.config['LEGAL_API_URL'] + '/internal/tax_ids',
                     json=tax_ids,
                     headers={'Content-Type': CONTENT_TYPE_JSON, 'Authorization': f'Bearer {token}'},
                     timeout=AccountService.timeout