Merge pull request #1520 from bcgov/chore/alex-etl-launch

Chore: Minor etl changes
bcgov · Dec 23, 2024 · d62ab2a · d62ab2a
2 parents 92591f1 + 5566bde
commit d62ab2a
Show file tree

Hide file tree

Showing 11 changed files with 42 additions and 11 deletions.
diff --git a/backend/lcfs/web/application.py b/backend/lcfs/web/application.py
@@ -31,6 +31,7 @@
     "http://localhost:3000",
     "https://lcfs-dev.apps.silver.devops.gov.bc.ca",
     "https://lcfs-test.apps.silver.devops.gov.bc.ca",
+    "https://lcfs-prod.apps.silver.devops.gov.bc.ca",
     "https://lcfs.apps.silver.devops.gov.bc.ca",
 ]
 

diff --git a/etl/data-transfer.sh b/etl/data-transfer.sh
@@ -122,6 +122,9 @@ if [ "$direction" = "import" ]; then
     oc exec $pod_name -- bash -c "rm /tmp/$db_name.tar"
     echo
 
+    echo "** Cleaning up local dump file"
+    rm $db_name.tar
+
 elif [ "$direction" = "export" ]; then
     echo "** Starting pg_dump on local container"
     docker exec $local_container bash -c "pg_dump -U $db_name -F t --no-privileges --no-owner -c -d $db_name > /tmp/$db_name.tar"
@@ -131,24 +134,30 @@ elif [ "$direction" = "export" ]; then
     docker cp $local_container:/tmp/$db_name.tar ./
     echo
 
+    echo "** Preparing .tar file for OpenShift pod"
+    mkdir -p tmp_transfer
+    mv $db_name.tar tmp_transfer/
+    echo
+
     echo "** Uploading .tar file to OpenShift pod"
-    oc rsync ./$db_name.tar $pod_name:/tmp/
+    oc rsync ./tmp_transfer $pod_name:/tmp/
     echo
 
     echo "** Restoring database on OpenShift pod"
-    oc exec $pod_name -- bash -c "pg_restore -U postgres --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/$db_name.tar" || true
+    oc exec $pod_name -- bash -c "pg_restore -U postgres --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/tmp_transfer/$db_name.tar" || true
+    echo
+
+    echo "** Cleaning up temporary files on OpenShift pod"
+    oc exec $pod_name -- bash -c "rm -rf /tmp/tmp_transfer"
     echo
 
     echo "** Cleaning up dump file from local container"
-    docker exec $local_container bash -c "rm /tmp/$db_name.tar"
+    docker exec $local_container bash -c "rm /tmp/tmp_transfer" || true
     echo
-fi
 
-echo "** Cleaning up dump file from local container"
-docker exec $local_container bash -c "rm /tmp/$db_name.tar" || true
-echo
+    echo "** Cleaning up local temporary directory"
+    rm -rf tmp_transfer
+fi
 
-echo "** Cleaning up local dump file"
-rm $db_name.tar
 
 echo "** Finished data transfer and cleanup"
diff --git a/etl/database/nifi-registry-primary.mv.db b/etl/database/nifi-registry-primary.mv.db
diff --git a/etl/nifi/conf/flow.json.gz b/etl/nifi/conf/flow.json.gz
diff --git a/etl/nifi/conf/flow.xml.gz b/etl/nifi/conf/flow.xml.gz
diff --git a/etl/nifi_scripts/adminAdjTrxn.groovy b/etl/nifi_scripts/adminAdjTrxn.groovy
@@ -5,6 +5,8 @@ import java.sql.ResultSet
 import java.time.OffsetDateTime
 import java.sql.Timestamp
 
+log.warn("**** STARTED ADMIN ADJUSTMENT ETL ****")
+
 def SOURCE_QUERY = """
 WITH
         internal_comment AS (
@@ -433,4 +435,6 @@ def insertadminAdjustment(ResultSet rs, PreparedStatement adminAdjustmentStmt,
     adminAdjustmentStmt.setInt(11, rs.getInt('admin_adjustment_id'))
     def result = adminAdjustmentStmt.executeQuery()
     return result.next() ? result.getInt('admin_adjustment_id') : null
-}
+}
+
+log.warn("**** COMPLETED ADMIN ADJUSTMENT ETL ****")
diff --git a/etl/nifi_scripts/compliance_report.groovy b/etl/nifi_scripts/compliance_report.groovy
@@ -1014,3 +1014,5 @@ def updateComplianceReportWithTransaction(Connection conn, int lcfsReportId, int
 // =========================================
 // Script Termination
 // =========================================
+
+log.warn("**** COMPLETED COMPLIANCE REPORT ETL ****")
diff --git a/etl/nifi_scripts/initiativeAgrmtTrxn.groovy b/etl/nifi_scripts/initiativeAgrmtTrxn.groovy
@@ -5,6 +5,8 @@ import java.sql.ResultSet
 import java.time.OffsetDateTime
 import java.sql.Timestamp
 
+log.warn("**** STARTING INITIATIVE AGREEMENT ETL ****")
+
 def SOURCE_QUERY = """
 WITH
         internal_comment AS (
@@ -433,4 +435,6 @@ def insertInitiativeAgreement(ResultSet rs, PreparedStatement initiativeAgreemen
     initiativeAgreementStmt.setInt(11, rs.getInt('initiative_agreement_id'))
     def result = initiativeAgreementStmt.executeQuery()
     return result.next() ? result.getInt('initiative_agreement_id') : null
-}
+}
+
+log.warn("**** COMPLETED INITIATIVE AGREEMENT ETL ****")
diff --git a/etl/nifi_scripts/organization.groovy b/etl/nifi_scripts/organization.groovy
@@ -3,6 +3,8 @@ import java.sql.PreparedStatement
 import java.sql.ResultSet
 import java.util.Random
 
+log.warn("**** STARTING ORGANIZATION ETL ****")
+
 // SQL query to fetch data from the source
 def sourceQuery = """
     SELECT
@@ -227,3 +229,5 @@ try {
     if (sourceConn != null) sourceConn.close()
     if (destinationConn != null) destinationConn.close()
 }
+
+log.warn("**** COMPLETED ORGANIZATION ETL ****")
diff --git a/etl/nifi_scripts/transfer.groovy b/etl/nifi_scripts/transfer.groovy
@@ -5,6 +5,8 @@ import java.sql.ResultSet
 import java.time.OffsetDateTime
 import java.sql.Timestamp
 
+log.warn("**** STARTING TRANSFER ETL ****")
+
 def SOURCE_QUERY = """
           WITH
         internal_comment AS (
@@ -515,3 +517,4 @@ def insertTransfer(ResultSet rs, PreparedStatement transferStmt, Long fromTransa
     return result.next() ? result.getInt('transfer_id') : null
 }
 
+log.warn("**** COMPLETED TRANSFER ETL ****")
diff --git a/etl/nifi_scripts/user.groovy b/etl/nifi_scripts/user.groovy
@@ -2,6 +2,8 @@ import java.sql.Connection
 import java.sql.PreparedStatement
 import java.sql.ResultSet
 
+log.warn("**** STARTING USER ETL ****")
+
 // SQL query to extract user profiles
 def userProfileQuery = """
     SELECT id as user_profile_id,
@@ -154,3 +156,5 @@ try {
     if (sourceConn != null) sourceConn.close()
     if (destinationConn != null) destinationConn.close()
 }
+
+log.warn("**** COMPLETED USER ETL ****")