diff --git a/backend/lcfs/web/application.py b/backend/lcfs/web/application.py index 8aef6126c..f107d499f 100644 --- a/backend/lcfs/web/application.py +++ b/backend/lcfs/web/application.py @@ -31,6 +31,7 @@ "http://localhost:3000", "https://lcfs-dev.apps.silver.devops.gov.bc.ca", "https://lcfs-test.apps.silver.devops.gov.bc.ca", + "https://lcfs-prod.apps.silver.devops.gov.bc.ca", "https://lcfs.apps.silver.devops.gov.bc.ca", ] diff --git a/etl/data-transfer.sh b/etl/data-transfer.sh index 8a9acf33a..9d3e97357 100755 --- a/etl/data-transfer.sh +++ b/etl/data-transfer.sh @@ -122,6 +122,9 @@ if [ "$direction" = "import" ]; then oc exec $pod_name -- bash -c "rm /tmp/$db_name.tar" echo + echo "** Cleaning up local dump file" + rm $db_name.tar + elif [ "$direction" = "export" ]; then echo "** Starting pg_dump on local container" docker exec $local_container bash -c "pg_dump -U $db_name -F t --no-privileges --no-owner -c -d $db_name > /tmp/$db_name.tar" @@ -131,24 +134,30 @@ elif [ "$direction" = "export" ]; then docker cp $local_container:/tmp/$db_name.tar ./ echo + echo "** Preparing .tar file for OpenShift pod" + mkdir -p tmp_transfer + mv $db_name.tar tmp_transfer/ + echo + echo "** Uploading .tar file to OpenShift pod" - oc rsync ./$db_name.tar $pod_name:/tmp/ + oc rsync ./tmp_transfer $pod_name:/tmp/ echo echo "** Restoring database on OpenShift pod" - oc exec $pod_name -- bash -c "pg_restore -U postgres --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/$db_name.tar" || true + oc exec $pod_name -- bash -c "pg_restore -U postgres --dbname=$db_name --no-owner --clean --if-exists --verbose /tmp/tmp_transfer/$db_name.tar" || true + echo + + echo "** Cleaning up temporary files on OpenShift pod" + oc exec $pod_name -- bash -c "rm -rf /tmp/tmp_transfer" echo echo "** Cleaning up dump file from local container" - docker exec $local_container bash -c "rm /tmp/$db_name.tar" + docker exec $local_container bash -c "rm /tmp/tmp_transfer" || true echo -fi -echo "** Cleaning up dump file from local container" -docker exec $local_container bash -c "rm /tmp/$db_name.tar" || true -echo + echo "** Cleaning up local temporary directory" + rm -rf tmp_transfer +fi -echo "** Cleaning up local dump file" -rm $db_name.tar echo "** Finished data transfer and cleanup" diff --git a/etl/database/nifi-registry-primary.mv.db b/etl/database/nifi-registry-primary.mv.db index c2cf74344..f5b1bf947 100644 Binary files a/etl/database/nifi-registry-primary.mv.db and b/etl/database/nifi-registry-primary.mv.db differ diff --git a/etl/nifi/conf/flow.json.gz b/etl/nifi/conf/flow.json.gz index 06afa1e60..e2e4eaaef 100644 Binary files a/etl/nifi/conf/flow.json.gz and b/etl/nifi/conf/flow.json.gz differ diff --git a/etl/nifi/conf/flow.xml.gz b/etl/nifi/conf/flow.xml.gz index 14109c7fc..203856fcb 100644 Binary files a/etl/nifi/conf/flow.xml.gz and b/etl/nifi/conf/flow.xml.gz differ diff --git a/etl/nifi_scripts/adminAdjTrxn.groovy b/etl/nifi_scripts/adminAdjTrxn.groovy index 5492bbf86..27806df95 100644 --- a/etl/nifi_scripts/adminAdjTrxn.groovy +++ b/etl/nifi_scripts/adminAdjTrxn.groovy @@ -5,6 +5,8 @@ import java.sql.ResultSet import java.time.OffsetDateTime import java.sql.Timestamp +log.warn("**** STARTED ADMIN ADJUSTMENT ETL ****") + def SOURCE_QUERY = """ WITH internal_comment AS ( @@ -433,4 +435,6 @@ def insertadminAdjustment(ResultSet rs, PreparedStatement adminAdjustmentStmt, adminAdjustmentStmt.setInt(11, rs.getInt('admin_adjustment_id')) def result = adminAdjustmentStmt.executeQuery() return result.next() ? result.getInt('admin_adjustment_id') : null -} \ No newline at end of file +} + +log.warn("**** COMPLETED ADMIN ADJUSTMENT ETL ****") diff --git a/etl/nifi_scripts/compliance_report.groovy b/etl/nifi_scripts/compliance_report.groovy index f9345daa3..18ea947a5 100644 --- a/etl/nifi_scripts/compliance_report.groovy +++ b/etl/nifi_scripts/compliance_report.groovy @@ -1014,3 +1014,5 @@ def updateComplianceReportWithTransaction(Connection conn, int lcfsReportId, int // ========================================= // Script Termination // ========================================= + +log.warn("**** COMPLETED COMPLIANCE REPORT ETL ****") diff --git a/etl/nifi_scripts/initiativeAgrmtTrxn.groovy b/etl/nifi_scripts/initiativeAgrmtTrxn.groovy index 8e3f9f121..cb615d037 100644 --- a/etl/nifi_scripts/initiativeAgrmtTrxn.groovy +++ b/etl/nifi_scripts/initiativeAgrmtTrxn.groovy @@ -5,6 +5,8 @@ import java.sql.ResultSet import java.time.OffsetDateTime import java.sql.Timestamp +log.warn("**** STARTING INITIATIVE AGREEMENT ETL ****") + def SOURCE_QUERY = """ WITH internal_comment AS ( @@ -433,4 +435,6 @@ def insertInitiativeAgreement(ResultSet rs, PreparedStatement initiativeAgreemen initiativeAgreementStmt.setInt(11, rs.getInt('initiative_agreement_id')) def result = initiativeAgreementStmt.executeQuery() return result.next() ? result.getInt('initiative_agreement_id') : null -} \ No newline at end of file +} + +log.warn("**** COMPLETED INITIATIVE AGREEMENT ETL ****") diff --git a/etl/nifi_scripts/organization.groovy b/etl/nifi_scripts/organization.groovy index e89e0a21d..d9b826e14 100644 --- a/etl/nifi_scripts/organization.groovy +++ b/etl/nifi_scripts/organization.groovy @@ -3,6 +3,8 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.util.Random +log.warn("**** STARTING ORGANIZATION ETL ****") + // SQL query to fetch data from the source def sourceQuery = """ SELECT @@ -227,3 +229,5 @@ try { if (sourceConn != null) sourceConn.close() if (destinationConn != null) destinationConn.close() } + +log.warn("**** COMPLETED ORGANIZATION ETL ****") diff --git a/etl/nifi_scripts/transfer.groovy b/etl/nifi_scripts/transfer.groovy index 219a63c78..88cdf9cd9 100644 --- a/etl/nifi_scripts/transfer.groovy +++ b/etl/nifi_scripts/transfer.groovy @@ -5,6 +5,8 @@ import java.sql.ResultSet import java.time.OffsetDateTime import java.sql.Timestamp +log.warn("**** STARTING TRANSFER ETL ****") + def SOURCE_QUERY = """ WITH internal_comment AS ( @@ -515,3 +517,4 @@ def insertTransfer(ResultSet rs, PreparedStatement transferStmt, Long fromTransa return result.next() ? result.getInt('transfer_id') : null } +log.warn("**** COMPLETED TRANSFER ETL ****") diff --git a/etl/nifi_scripts/user.groovy b/etl/nifi_scripts/user.groovy index 67b4fee35..77fb9bdf7 100644 --- a/etl/nifi_scripts/user.groovy +++ b/etl/nifi_scripts/user.groovy @@ -2,6 +2,8 @@ import java.sql.Connection import java.sql.PreparedStatement import java.sql.ResultSet +log.warn("**** STARTING USER ETL ****") + // SQL query to extract user profiles def userProfileQuery = """ SELECT id as user_profile_id, @@ -154,3 +156,5 @@ try { if (sourceConn != null) sourceConn.close() if (destinationConn != null) destinationConn.close() } + +log.warn("**** COMPLETED USER ETL ****")