[FEATURE] Faz upload de batches de dados sequencialmente (#550)

* feat: upload stuff in batches

* feat: clear partitions before uploading

* fix: add missing upstream deps

* feat: upload full batch at once

* fix: get next batch, feat: delete multiple blobs at once

* chore: use datetime as suffix for data files

* fix: `delete_blobs_list` args

* fix: `delete_blobs_list` args

* chore: better logging in dump_upload_batches

* feat: add paramenter log_number_of_batches in the flow

* feat: add paramenter log_number_of_batches in the flow

* chore: refactor for using pyodbc instead of pymssql

* fix: install unixodbc-dev

* fix: install odbc driver

* chore: improve Docker image

* fix: disable cert check

* chore: trigger dump_db new staging register

* fix: try to fix pyodbc connstring

* fix: add port in pyodbc

* fix: add port in pyodbc

* chore: log pyodbc drivers

* fix: rollback to odbc17, trust server cert

* chore(ci): delete previous comments

* fix: modify min tls version

* chore: new connstring

* fix: rollback debian version

---------

Co-authored-by: d116626 <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -43,6 +43,12 @@ jobs:
         continue-on-error: true
         run: |
           python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}"
+      - name: Delete previous comments
+        uses: izhangzhihao/delete-comment@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          delete_user_name: github-actions[bot]
+          issue_number: ${{ github.event.number }}  # remove comments from the current PR
       - name: Comment PR
         uses: thollander/actions-comment-pull-request@v1
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # Build arguments
-ARG PYTHON_VERSION=3.9-slim
+ARG PYTHON_VERSION=3.9-slim-buster
 
 # Get Oracle Instant Client
 FROM curlimages/curl:7.81.0 as curl-step
@@ -22,17 +22,24 @@ FROM python:${PYTHON_VERSION}
 # Setting environment with prefect version
 ARG PREFECT_VERSION=0.15.9
 ENV PREFECT_VERSION $PREFECT_VERSION
+ENV DEBIAN_FRONTEND=noninteractive
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Setting environment with Oracle Instant Client URL
 
-# Setup Oracle Instant Client
+# Setup Oracle Instant Client and SQL Server ODBC Driver
 WORKDIR /opt/oracle
 COPY --from=unzip-step /tmp/instantclient_21_5 /opt/oracle/instantclient_21_5
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y libaio1 && \
+    apt-get install --no-install-recommends -y curl gnupg2 libaio1 && \
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
+    echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list && \
+    apt-get update && \
+    ACCEPT_EULA=Y apt-get install --no-install-recommends -y msodbcsql17 openssl unixodbc-dev && \
     rm -rf /var/lib/apt/lists/* && \
     sh -c "echo /opt/oracle/instantclient_21_5 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \
     ldconfig
+COPY ./openssl.cnf /etc/ssl/openssl.cnf
 
 # Setup virtual environment and prefect
 ENV VIRTUAL_ENV=/opt/venv