Merge pull request #547 from OP-TED/feature/TED4-170-feedback

Feature/ted4 170 feedback

.gitattributes

@@ -0,0 +1 @@
+*.jar filter=lfs diff=lfs merge=lfs -text

.github/workflows/unit-tests.yml

@@ -25,6 +25,7 @@ jobs:
           python-version: 3.8
       - name: Install dependencies
         run: |
+          sudo apt-get update
           sudo apt-get install -y libssl-dev libcurl4-openssl-dev
           python -m pip install --upgrade setuptools pip wheel
           make install

.gitignore

@@ -118,4 +118,5 @@ package.json
 .limes/*
 *.ser
 .DS_Store
-.scannerwork/*
+.scannerwork/*
+/infra/alpine/libraries/

Makefile

@@ -95,6 +95,7 @@ create-env-airflow:
 	@ cp requirements.txt ./infra/airflow/
 	@ cp -r ted_sws ./infra/airflow/
 	@ cp -r dags ./infra/airflow/
+	@ cp -r libraries ./infra/airflow/
 
 
 build-airflow: guard-ENVIRONMENT create-env-airflow build-externals

dags/pipelines/notice_processor_pipelines.py

@@ -38,7 +38,8 @@ def notice_transformation_pipeline(notice: Notice, mongodb_client: MongoClient)
             message=f"This notice {notice.ted_id} is not eligible for transformation. Notice info: "
                     f"form_number=[{notice.normalised_metadata.form_number}],"
                     f" eform_subtype=[{notice.normalised_metadata.eforms_subtype}], "
-                    f"xsd_version=[{notice.normalised_metadata.xsd_version}]. Check mapping suites!",
+                    f"xsd_version=[{notice.normalised_metadata.xsd_version}], "
+                    f"eform_sdk_version=[{notice.normalised_metadata.eform_sdk_version}]. Check mapping suites!",
             notice_id=notice.ted_id, domain_action=notice_transformation_pipeline.__name__, notice_status=notice.status,
             notice_form_number=notice.normalised_metadata.form_number,
             notice_eforms_subtype=notice.normalised_metadata.eforms_subtype)

filters/fortify-exclusion.properties

@@ -0,0 +1 @@
+excludePatterns=**/src/docs/**/*,**/src/tests/**/*

filters/odc-exclusion.properties

@@ -0,0 +1 @@
+excludePatterns=**/src/docs/**/*,**/src/tests/**/*

infra/airflow-cluster/Dockerfile

@@ -15,6 +15,8 @@ RUN apt-get update && apt-get install -y \
 # back to normal user
 USER airflow
 
+COPY libraries /home/airflow
+
 # requirements.txt shall be made availble from the **ted-sws** GitHub repository
 COPY requirements.txt /opt/airflow
 

infra/airflow/Dockerfile

@@ -15,11 +15,13 @@ RUN apt-get update && apt-get install -y \
 # back to normal user
 USER airflow
 
+COPY libraries /home/airflow
 # requirements.txt shall be made availble from the **ted-sws** GitHub repository
 COPY requirements.txt /opt/airflow
 
 # working in the /opt/airflow
 WORKDIR /opt/airflow
+
 RUN mkdir -p ./dags ./ted_sws
 
 

libraries/.saxon/notices/JAMESCLARK.txt

@@ -0,0 +1,31 @@
+
+(This notice is included in the Saxon distribution because Saxon's XPath parser
+was originally derived from an XPath parser written by James Clark and made available
+under this license. The Saxon XPath parser has since diverged very substantially, but
+there are traces of the original code still present.)
+
+Copyright (c) 1998, 1999 James Clark
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL JAMES CLARK BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of James Clark shall
+not be used in advertising or otherwise to promote the sale, use or
+other dealings in this Software without prior written authorization
+from James Clark.

libraries/.saxon/notices/JLINE2.txt

@@ -0,0 +1,39 @@
+(This notice is included in the Saxon distribution because the JLine2
+library is included, unmodified, in the Saxon distribution. It is used
+to support interactive command line editing, autocompletion, and session
+history by the Gizmo utility.)
+
+Copyright (c) 2002-2016, the original author or authors.
+All rights reserved.
+
+http://www.opensource.org/licenses/bsd-license.php
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the following
+conditions are met:
+
+Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with
+the distribution.
+
+Neither the name of JLine nor the names of its contributors
+may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.