Merge pull request #26 from dqops/1.4.0

1.4.0

.run/dqo run.run.xml

@@ -5,7 +5,7 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.3.0.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.4.0.jar" />
     <option name="VM_PARAMETERS" value="-XX:MaxRAMPercentage=60.0 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED" />
     <option name="PROGRAM_PARAMETERS" value="--server.port=8888" />
     <option name="WORKING_DIRECTORY" value="C:\dev\dqoado" />

CHANGELOG.md

@@ -1,4 +1,7 @@
-# 1.3.0
-* Screens that show a list of tables and the data quality status divided by data quality dimensions
-* Analyze nested values in JSON fields
-* Analyze files on Azure storage buckets, using the az:// prefix.
+# 1.4.0
+* Ensure that only one instance of DuckDB engine is loaded
+* Profiling and monitoring checks do not render the group by time period (current time)
+* Small UI fixes
+* Default check patterns (profiles) support disabling and targeting multiple tables and columns
+* Global search screens to show tables and columns, filtered by labels
+* Data quality check recalibration for failed data quality checks initiated from an incident details screen

Dockerfile

@@ -68,6 +68,7 @@ COPY --from=dqo-home /dqo/home home
 
 COPY distribution/dqo_docker_entrypoint.sh /dqo/home/bin/
 RUN chmod +x /dqo/home/bin/dqo_docker_entrypoint.sh
+RUN mkdir -p /etc/pki/tls/certs && ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
 
 # copy spring dependencies
 ARG DEPENDENCY=/workspace/app/dqops/target/dependency

Dockerfile-fast

@@ -48,6 +48,7 @@ COPY --from=dqo-home /dqo/home home
 
 COPY distribution/dqo_docker_entrypoint.sh /dqo/home/bin/
 RUN chmod +x /dqo/home/bin/dqo_docker_entrypoint.sh
+RUN mkdir -p /etc/pki/tls/certs && ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
 
 # copy spring dependencies
 ARG DEPENDENCY=/workspace/app/home

VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

distribution/dqo_docker_entrypoint.sh

@@ -16,7 +16,7 @@
 #
 
 if [ -z "$DQO_JAVA_OPTS" ]; then
-  export DQO_JAVA_OPTS=-XX:MaxRAMPercentage=80.0
+  export DQO_JAVA_OPTS="-XX:MaxRAMPercentage=80.0"
 fi
 
 _term() {

distribution/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-distribution</artifactId>
-    <version>1.3.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
+    <version>1.4.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
 	<name>dqo-distribution</name>
 	<description>DQOps Data Quality Operations Center final assembly</description>
 	<packaging>pom</packaging>

distribution/python/dqops/client/api/columns/get_columns.py

@@ -5,24 +5,40 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
+from ...models.check_type import CheckType
 from ...models.column_list_model import ColumnListModel
-from ...types import Response
+from ...types import UNSET, Response, Unset
 
 
 def _get_kwargs(
     connection_name: str,
     schema_name: str,
     table_name: str,
+    *,
+    data_quality_status: Union[Unset, None, bool] = UNSET,
+    check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Dict[str, Any]:
     pass
 
+    params: Dict[str, Any] = {}
+    params["dataQualityStatus"] = data_quality_status
+
+    json_check_type: Union[Unset, None, str] = UNSET
+    if not isinstance(check_type, Unset):
+        json_check_type = check_type.value if check_type else None
+
+    params["checkType"] = json_check_type
+
+    params = {k: v for k, v in params.items() if v is not UNSET and v is not None}
+
     return {
         "method": "get",
         "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/columns".format(
             connectionName=connection_name,
             schemaName=schema_name,
             tableName=table_name,
         ),
+        "params": params,
     }
 
 
@@ -61,6 +77,8 @@ def sync_detailed(
     table_name: str,
     *,
     client: AuthenticatedClient,
+    data_quality_status: Union[Unset, None, bool] = UNSET,
+    check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Response[List["ColumnListModel"]]:
     """getColumns
 
@@ -70,6 +88,8 @@ def sync_detailed(
         connection_name (str):
         schema_name (str):
         table_name (str):
+        data_quality_status (Union[Unset, None, bool]):
+        check_type (Union[Unset, None, CheckType]):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -83,6 +103,8 @@ def sync_detailed(
         connection_name=connection_name,
         schema_name=schema_name,
         table_name=table_name,
+        data_quality_status=data_quality_status,
+        check_type=check_type,
     )
 
     response = client.get_httpx_client().request(
@@ -98,6 +120,8 @@ def sync(
     table_name: str,
     *,
     client: AuthenticatedClient,
+    data_quality_status: Union[Unset, None, bool] = UNSET,
+    check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Optional[List["ColumnListModel"]]:
     """getColumns
 
@@ -107,6 +131,8 @@ def sync(
         connection_name (str):
         schema_name (str):
         table_name (str):
+        data_quality_status (Union[Unset, None, bool]):
+        check_type (Union[Unset, None, CheckType]):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -121,6 +147,8 @@ def sync(
         schema_name=schema_name,
         table_name=table_name,
         client=client,
+        data_quality_status=data_quality_status,
+        check_type=check_type,
     ).parsed
 
 
@@ -130,6 +158,8 @@ async def asyncio_detailed(
     table_name: str,
     *,
     client: AuthenticatedClient,
+    data_quality_status: Union[Unset, None, bool] = UNSET,
+    check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Response[List["ColumnListModel"]]:
     """getColumns
 
@@ -139,6 +169,8 @@ async def asyncio_detailed(
         connection_name (str):
         schema_name (str):
         table_name (str):
+        data_quality_status (Union[Unset, None, bool]):
+        check_type (Union[Unset, None, CheckType]):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -152,6 +184,8 @@ async def asyncio_detailed(
         connection_name=connection_name,
         schema_name=schema_name,
         table_name=table_name,
+        data_quality_status=data_quality_status,
+        check_type=check_type,
     )
 
     response = await client.get_async_httpx_client().request(**kwargs)
@@ -165,6 +199,8 @@ async def asyncio(
     table_name: str,
     *,
     client: AuthenticatedClient,
+    data_quality_status: Union[Unset, None, bool] = UNSET,
+    check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Optional[List["ColumnListModel"]]:
     """getColumns
 
@@ -174,6 +210,8 @@ async def asyncio(
         connection_name (str):
         schema_name (str):
         table_name (str):
+        data_quality_status (Union[Unset, None, bool]):
+        check_type (Union[Unset, None, CheckType]):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -189,5 +227,7 @@ async def asyncio(
             schema_name=schema_name,
             table_name=table_name,
             client=client,
+            data_quality_status=data_quality_status,
+            check_type=check_type,
         )
     ).parsed

distribution/python/dqops/client/api/incidents/disable_checks_for_incident.py

@@ -0,0 +1,127 @@
+from http import HTTPStatus
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...types import Response
+
+
+def _get_kwargs(
+    connection_name: str,
+    year: int,
+    month: int,
+    incident_id: str,
+) -> Dict[str, Any]:
+    pass
+
+    return {
+        "method": "post",
+        "url": "api/incidents/{connectionName}/{year}/{month}/{incidentId}/checks/disable".format(
+            connectionName=connection_name,
+            year=year,
+            month=month,
+            incidentId=incident_id,
+        ),
+    }
+
+
+def _parse_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Optional[Any]:
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Response[Any]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    connection_name: str,
+    year: int,
+    month: int,
+    incident_id: str,
+    *,
+    client: AuthenticatedClient,
+) -> Response[Any]:
+    """disableChecksForIncident
+
+     Disables all data quality checks that caused a given data quality incident.
+
+    Args:
+        connection_name (str):
+        year (int):
+        month (int):
+        incident_id (str):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Any]
+    """
+
+    kwargs = _get_kwargs(
+        connection_name=connection_name,
+        year=year,
+        month=month,
+        incident_id=incident_id,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio_detailed(
+    connection_name: str,
+    year: int,
+    month: int,
+    incident_id: str,
+    *,
+    client: AuthenticatedClient,
+) -> Response[Any]:
+    """disableChecksForIncident
+
+     Disables all data quality checks that caused a given data quality incident.
+
+    Args:
+        connection_name (str):
+        year (int):
+        month (int):
+        incident_id (str):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Any]
+    """
+
+    kwargs = _get_kwargs(
+        connection_name=connection_name,
+        year=year,
+        month=month,
+        incident_id=incident_id,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)