Skip to content

Commit

Permalink
Improvements (#30)
Browse files Browse the repository at this point in the history 
bugfix and adding more queries
  • Loading branch information
@valentin-gauthier-geosiris
valentin-gauthier-geosiris authored Nov 3, 2023
1 parent 205cf10 commit f942ad5
Show file tree
Hide file tree
Showing 5 changed files with 316 additions and 61 deletions.
144 changes: 126 additions & 18 deletions etpclient/etp/requester.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import re
import zipfile
from lxml import etree
from io import BytesIO

from lxml.etree import (
Element,
Expand Down Expand Up @@ -61,12 +62,18 @@
from etptypes.energistics.etp.v12.protocol.discovery.get_resources import (
GetResources,
)
from etptypes.energistics.etp.v12.protocol.discovery.get_deleted_resources import (
GetDeletedResources,
)
from etptypes.energistics.etp.v12.protocol.store.put_data_objects import (
PutDataObjects,
)
from etptypes.energistics.etp.v12.protocol.store.get_data_objects import (
GetDataObjects,
)
from etptypes.energistics.etp.v12.protocol.store.delete_data_objects import (
DeleteDataObjects,
)
from etptypes.energistics.etp.v12.datatypes.object.data_object import (
DataObject,
)
Expand All @@ -88,6 +95,14 @@
GetDataArrayMetadata,
)


from etptypes.energistics.etp.v12.protocol.supported_types.get_supported_types import (
GetSupportedTypes,
)
from etptypes.energistics.etp.v12.protocol.supported_types.get_supported_types_response import (
GetSupportedTypesResponse,
)

from etptypes.energistics.etp.v12.datatypes.data_value import DataValue
from etptypes.energistics.etp.v12.datatypes.object.resource import Resource
from etptypes.energistics.etp.v12.datatypes.object.dataspace import Dataspace
Expand All @@ -100,6 +115,11 @@

from etpclient.etp.h5_handler import generate_put_data_arrays

from etpclient.utils import (
xml_get_type,
get_xml_tree_string,
)


ENERGYML_NAMESPACES = {
"eml": "http://www.energistics.org/energyml/data/commonv2",
Expand Down Expand Up @@ -194,6 +214,16 @@ def find_uuid_in_xml(xml_content: bytes) -> str:
return None


def get_root_type_in_xml(xml_content: bytes) -> str:
try:
tree = ElementTree(fromstring(xml_content))
root = tree.getroot()
return root.tag
except etree.XMLSyntaxError:
print("Error reading xml")
return None


def request_session():
return RequestSession(
applicationName="Geosiris etp client",
Expand Down Expand Up @@ -288,7 +318,7 @@ def put_dataspace(dataspace_names: list):
return PutDataspaces(dataspaces=ds_map)


def delete_dataspace(dataspace_names: str):
def delete_dataspace(dataspace_names: list):
ds_map = {}
for ds_name in dataspace_names:
ds_map[str(len(ds_map))] = (
Expand All @@ -299,7 +329,36 @@ def delete_dataspace(dataspace_names: str):
return DeleteDataspaces(uris=ds_map)


def put_data_object_by_path(path: str, dataspace_name: str = None):
def delete_data_object(uris: list):
print(
"Sending delete_data_object : ", {i: uris[i] for i in range(len(uris))}
)
return DeleteDataObjects(
uris={i: uris[i] for i in range(len(uris))},
prune_contained_objects=False,
)


def get_deleted_resources(
dataspace_names: str,
delete_time_filter: int = None,
data_object_types: list = [],
):
ds_uri = (
"eml:///dataspace('" + dataspace_names + "')"
if "eml:///" not in dataspace_names
else dataspace_names
)
return GetDeletedResources(
dataspace_uri=ds_uri,
delete_time_filter=delete_time_filter,
data_object_types=data_object_types,
)


def put_data_object_by_path(
path: str, dataspace_name: str = None, uuids_filter: list = None
):
result = []
# try:
if path.endswith(".xml"):
Expand All @@ -317,9 +376,14 @@ def put_data_object_by_path(path: str, dataspace_name: str = None):
# print('%s (%s --> %s)' % (zinfo.filename, zinfo.file_size, zinfo.compress_size))
with zfile.open(zinfo.filename) as myfile:
file_content = myfile.read()
if (
findUuid(zinfo.filename) is not None
or find_uuid_in_xml(file_content) is not None
uuid = findUuid(zinfo.filename)
if uuid is None:
uuid = find_uuid_in_xml(file_content)
print(f"UUID {uuid}")
if uuid is not None and (
uuids_filter is None
or len(uuids_filter) == 0
or uuid in uuids_filter
):
do_lst[len(do_lst)] = _create_data_object(
file_content.decode("utf-8"),
Expand Down Expand Up @@ -388,6 +452,37 @@ def get_close_session(reason="We have finished"):
return CloseSession(reason=reason)


# _____ __ ________
# / ___/__ ______ ____ ____ _____/ /____ ____/ /_ __/_ ______ ___ _____
# \__ \/ / / / __ \/ __ \/ __ \/ ___/ __/ _ \/ __ / / / / / / / __ \/ _ \/ ___/
# ___/ / /_/ / /_/ / /_/ / /_/ / / / /_/ __/ /_/ / / / / /_/ / /_/ / __(__ )
# /____/\__,_/ .___/ .___/\____/_/ \__/\___/\__,_/ /_/ \__, / .___/\___/____/
# /_/ /_/ /____/_/


def get_supported_types(
uri: str,
count: bool = True,
return_empty_types: bool = True,
scope: str = "self",
):
if not uri.startswith("eml:///"):
uri = f"eml:///dataspace('{uri}')"
if isinstance(count, str):
count = count.lower() == "true"
if isinstance(return_empty_types, str):
return_empty_types = return_empty_types.lower() == "true"
print(
f"==> uri={uri}, count={count}, return_empty_types={return_empty_types}"
)
return GetSupportedTypes(
uri=uri,
count_objects=count,
return_empty_types=return_empty_types,
scope=get_scope(scope),
)


# ____ __ ___
# / __ \____ _/ /_____ _/ | ______________ ___ __
# / / / / __ `/ __/ __ `/ /| | / ___/ ___/ __ `/ / / /
Expand Down Expand Up @@ -433,6 +528,7 @@ def put_data_array(
h5_file_path: str,
dataspace_name: str,
):
print("FILE ", epc_or_xml_file_path)
result = []
if epc_or_xml_file_path.endswith(".epc"):
zfile = zipfile.ZipFile(epc_or_xml_file_path, "r")
Expand All @@ -447,13 +543,13 @@ def put_data_array(
or len(uuids_filter) == 0
or uuid in uuids_filter
):
# print("Uuid : ", uuid)
with zfile.open(zinfo.filename) as myfile:
result += generate_put_data_arrays(
myfile.read().decode("utf-8"),
h5_file_path,
dataspace_name,
)
print("> Uuid filtered: ", uuid)
# with zfile.open(zinfo.filename) as myfile:
# result += generate_put_data_arrays(
# myfile.read().decode("utf-8"),
# h5_file_path,
# dataspace_name,
# )
else:
pass
# print("Not imported ", uuid)
Expand All @@ -472,9 +568,10 @@ async def put_data_array_sender(
epc_or_xml_file_path: str,
h5_file_path: str,
dataspace_name: str,
type_filter: str = None,
):
print(
f"uuids_filter : {uuids_filter} epc_or_xml_file_path : {epc_or_xml_file_path} h5_file_path : {h5_file_path} dataspace_name : {dataspace_name} "
f"uuids_filter : {uuids_filter} epc_or_xml_file_path : {epc_or_xml_file_path} h5_file_path : {h5_file_path} dataspace_name : {dataspace_name} type_filter : {type_filter} "
)
if epc_or_xml_file_path.endswith(".epc"):
zfile = zipfile.ZipFile(epc_or_xml_file_path, "r")
Expand All @@ -484,26 +581,37 @@ async def put_data_array_sender(
and findUuid(zinfo.filename) is not None
):
uuid = findUuid(zinfo.filename)
if (
accept_file = (
uuids_filter is None
or len(uuids_filter) == 0
or uuid in uuids_filter
):
print("Uuid : ", uuid)
)
if type_filter is not None:
with zfile.open(zinfo.filename) as myfile:
file_content = myfile.read()
file_type = xml_get_type(
get_xml_tree_string(file_content)
)
accept_file = accept_file or re.match(
type_filter, file_type
)

if accept_file:
print(" > accept_file Uuid : ", uuid)
with zfile.open(zinfo.filename) as myfile:
for pda in generate_put_data_arrays(
myfile.read().decode("utf-8"),
h5_file_path,
dataspace_name,
):
print(type(pda), pda)
# print(type(pda), pda)
try:
yield await websocket.send_no_wait(pda)
except Exception as e:
print("ERROR : ", e)
else:
print("Not imported ", uuid, " -- ", uuid in uuids_filter)
pass
# print("Not imported ", uuid)
zfile.close()
else:
with open(epc_or_xml_file_path) as f:
Expand Down
Loading

0 comments on commit f942ad5

Please sign in to comment.