diff --git a/scripts/check_files_compliance.py b/scripts/check_files_compliance.py index 0555aee3c1..d14574cb39 100755 --- a/scripts/check_files_compliance.py +++ b/scripts/check_files_compliance.py @@ -137,7 +137,8 @@ def ignore(root, file): files_names = ['.gitignore', '.valgrindrc', '.valgrindSuppressions', 'LICENSE', 'ContributionPolicy.txt', 'CHANGES_NEXT_RELEASE', 'Changelog', 'compileInfo.h', 'unittests_that_fail_sporadically.txt', 'Vagrantfile', 'contextBroker.ubuntu', - 'mkdocs.yml', 'fiware-ngsiv2-reference.errata', 'ServiceRoutines.txt', '.readthedocs.yml', 'uncrustify.cfg' ] + 'mkdocs.yml', 'fiware-ngsiv2-reference.errata', 'ServiceRoutines.txt', '.readthedocs.yml', 'uncrustify.cfg', + 'requirements.txt'] if file in files_names: return True if 'scripts' in root and (file == 'cpplint.py' or file == 'pdi-pep8.py' or file == 'uncrustify.cfg' \ diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md new file mode 100644 index 0000000000..30474120a6 --- /dev/null +++ b/scripts/entities_consistency/README.md @@ -0,0 +1,176 @@ +The entities consistency script analyze the contents of the entities database in orion DBs and +check several consistency rules, reporting violations found. + +Ref: [entity document database model]([../../doc/manuals/admin/database_model.md#entities-collection]) + +## Requirements + +This script is designed to work with Python 3. Install the dependencies in the `requirements.txt` file before using it. +Usage of virtual env is recommended. + +## Usage + +Run `entities_consistency.py -h` for arguments details. + +## Rules + +* Rules 1x: DB inconsistencies (use to be severe problems) +* Rules 2x: Syntax restrictions +* Rules 9x: Usage of legacy features + +### Rule 10: `_id` field consistency + +Each entity in DB has an `_id` field with three subfields: + +* `id` +* `type` +* `servicePath` + +### Rule 11: mandatory fields in entity + +The following fields are mandatory: + +* `attrNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. + +### Rule 12: mandatory fields in attribute + +The following subfields are mandatory for every attribute: + +* `mdNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. + +### Rule 13: `attrNames` field consistency + +For each item in `attrNames` array there is a corresponding key in `attrs` object and the other way around. + +### Rule 14: `mdNames` field consistency + +For every attribute, for each item in `mdNames` array there is a corresponding key in `md` object and the other way around. + +### Rule 15: not swapped subkeys in `_id` + +In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with +`_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` +equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` uniqueness constraint. + +This rule checks that this is not happening in the entities collection. + +### Rule 16: `location` field consistency + +Check that location in consistent. In particular: + +* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type (without `ignoreTypes` and without `null` value). +* If one of such attribute is found, check the `location` field is found and its content is consistent +* If none of such attribute is found, check no `location` field is not found + * If the location of the entity is defined using deprecated `location` metadata it will be detected as this case. Additional rules in the 9x group can help to diagnose this situation. + +This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. + +### Rule 17: `lastCorrelator` existence + +Check if `lastCorrelator` is included. + +This field was introduced in [Orion 1.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/1.8.0) (released in September 11th, 2017). + +### Rule 20: entity id syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 21: entity type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 22: entity servicePath syntax + +Check [servicePath documentation](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path) + +### Rule 23: attribute name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 24: attribute type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 25: metadata name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 26: metadata type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` + +Check usage of deprecated geo-location types, i.e: + +* `geo:point` +* `geo:line` +* `geo:box` +* `geo:polygon` + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field + +Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group. + +### Rule 91: detect usage of more than one legacy `location` metadata + +Check usage of `location` in more than one attribute of the same entity. + +Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` + +The value of the `location` metadata should be `WGS84` or `WSG84`. + +Additional consideration: + +* Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 93: detect usage of redundant legacy `location` + +Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the +same attribute. + +Suggested action is to remove the `location` metadata. + +Additional, considerations: + + * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. + * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 94: detect usage of not redundant legacy `location` + +Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. +same attribute. + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field +* Remove the `location` metadata from the attribute + +Additional, considerations: + +* This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +## Testing + +You can test the `entities_consistency.py` script this qy: + +1. Populate `orion-validation` DB with testing document. To do so, copy-paste the content of the `validation_data.js` in `mongosh` +2. Run `test_entities_consistency.py` test and check the log output + +You can also run `test_entities_consistenct.py` under coverage to check every rule is covering all the possible validation cases. diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/entities_consistency/entities_consistency.py new file mode 100644 index 0000000000..fd375cbccc --- /dev/null +++ b/scripts/entities_consistency/entities_consistency.py @@ -0,0 +1,876 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +from pymongo import MongoClient +from deepdiff import DeepDiff +import argparse +import logging +import json +import sys +import re + + +# Helper functions + +def is_geo_type(attr_type): + """ + Return True if attr type passed as argument is a geo type + + :param attr_type: the attr type to evaluate + """ + + return attr_type == 'geo:point' or attr_type == 'geo:line' or attr_type == 'geo:box' \ + or attr_type == 'geo:polygon' or attr_type == 'geo:json' + + +def ignore_type(attr): + """ + Return true if attribute has the ignoreType metadata + """ + return 'md' in attr and 'ignoreType' in attr['md'] + + +def to_geo_json(attr): + """ + Return the GeoJSON corresponding to an attribute location, taking into account the type + + Useful ref: https://github.com/telefonicaid/fiware-orion/blob/3.9.0/doc/manuals/orion-api.md + """ + + if attr['type'] == 'geo:point': + # "value": "41.3763726, 2.186447514", + coords = attr['value'].split(",") + return { + 'type': 'Point', + 'coordinates': [float(coords[1]), float(coords[0])] + } + elif attr['type'] == 'geo:line': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + coordinates.append([float(coords[1]), float(coords[0])]) + return { + 'type': 'LineString', + 'coordinates': coordinates + } + elif attr['type'] == 'geo:box': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + # The first pair is the lower corner, the second is the upper corner. + lower_corner_0 = attr['value'][0].split(",") + upper_corner_0 = attr['value'][1].split(",") + lower_corner_1 = [upper_corner_0[0], lower_corner_0[1]] + upper_corner_1 = [lower_corner_0[0], upper_corner_0[1]] + return { + 'type': 'Polygon', + 'coordinates': [[ + [float(lower_corner_0[1]), float(lower_corner_0[0])], + [float(lower_corner_1[1]), float(lower_corner_1[0])], + [float(upper_corner_0[1]), float(upper_corner_0[0])], + [float(upper_corner_1[1]), float(upper_corner_1[0])], + [float(lower_corner_0[1]), float(lower_corner_0[0])] + ]] + } + elif attr['type'] == 'geo:polygon': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363", + # "40.63913831188419, -8.653321266174316" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + last = [float(coords[1]), float(coords[0])] + coordinates.append([float(coords[1]), float(coords[0])]) + coords.append(last) # so we have a closed shape + return { + 'type': 'Polygon', + 'coordinates': [coordinates] + } + elif attr['type'] == 'geo:json': + return attr['value'] + else: + logger.error(f"unknown geo location type: {attr['type']}") + return None + + +def convert_strings_to_numbers(data): + """ + Generated by ChatGPT :) + """ + if isinstance(data, dict): + # If it's a dictionary, apply the function recursively to its values + return {key: convert_strings_to_numbers(value) for key, value in data.items()} + elif isinstance(data, list): + # If it's a list, apply the function recursively to its elements + return [convert_strings_to_numbers(item) for item in data] + elif isinstance(data, str): + # If it's a string, try converting it to a number + try: + return int(data) + except ValueError: + try: + return float(data) + except ValueError: + # If it's not a valid number, leave it as it is + return data + else: + # If it's neither a dictionary, list, nor string, leave it as it is + return data + + +def check_id(id): + """ + Common checks for several rules + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#identifiers-syntax-restrictions + """ + # Minimum field length is 1 character + l = len(id) + if l == 0: + return f'length ({l}) shorter than minimum allowed (1)' + # Maximum field length is 256 characters + if l > 256: + return f'length ({l}) greater than maximum allowed (256)' + + # Following chars are not used (general): < > " ' = ; ( ) + if re.search('[<>"\'=;()]', id): + return f'contains forbidden chars (general)' + + # Following chars are not used (identifiers): whitespace, & ? / # + if re.search('[\s&?/#]', id): + return f'contains forbidden chars (identifiers)' + + return None + + +# Rules functions +def rule10(entity): + """ + Rule 10: `_id` field consistency + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['id', 'type', 'servicePath']: + if field not in entity['_id']: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing subfields in _id: {', '.join(missing_fields)}" + else: + return None + + +def rule11(entity): + """ + Rule 11: mandatory fields in entity + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['attrNames', 'creDate', 'modDate']: + if field not in entity: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing fields: {', '.join(missing_fields)}" + else: + return None + + +def rule12(entity): + """ + Rule 12: mandatory fields in attribute + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + missing_fields = [] + + for field in ['mdNames', 'creDate', 'modDate']: + if field not in entity['attrs'][attr]: + missing_fields.append(field) + + if len(missing_fields) > 0: + s.append(f"in attribute '{attr}' missing fields: {', '.join(missing_fields)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule13(entity): + """ + Rule 13: `attrNames` field consistency + + See README.md for an explanation of the rule + """ + # note the omission of attrNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + # attrNames in attrs + attrnames_not_in_attrs = [] + if 'attrNames' in entity: + for attr in entity['attrNames']: + if attr.replace('.', '=') not in entity['attrs']: + attrnames_not_in_attrs.append(attr) + + # attrs in attrNames + attrs_not_in_attrnames = [] + if 'attrs' in entity: + for attr in entity['attrs']: + if 'attrNames' not in entity or attr.replace('=', '.') not in entity['attrNames']: + attrs_not_in_attrnames.append(attr) + + s = [] + if len(attrnames_not_in_attrs) > 0: + s.append(f"attributes in attrNames not found in attrs object: {','.join(attrnames_not_in_attrs)}") + if len(attrs_not_in_attrnames) > 0: + s.append(f"attributes in attrs object not found in attrNames: {','.join(attrs_not_in_attrnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule14(entity): + """ + Rule 14: `mdNames` field consistency + + See README.md for an explanation of the rule + """ + # note the omission of mdNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + s = [] + for item in entity['attrs']: + attr = entity['attrs'][item] + # mdNames in md + if 'mdNames' in attr: + mdnames_not_in_md = [] + for md in attr['mdNames']: + if md.replace('.', '=') not in attr['md']: + mdnames_not_in_md.append(md) + + # md in mdNames + md_not_in_mdnames = [] + if 'md' in attr: + for md in attr['md']: + if 'mdNames' not in attr or md.replace('=', '.') not in attr['mdNames']: + md_not_in_mdnames.append(md) + + if len(mdnames_not_in_md) > 0: + s.append( + f"in attribute '{item}' metadata in mdNames not found in md object: {', '.join(mdnames_not_in_md)}") + if len(md_not_in_mdnames) > 0: + s.append( + f"in attribute '{item}' metadata in md object not found in mdNames: {', '.join(md_not_in_mdnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule15(entities_collection): + """ + Rule 15: not swapped subkeys in `_id` + + See README.md for an explanation of the rule + + This is a global rule, so the parameter is not an individual entity but the whole entities collection. + """ + s = [] + for entity in entities_collection.aggregate( + [ + { + '$group': { + '_id': {'id': '$_id.id', 'type': '$_id.type', 'servicePath': '$_id.servicePath'}, + 'count': {'$sum': 1} + } + }, + { + '$match': {'count': {'$gt': 1}} + } + ] + ): + id = entity['_id']['id'] + type = entity['_id']['type'] + service_path = entity['_id']['servicePath'] + count = entity['count'] + s.append( + f"_id uniqueness violation for entity id='{id}' type='{type}' servicePath='{service_path}' found {count} times") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule16(entity): + """ + Rule 16: `location` field consistency + + See README.md for an explanation of the rule + """ + # check that as much as one attribute is using geo type + geo_attrs = [] + for attr in entity['attrs']: + # type existence in attribute is checked by another rule + if 'type' in entity['attrs'][attr] and is_geo_type(entity['attrs'][attr]['type']) and not ignore_type(entity['attrs'][attr]): + geo_attrs.append(attr) + + if len(geo_attrs) > 1: + return f"more than one attribute with geo type: {', '.join(geo_attrs)}" + + if len(geo_attrs) == 1: + # If geo attr found, then check that there is consistent location field + geo_attr = geo_attrs[0] + geo_type = entity['attrs'][geo_attr]['type'] + if entity['attrs'][geo_attr]['value'] is None: + # if null value in geolocation attribute, then location field must not be present + if 'location' in entity: + return f"geo location '{geo_attr}' ({geo_type}) with null value and location field found" + else: + # not null value in geo location attribute case + if 'location' not in entity: + return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity" + if entity['location']['attrName'] != geo_attr: + return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" + + geo_json = to_geo_json(entity['attrs'][geo_attr]) + + # https://www.testcult.com/deep-comparison-of-json-in-python/ + diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) + if diff: + # A typical difference is that attribute value uses strings and location uses numbers + # (this happens when the location was created/updated using NGSIv1). We try to identify that case + geo_json = convert_strings_to_numbers(geo_json) + if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ + f"should use numbers for coordinates instead of strings" + else: + # Other causes + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" + else: # len(geo_attrs) == 0 + # If no geo attr found, check there isn't a location field + if 'location' in entity: + return f"location field detected but no geo attribute is present (maybe metadata location is used?)" + + +def rule17(entity): + """ + Rule 17: `lastCorrelator` existence + + See README.md for an explanation of the rule + """ + if 'lastCorrelator' not in entity: + return f"missing lastCorrelator" + else: + return None + + +def rule20(entity): + """ + Rule 20: entity id syntax + + See README.md for an explanation of the rule + """ + + # The existence of id in _id is checked by another rule + if 'id' in entity['_id']: + r = check_id(entity['_id']['id']) + if r is not None: + return f"entity id ({entity['_id']['id']}) syntax violation: {r}" + + return None + + +def rule21(entity): + """ + Rule 21: entity type syntax + + See README.md for an explanation of the rule + """ + + # The existence of type in _id is checked by another rule + if 'type' in entity['_id']: + r = check_id(entity['_id']['type']) + if r is not None: + return f"entity type ({entity['_id']['type']}) syntax violation: {r}" + + return None + + +def rule22(entity): + """ + Rule 22: entity servicePath syntax + + See README.md for an explanation of the rule + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path + """ + + # servicePath existence is checked by another rule + if 'servicePath' not in entity['_id']: + return None + + sp = entity['_id']['servicePath'] + # Scope must start with / (only "absolute" scopes are allowed) + if not sp.startswith('/'): + return f"servicePath '{sp}' does not starts with '/'" + + # This special case can be problematic (as split will detect always a level and the minimum length rule + # will break. So early return + if sp == '/': + return None + + # 10 maximum scope levels in a path + sp_levels = sp[1:].split('/') + if len(sp_levels) > 10: + return f"servicePath has {len(sp_levels)} tokens but the limit is 10" + + # 50 maximum characters in each level (1 char is minimum), only alphanumeric and underscore allowed + for i in range(len(sp_levels)): + if len(sp_levels[i]) == 0: + return f'servicePath level #{i} length is 0 but minimum is 1' + if len(sp_levels[i]) > 50: + return f'servicePath level #{i} length is {len(sp_levels[i])} but maximum is 50' + if re.search('[^a-zA-Z0-9_]', sp_levels[i]): + return f"unallowed characters in '{sp_levels[i]}' in servicePath level #{i}" + + +def rule23(entity): + """ + Rule 23: attribute name syntax + + See README.md for an explanation of the rule + """ + s = [] + + # attrNames-attrs consistency is checked by another rule. In the present rule we cannot assume + # they are consistent, so we do a union of both sets + attrs_to_check = set() + if 'attrNames' in entity: + attrs_to_check.update(entity['attrNames']) + for attr in entity['attrs']: + attrs_to_check.add(attr.replace('=', '.')) + + # check in the resulting union set + for attr in attrs_to_check: + r = check_id(attr) + if r is not None: + s.append(f"attribute name ({attr}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule24(entity): + """ + Rule 24: attribute type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'type' not in entity['attrs'][attr]: + s.append(f"in attribute '{attr}' type is missing") + else: + type = entity['attrs'][attr]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule25(entity): + """ + Rule 25: metadata name syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + # mdNames-md consistency is checked by another rule. In the present rule we cannot assume + # they are consistent, so we do a union of both sets + md_to_check = set() + if 'mdNames' in entity['attrs'][attr]: + md_to_check.update(entity['attrs'][attr]['mdNames']) + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + md_to_check.add(md.replace('=', '.')) + + # check in the resulting union set + for md in md_to_check: + r = check_id(md) + if r is not None: + s.append(f"in attribute '{attr}' metadata name ({md}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule26(entity): + """ + Rule 26: metadata type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if 'type' not in entity['attrs'][attr]['md'][md]: + s.append(f"in attribute '{attr}' metadata '{md}' type is missing") + else: + type = entity['attrs'][attr]['md'][md]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' metadata '{md}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule90(entity): + """ + Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` + + See README.md for an explanation of the rule + """ + + # note we could have more than one case, as ignoreType can be in use + s = [] + for attr in entity['attrs']: + # type existence in attribute is checked by another rule + if 'type' in entity['attrs'][attr]: + type = entity['attrs'][attr]['type'] + if is_geo_type(type) and type != 'geo:json': + s.append(f"{attr} ({type})") + + # + if len(s) > 0: + return f"usage of deprecated geo type in attributes: {', '.join(s)}" + + +def rule91(entity): + """ + Rule 91: detect usage of more than one legacy `location` metadata + + See README.md for an explanation of the rule + """ + attrs = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md']: + attrs.append(attr) + + if len(attrs) > 1: + return f"location metadata found {len(attrs)} times in attributes: {', '.join(attrs)} (maximum should be just 1)" + else: + return None + + +def rule92(entity): + """ + Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md']: + location_value = entity['attrs'][attr]['md']['location']['value'] + if location_value != 'WGS84' and location_value != 'WSG84': + s.append(f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule93(entity): + """ + Rule 93: detect usage of redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location' and is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' redundant location metadata found (attribute is already using {entity['attrs'][attr]['type']} type)" + + return None + + +def rule94(entity): + """ + Rule 94: detect usage of not redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location' and not is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' location metadata found (attribute type is {entity['attrs'][attr]['type']})" + + return None + + +rules = [ + # Rules 1x + { + 'label': 'Rule10', + 'global': False, + 'func': rule10 + }, + { + 'label': 'Rule11', + 'global': False, + 'func': rule11 + }, + { + 'label': 'Rule12', + 'global': False, + 'func': rule12 + }, + { + 'label': 'Rule13', + 'global': False, + 'func': rule13 + }, + { + 'label': 'Rule14', + 'global': False, + 'func': rule14 + }, + { + 'label': 'Rule15', + 'global': True, + 'func': rule15 + }, + { + 'label': 'Rule16', + 'global': False, + 'func': rule16 + }, + { + 'label': 'Rule17', + 'global': False, + 'func': rule17 + }, + # Rules 2x + { + 'label': 'Rule20', + 'global': False, + 'func': rule20 + }, + { + 'label': 'Rule21', + 'global': False, + 'func': rule21 + }, + { + 'label': 'Rule22', + 'global': False, + 'func': rule22 + }, + { + 'label': 'Rule23', + 'global': False, + 'func': rule23 + }, + { + 'label': 'Rule24', + 'global': False, + 'func': rule24 + }, + { + 'label': 'Rule25', + 'global': False, + 'func': rule25 + }, + { + 'label': 'Rule26', + 'global': False, + 'func': rule26 + }, + # Rules 9x + { + 'label': 'Rule90', + 'global': False, + 'func': rule90 + }, + { + 'label': 'Rule91', + 'global': False, + 'func': rule91 + }, + { + 'label': 'Rule92', + 'global': False, + 'func': rule92 + }, + { + 'label': 'Rule93', + 'global': False, + 'func': rule93 + }, + { + 'label': 'Rule94', + 'global': False, + 'func': rule94 + } +] + + +def process_db(logger, db_name, db_conn, query, rules_exp): + """ + Process an individual DB + + :param logger: logger object + :param db_name: the name of the DB to process + :param db_conn: connection to MongoDB + :param query: query to filter entities to be processed + :param rules_exp: regular expression to filter rules to apply + :return: fails + """ + + logger.info(f'Processing {db_name}') + n = 0 + fails = 0 + + # check collection existence + if 'entities' not in db_conn[db_name].list_collection_names(): + logger.warning(f'collections entities not found in {db_name} database, nothing to do') + return + + # apply global rules + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if rule['global']: + s = rule['func'](db_conn[db_name]['entities']) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation in entities collection: {s}') + fails += 1 + + # apply per-entity rules + for entity in db_conn[db_name]['entities'].find(query): + n += 1 + id_string = json.dumps(entity['_id']) + logger.debug(f'* processing entity {id_string}') + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if not rule['global']: + s = rule['func'](entity) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') + fails += 1 + + logger.info(f'processed {n} entities ({fails} rule violations)') + + return fails + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog='entities_consistency', + description='Check consistency in Orion entities collection in DB') + + parser.add_argument('--mongoUri', dest='mongo_uri', default='mongodb://localhost:27017', + help='MongoDB URI. Default is mongodb://localhost:27017') + parser.add_argument('--db', dest='db', + help='DB name to check. If omitted all DBs starting with "orion" will be checked.') + parser.add_argument('--query', dest='query', default='{}', + help='query to filter entities to check, in JSON MongoDB query language. By default, ' + 'all entities in the collection will be checked.') + parser.add_argument('--rulesExp', dest='rules_exp', + help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') + parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', + help='log level. Default is INFO') + args = parser.parse_args() + + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName(args.log_level), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB + mongo_client = MongoClient(args.mongo_uri) + db_names = mongo_client.list_database_names() + + # to remove starting and trailing ' char, in case it is used + query = json.loads(args.query.replace("'", "")) + + fails = 0 + if args.db is not None: + if args.db in db_names: + fails += process_db(logger, args.db, mongo_client, query, args.rules_exp) + else: + logger.fatal(f'database {args.db} does not exist') + sys.exit(1) + else: + # Process all Orion databases + for db_name in db_names: + if db_name.startswith('orion-'): + fails += process_db(logger, db_name, mongo_client, query, args.rules_exp) + + logger.info(f'total rule violations: {fails}') diff --git a/scripts/entities_consistency/requirements.txt b/scripts/entities_consistency/requirements.txt new file mode 100644 index 0000000000..99c8a008ba --- /dev/null +++ b/scripts/entities_consistency/requirements.txt @@ -0,0 +1,2 @@ +pymongo==4.6.1 +deepdiff==6.7.1 diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/entities_consistency/test_entities_consistency.py new file mode 100644 index 0000000000..eb0319c901 --- /dev/null +++ b/scripts/entities_consistency/test_entities_consistency.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +# This is not the usual test that stimulates function and check assertions in the results :) +# +# Before running this test you have to load the entities testing set (validation_data.js) in the 'orion-validation' +# database in the local MongoDB database (check the MongoURI to match the one in your environment). You will get +# the result as log output. +# +# You can run this test under coverage, so you can check the coverage of the different rules in +# entities_consistency.py + +import unittest +from pymongo import MongoClient +import logging + +from entities_consistency import process_db + +class TestEntitiesConsistency(unittest.TestCase): + def test_process_db(self): + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName('INFO'), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB and process validation DB + mongo_client = MongoClient('mongodb://localhost:27017') + process_db(logger, 'orion-validation', mongo_client, {}, None) diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js new file mode 100644 index 0000000000..dde81aa544 --- /dev/null +++ b/scripts/entities_consistency/validation_data.js @@ -0,0 +1,4208 @@ +/* + # Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U + # + # This file is part of Orion Context Broker. + # + # Orion Context Broker is free software: you can redistribute it and/or + # modify it under the terms of the GNU Affero General Public License as + # published by the Free Software Foundation, either version 3 of the + # License, or (at your option) any later version. + # + # Orion Context Broker is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero + # General Public License for more details. + # + # You should have received a copy of the GNU Affero General Public License + # along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. + # + # For those usages not covered by this license please contact with + # iot_support at tid dot es + */ + +db.getSiblingDB("orion-validation").entities.insertMany([ + { + "_id": { + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.1: missing entity id", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.2", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.2: missing entity type", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.3", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.3: missing entity servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.1", + "type": "T", + "servicePath": "/SS" + }, + "attrs": { + "desc": { + "value": "Rule11.1: missing attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.2: missing entity creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.3: missing entity modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.1: missing attribute mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + } + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.2: missing attribute creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.3: missing attribute modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.1: attribute in attrsName but not in attrs object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.2: attribute in attrs object but not in attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.3: attribute in attrsName but not in attrs object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2" + ], + "attrs": { + "desc": { + "value": "Rule13.4: attribute in attrs object but not in attrNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.1: md in mdNames but not in md object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.2: md in md object but not in mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.3: md in mdNames but not in md object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.4: md in md object but not in mdNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule15", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.1: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "type": "T", + "id": "Rule15", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.2: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "servicePath": "/SS", + "id": "Rule15", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.3: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation", + "otherLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.1: more than one geo-attribute", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + }, + "otherLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.2", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.2: no geo-attribute but location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.3", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.3: geo-attribute with null value and location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": null, + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.4", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.4: geo-attribute but not location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.5", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.5: location.attrName inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "ignoredLocation", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.6", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.6: geo:point coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 20, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.7", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.7: geo:line coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:line", + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "mdNames": [] + } + }, + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "location": { + "attrName": "location", + "coords": { + "type": "LineString", + "coordinates": [ + [ + 2, + 1 + ], + [ + 40, + 3 + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.8", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.8: geo:box coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:box", + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "mdNames": [] + } + }, + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 2, + 3 + ], + [ + 4, + 30 + ], + [ + 4, + 1 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.9", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.9: geo:polygon inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "10, 20", + "10, -20", + "1, 2" + ], + "type": "geo:polygon", + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "mdNames": [] + } + }, + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 200, + 10 + ], + [ + -20, + 10 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.10", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.10: geo:json coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -200 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.11", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.11: geo:json coords strings", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + "1", + "2" + ], + [ + "10", + "20" + ], + [ + "10", + "-20" + ], + [ + "1", + "2" + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule17.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule17.1: missing lastCorrelator field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858 + }, + { + "_id": { + "id": "", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.1: id syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.2: id syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.3(", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.3: id syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.4#", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.4: id syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.1", + "type": "", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.2", + "type": "Txxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.3", + "type": "T(", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.3: type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.4", + "type": "T#", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.4: type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.1", + "type": "T", + "servicePath": "SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: servicePath does not starts with slash", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.2", + "type": "T", + "servicePath": "/S1/S2/S3/S4/S5/S6/S7/S8/S9/S10/S11" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: more than 10 levels in servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.3", + "type": "T", + "servicePath": "/S1//S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.3: servicePath level less than minimum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.4", + "type": "T", + "servicePath": "/S1/Sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.4: servicePath level greater than maximum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.5", + "type": "T", + "servicePath": "/S1/S#/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.5: servicePath syntax problem", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.1: attr name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.2: attr name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1(", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.3: attr name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1(": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1#", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.4: attr name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1#": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.1: attr type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.2: attr type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.3: attr type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number(", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.4: attr type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number#", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.5", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.5: attr type is missing", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.1: md name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.2: md name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.3: md name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1(": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1(", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.4: md name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1#": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1#", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.1: md type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.2: md type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.3: md type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number(", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.4: md type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number#", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.5", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.5: md type is missing", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.1: usage of legacy geo:point", + "type": "Text", + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "mdNames": [] + } + }, + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.2", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.2: usage of legacy geo:line", + "type": "Text", + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:line", + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "mdNames": [] + } + }, + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "location": { + "attrName": "location", + "coords": { + "type": "LineString", + "coordinates": [ + [ + 2, + 1 + ], + [ + 4, + 3 + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.3", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.3: usage of legacy geo:box", + "type": "Text", + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:box", + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "mdNames": [] + } + }, + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 2, + 3 + ], + [ + 4, + 3 + ], + [ + 4, + 1 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.4", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.4: usage of legacy geo:polygon", + "type": "Text", + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "10, 20", + "10, -20", + "1, 2" + ], + "type": "geo:polygon", + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "mdNames": [] + } + }, + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 20, + 10 + ], + [ + -20, + 10 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule91.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule91.1: no more than one location metadata", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "polygon", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + }, + "ignoredLocation": { + "value": null, + "type": "point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + }, + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "ignoreType", + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule92.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule92.1: location must be WSG84 or WGS84", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "polygon", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG99" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule93.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule93.1: redundant metadata location", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule94.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule94.1: non redundant location metadata", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "40.418889, -3.691944", + "type": "coords", + "creDate": 1706007163.0475833, + "modDate": 1706007163.0475833, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1706007163.0475833, + "modDate": 1706007163.0475833, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.691944, + 40.418889 + ] + } + }, + "lastCorrelator": "88e1bc10-b9dd-11ee-8755-080027cd35f1" + } +]) diff --git a/scripts/managedb/check_entities_consistency.py b/scripts/managedb/check_entities_consistency.py deleted file mode 100755 index 192d90803f..0000000000 --- a/scripts/managedb/check_entities_consistency.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this -# script to a file - -# Checks done: -# -# 1. heck that there is no duplication in _id fields (for mongo {_id: {x: 1, y: 2}} and {_id: {y:2, x:1}} are different documents) -# 2. All attribute in 'attrs' are in 'attrNames' and the other way around -# -# -# FIXME: integrate the checkings in check_location_coherence.py into this script (maybe using some kind of -# modular approach?) - - -__author__ = 'fermin' - -from pymongo import MongoClient -import json -import sys -from datetime import datetime -from time import sleep - -ATTRS = 'attrs' -ATTRNAMES = 'attrNames' - -def date2string(time): - """ - Convert date to string - - :param time: date time (as timestamp) - :return: string representing the date time - """ - - return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def base_name(attr): - """ - Returns base name for attr, without take into account id (if any) - :param attr: - :return: - """ - return attr.split('()')[0] - -def check_entity_dup(entity_doc): - """ - Check for entity duplicates in DB - :param entity_doc: entity document - :return: [], a list of errors otherwise - """ - - query = {'_id.id': entity_doc['_id']['id']} - - if 'type' in entity_doc['_id']: - query['_id.type'] = entity_doc['_id']['type'] - else: - query['_id.type'] = {'$exists': False} - - if 'servicePath' in entity_doc['_id']: - query['_id.servicePath'] = entity_doc['_id']['servicePath'] - else: - query['_id.servicePath'] = {'$exists': False} - - c = db[COL].find(query).count() - if c > 1: - return [' * ERROR: duplicated entities number with same _id subfields: {0}'.format(c)] - else: - return [] - - -def check_attrs(attrs, attr_names): - """ - Check attrs - - :param attrs: key-value attrs - :param attr_names: list with attr_names - :return: [], a list of errors otherwise - """ - - r = [] - - # We have found that some times attrs containts non-ASCII characters and printing can be problematic if - # we don't use .encode('utf-8') - - # All in attrs is in attr_names - for attr in attrs.keys(): - if base_name(attr).replace('=','.') not in attr_names: - r.append(' * ERROR: base name of attr <{0}> in attrs keymap cannot be found in attrNames list'.format(attr.encode('utf-8'))) - - # All in attr_names is in attrs - for attr in attr_names: - # map() used to get only the first token in attrs keys - if attr not in map(lambda x: x.split('()')[0].replace('=','.'), attrs.keys()): - r.append(' * ERROR: attr <{0}> in attrName list cannot be found in attrs keymap'.format(attr.encode('utf-8'))) - - return r - - -########################## -# Main program starts here - -if len(sys.argv) != 2: - print "invalid number of arguments, please specifi db name (e.g. 'orion')" - sys.exit() - -DB = sys.argv[1] -COL = 'entities' - -uri = 'mongodb://localhost:27017' -client = MongoClient(uri) -db = client[DB] - -# At the end, n_processed = no_ok + n_not_ok -n_processed = 0 -n_ok = 0 -n_not_ok = 0 - -total = db[COL].count() - -print "- checking entities collection (%d entities), this may take a while... " % total - -# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have -# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use -# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). -# The used batch_size value is an heuristic -for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): - - n_processed += 1 - - sys.stdout.write('- processing entity: %d/%d \r' % (n_processed, total) ) - sys.stdout.flush() - - - errors = check_entity_dup(doc) + check_attrs(doc[ATTRS], doc[ATTRNAMES]) - - if len(errors) == 0: - n_ok += 1 - else: - print '- {0}: entity {1} ({2}): metadata ID detected'.format(n_processed, json.dumps(doc['_id']), date2string(doc['modDate'])) - print '\n'.join(errors) - n_not_ok += 1 - -print '- processing entity: %d/%d' % (n_processed, total) -print '- entities analyzed: %d' % n_processed -print ' * entities OK: %d' % n_ok -print ' * entities with consistency problems: %d' % n_not_ok - -""" -def check_dup_id_subfields(): - - for doc in db[COL].find(): - query = {'_id.id': doc['_id']['id']} - - if 'type' in doc['_id']: - query['_id.type'] = doc['_id']['type'] - else: - query['_id.type'] = {'$exists': False} - - if 'servicePath' in doc['_id']: - query['_id.servicePath'] = doc['_id']['servicePath'] - else: - query['_id.servicePath'] = {'$exists': False} - - c = db[COL].find(query).count() - if c > 1: - warn('<%s> has duplicated entities with same _id subfields: %d' % (str(doc['_id']), c)) - -""" diff --git a/scripts/managedb/check_location_coherence.py b/scripts/managedb/check_location_coherence.py deleted file mode 100755 index 41133bfb33..0000000000 --- a/scripts/managedb/check_location_coherence.py +++ /dev/null @@ -1,718 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this -# script to a file - -# This script requires: -# -# pymongo==3.0.3 -# geojson==2.4.1 - -__author__ = 'fermin' - -import sys -import json -import pymongo -import geojson -from datetime import datetime - - -############################################################## -# BEGIN of the configuration part (don't touch above this line ;) - -uri = 'mongodb://localhost:27017' - -autofix = False - -verbose = False - -# END of the configuration part (don't touch below this line ;) -############################################################## - - -ATTRS = 'attrs' -GEO_TYPES = [ 'geo:point', 'geo:line', 'geo:box', 'geo:polygon', 'geo:json'] - -def flatten(_id): - """ - The way in which Python manage dictionaries doesn't make easy to be sure - of field ordering, which is important for MongoDB in the case of using an - embedded document for _id. This function helps. - - :param _id: JSON document containing id, type and servicePath - :return: a "flatten" version of the _id - """ - - r = {'_id.id': _id['id']} - - if 'type' in _id: - r['_id.type'] = _id['type'] - else: - r['_id.type'] = {'$exists': False} - - if 'servicePath' in _id: - r['_id.servicePath'] = _id['servicePath'] - else: - r['_id.servicePath'] = {'$exists': False} - - return r - -def update_ok(doc, check_attrs): - """ - Check that entity document was updated correctly at DB. - - :param doc: the doc to check - :param check_attrs: list of attributes which existente is checked - """ - - if not ATTRS in doc: - #print "debug1: no attrs" - return False - - for attr in check_attrs: - - if attr not in doc[ATTRS]: - #print "debug2: %s" % attr - return False - - return True - - -def fix_location_geopoint(entity, geo_attr, coords): - """ - Fix location entity (geo:point), adding the missing field. - - :param entity: entity to fix - :param geo_attr: name of the geo:point attribute - :param coords: coordinates of the geo:point attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - # Get coordinates and revert it (GeoJSON uses that) - try: - # Sanity check - if len(coords.split(',')) != 2: - raise ValueError - - coordinates = [float(coords.split(',')[1]), float(coords.split(',')[0])] - except ValueError: - return 'FAIL coordinates parsing error <%s>' % coords - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': coordinates - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - -def fix_empty_geopoint(entity, geo_attr): - """ - Fix location with empty geo:point, seting it to 0,0 - - :param entity: entity to fix - :param geo_attr: name of the geo:point attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - entity[ATTRS][geo_attr]['value'] = '0,0' - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': [0, 0] - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - -def str2number(d): - """ - Returns a vector of parsed numbers (other elements are not touched) - - :param d: the vector to parse - :return: the parsed vector - """ - - if type(d) == list: - return map(str2number, d) - else: - try: - return float(d) - except: # Not a float, leave as it is - return d - - -def fix_location_geojson(entity, geo_attr, geo_json): - """ - Fix location entity (geo:json), adding the missing field. - - :param entity: entity to fix - :param geo_attr: name of the geo:json attribute - :param geo_json: GeoJSON object to use for location - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - if type(geo_json) != dict or not 'type' in geo_json.keys() or not 'coordinates' in geo_json.keys(): - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json) - - # If the geo:json was updated using NGSIv1, numbers would be string and the GeoJSON will not be valid. Thus, - # we first parse strings to number - geo_json_parsed = { - 'type': geo_json['type'], - 'coordinates': str2number(geo_json['coordinates']) - } - - # This is weird... :) - # - # As far as I have checked and at least for this version of geojson module (2.4.1), the geojson.loads(string - # function returns either a geojson object (if string corresponds to a valid GeoJSON) or the string - # passed as argument. In the first case, is_valid is a property of the object to check for True. In the - # second case a AttributeError: 'dict' object has no attribute 'is_valid' error is raised - - try: - if not geojson.loads(json.dumps(geo_json_parsed)).is_valid: - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json_parsed) - except: - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json_parsed) - - location = { - 'attrName': geo_attr, - 'coords': geo_json_parsed - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - -def fix_empty_geojson(entity, geo_attr): - """ - Fix location with empty geo:json, seting it to 0,0 - - :param entity: entity to fix - :param geo_attr: name of the geo:json attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - entity[ATTRS][geo_attr]['value'] = { - 'type': 'Point', - 'coordinates': [0, 0] - } - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': [0, 0] - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - - -def add_loc_point_attr(entity, location): - """ - Fix entity adding an attribute that match the location (of type Point) - - :param entity: the entity to fix - :param location: the location field - :return: - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - attr_name = location['attrName'] - - # Get coordinates and revert it (GeoJSON uses that) - coords0 = location['coords']['coordinates'][0] - coords1 = location['coords']['coordinates'][1] - - # Note coordinates are in opposite orden in attribute that in location - attr = { - 'type': 'geo:point', - 'value': "%d, %d" % (coords1, coords0) - } - - entity[ATTRS][attr_name] = attr - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS]}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - - -def extract_geo_attr(attrs): - """ - Given a key-value of attributes, returns the attribute with geo: type or None - if no geo: attribute is found. - - :param attrs: key-value of attribute to process - :return: an attribute name or None - """ - - for attr in attrs.keys(): - if attrs[attr]['type'] in GEO_TYPES: - return attr - - return None - - -def check_ngsiv1_location(attrs, location): - """ - Check if there is an attribute in the entity which match location field in the "NGSIv1 location way", i.e. - attribute type doesn't means anything but the value coordinates format matching the location - - :param attrs: entity attributes object to look - :param location: location field - :return: TRue if check is ok, False otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - # Only Point is allowed in NGSIv1 - if location['coords']['type'] != 'Point': - return False - - loc_attr = None - for attr in attrs.keys(): - if attr == location['attrName']: - loc_attr = attrs[attr] - break - - if loc_attr is None: - return False - - value = loc_attr['value'] - - # Coordinates format check - if len(value.split(',')) != 2: - return False - - # Check coordinates are equal - try: - if float(value.split(',')[0]) != location['coords']['coordinates'][1]: - return False - if float(value.split(',')[1]) != location['coords']['coordinates'][0]: - return False - - except ValueError: - # Format error: some of the token is not a valid float - return False - - return True - - -def date2string(time): - """ - Convert date to string - - :param time: date time (as timestamp) - :return: string representing the date time - """ - - return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def entity_dates(entity): - """ - Return a string describing entity creation and modification dates - :param entity: - :return: - """ - cre = '...' - if 'creDate' in entity: - cre = date2string(doc['creDate']) - - mod = '...' - if 'modDate' in entity: - mod = date2string(doc['modDate']) - - return '%s -> %s' % (cre, mod) - - -def safe_add(d, k): - """ - Add or create a key to dictionary if it doesn't already exists. - - :param d: - :param k: - :return: - """ - - if not k in d.keys(): - d[k] = True - - -def msg(m): - """ - Print a message if verbose is enabled - :param m: message to print - :return: - """ - - if verbose: - print m - - -########################## -# Main program starts here - -if len(sys.argv) != 2: - print "invalid number of arguments, use: ./check_location_coherence.py " - sys.exit() - -DB = sys.argv[1] -COL = 'entities' - -# Warn user -if autofix: - print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB - print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk." - print "If you are sure you want to continue type 'yes' and press Enter" - - confirm = raw_input() - - if (confirm != 'yes'): - sys.exit() - -client = pymongo.MongoClient(uri) -db = client[DB] - -need_help = False -processed = 0 - -# At the end, processed = no id + id duplicate attrs + id single attrs -counter_analysis = { - 'ngeo-nloc': 0, - 'geo-loc': 0, - 'geopoint-nloc': 0, - 'geojson-nloc': 0, - 'ngeo-loc': 0, - 'legacy': 0, - 'ngeo-locpoint': 0, - 'unknown-geo': 0, - 'corrupted-location': 0, - 'emptygeopoint': 0, - 'emptygeojson': 0, -} -# At the end, processed = untouched + changed + error -counter_update = { - 'untouched': 0, - 'changed': 0, - 'error': 0, -} - -not_fixable_types_found = {} -location_types_found = {} - -total = db[COL].count() - -print "- processing entities collection (%d entities) looking for location field coherence, this may take a while... " % total - -# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have -# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use -# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). -# The used batch_size value is an heuristic -for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): - - processed += 1 - - # Progress meter - #sys.stdout.write('- processing entity: %d/%d \r' % (processed, total) ) - #sys.stdout.flush() - - # FIXME: code should be refactored in per-case function blocks. Currently it is an awful piece of - # nested if-then code... but ok, this is just a quick and dirty script :) - - location = None - geo_attr = extract_geo_attr(doc[ATTRS]) - if 'location' in doc: - location = doc['location'] - - if location is None: - if geo_attr is None: - # Entity without geo: attribute and without location field. It's ok. - counter_analysis['ngeo-nloc'] += 1 - counter_update['untouched'] += 1 - else: # geo_attr is not None - geo_type = doc[ATTRS][geo_attr]['type'] - geo_value = doc[ATTRS][geo_attr]['value'] - - if geo_type == 'geo:point': - - if geo_value == '': - # Entity with empty geo:point is a degenerated case. Fixable, setting 0,0 as coordinates. - counter_analysis['emptygeopoint'] += 1 - if autofix: - result = fix_empty_geopoint(doc, geo_attr) - msg(' - {0}: fixing empty geo:point {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: empty geo:point {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - - counter_update['untouched'] += 1 - need_help = True - else: - # Entity with geo:point attribute but without location field. Fixable - counter_analysis['geopoint-nloc'] += 1 - if autofix: - result = fix_location_geopoint(doc, geo_attr, geo_value) - msg(' - {0}: fixing loc (geo:point) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ geo:point but wo/ location {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - - elif geo_type == 'geo:json': - - if geo_value == '': - # Entity with empty geo:json is a degenerated case. Fixable, setting 0,0 as coordinates. - counter_analysis['emptygeojson'] += 1 - if autofix: - result = fix_empty_geojson(doc, geo_attr) - msg(' - {0}: fixing empty geo:json {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: empty geo:json {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - - counter_update['untouched'] += 1 - need_help = True - else: - # Entity with geo:point attribute but without location field. Fixable - counter_analysis['geojson-nloc'] += 1 - if autofix: - result = fix_location_geojson(doc, geo_attr, geo_value) - msg(' - {0}: fixing loc (geo:json) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ geo:json but wo/ location {1} ({2})'.format(processed, - json.dumps(doc['_id']), - entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - - else: - # Entity with geo: attribute different than geo:point or geo:json but without location field. Not fixable - msg(' - {0}: unfixable {1} {2} ({3})'.format(processed, geo_type, json.dumps(doc['_id']), - entity_dates(doc))) - - - safe_add(not_fixable_types_found, geo_type) - - counter_analysis['unknown-geo'] += 1 - counter_update['untouched'] += 1 - need_help = True - - else: # location is not None - try: - loc_type = location['coords']['type'] - loc_attr = location['attrName'] - - if geo_attr is None: - # Entity without geo: attribute location but with location field. They may come from NGSIv1 - # or be a real problem. - if loc_type == 'Point' and check_ngsiv1_location(doc[ATTRS], location): - counter_analysis['legacy'] += 1 - counter_update['untouched'] += 1 - else: - if loc_type == 'Point' and not loc_attr in doc[ATTRS]: - # Location type is Point and attribute doesn't not exist. We can fix adding the corresponding - # attribute - counter_analysis['ngeo-locpoint'] += 1 - if autofix: - result = add_loc_point_attr(doc, location) - msg(' - {0}: fixing loc (Point) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ location Point but wo/ geo:point or NGSIv1 point {1} ({2})'.format(processed, - json.dumps(doc['_id']), entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - else: - # Location type is not point or attribute exits. Not fixable - msg(' - {0}: entity w/ location but not Point or attribute exists {1} ({2}) - location is {3}'.format(processed, - json.dumps(doc['_id']), - entity_dates(doc), - json.dumps(location))) - counter_analysis['ngeo-loc'] += 1 - counter_update['untouched'] += 1 - safe_add(location_types_found, loc_type) - need_help = True - - else: # geo_attr is not None - # Entity with geo: attribute and with location field. It's ok - # FIXME: it may happen that the GeoJSON at location field doesn't correspond with the one calculated - # from geo: attribute. However, we consider that possibility very rare so we are not checking it - counter_analysis['geo-loc'] += 1 - counter_update['untouched'] += 1 - - except KeyError: - # Location format is wrong - counter_analysis['corrupted-location'] += 1 - counter_update['untouched'] += 1 - - msg(' - {0}: entity w/ corrupted location {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), - json.dumps(location))) - - need_help = True - -print '- processing entity: %d/%d' % (processed, total) -print '- documents analyzed: %d' % processed -print ' * entities wo/ geo: attr & wo/ loc field (ok) %d' % counter_analysis['ngeo-nloc'] -print ' * entities w/ geo: attr & w/ loc field (ok) %d' % counter_analysis['geo-loc'] -print ' * entities wo/ geo: attr & w/ loc field - coherent (leg NGSIv1) (ok) %d' % counter_analysis['legacy'] -print ' ! entities wo/ geo: attr & w/ loc field - not coherent (fixable) %d' % counter_analysis['ngeo-locpoint'] -print ' ! entities wo/ geo: attr & w/ loc field - not coherent (unfixable) %d' % counter_analysis['ngeo-loc'] -print ' ! entities w/ geo:point attr & wo/ loc field (fixable) %d' % counter_analysis['geopoint-nloc'] -print ' ! entities w/ empty string in geo:point attr (fixable) %d' % counter_analysis['emptygeopoint'] -print ' ! entities w/ empty string in geo:json attr (fixable) %d' % counter_analysis['emptygeojson'] -print ' ! entities w/ geo:json attr and wo/ loc field (fixable) %d' % counter_analysis['geojson-nloc'] -print ' ! entities w/ other geo: attr and wo/ loc field (unfixable) %d' % counter_analysis['unknown-geo'] -print ' ! entities w/ detected corrupted location (unfixable) %d' % counter_analysis['corrupted-location'] - -if len(not_fixable_types_found.keys()) > 0: - print '* geo: types found without associated location field (except geo:point): %s' % ','.join(not_fixable_types_found.keys()) - -if len(location_types_found.keys()) > 0: - print '* loc types found without associated geo: attr not fixable: %s' % ','.join(location_types_found.keys()) - - -print '- documents processed: %d' % processed -print ' * untouched: %d' % counter_update['untouched'] -print ' * changed: %d' % counter_update['changed'] -print ' * attempt to change but error: %d' % counter_update['error'] - -if need_help: - print "------------------------------------------------------" - print "WARNING: some problem was found during the process. Ask for help!"