From b594b762b984ae4826e50c41b7a39c1fc22bb6cf Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 14:39:44 -0700 Subject: [PATCH 01/18] Add initial template to demangle DDS to SROS2 Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 341 ++++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 sros2/sros2/policy/templates/dds/policy.xsl diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl new file mode 100644 index 00000000..e3b38b23 --- /dev/null +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -0,0 +1,341 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a0fae7e8a04018ea183f8ff2e592cb83f4c3c55c Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 16:15:39 -0700 Subject: [PATCH 02/18] Deduplicate and alpha sort elements/atrabutes Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 107 +++++++++++++++++++- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index e3b38b23..baf76e64 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -1,8 +1,12 @@ + xmlns:ext="http://exslt.org/common" exclude-result-prefixes="ext ros xs xsl local"> @@ -50,6 +54,56 @@ + + + + + + + + + + + + + + + + + + + + @@ -86,11 +140,16 @@ + + + + + + select="ext:node-set($policy_deduplicated)"/> - @@ -308,11 +367,11 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ab536cb08233034525d7b609a80d7a547254f0da Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 20:39:30 -0700 Subject: [PATCH 03/18] Remove dead code Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 30 --------------------- 1 file changed, 30 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index baf76e64..8af4784c 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -367,36 +367,6 @@ - - - - - - - - From eed8d6bcae1baba4679d9fe91e92751e4eed4902 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 20:41:08 -0700 Subject: [PATCH 04/18] Set unique mode for substation Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index 8af4784c..6c45b2e8 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -239,21 +239,27 @@ - - - + + + + + + @@ -327,6 +333,13 @@ + + + + + + + From 76242a410837dce447cb8bca66c54a5ae35d626f Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 20:43:34 -0700 Subject: [PATCH 05/18] Adding deduplicate namespace and compress for profiles apply template on each profile to avoided crosstalk Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 135 +++++++++++++++----- 1 file changed, 104 insertions(+), 31 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index 6c45b2e8..6e22b7a1 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -104,6 +104,69 @@ .))[1]))]"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -111,43 +174,53 @@ - - - - - - - - - - - - - - - - - - - - + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + select="ext:node-set($policy)"/> @@ -340,7 +413,7 @@ - + From cf6436138ca3ab45541b99808b0f92e18d00c29e Mon Sep 17 00:00:00 2001 From: ruffsl Date: Mon, 5 Aug 2019 21:05:40 -0700 Subject: [PATCH 06/18] Check in initial discovery sqlite script Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 172 +++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 sros2/scripts/dds_sql_to_sros2_policy.py diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py new file mode 100644 index 00000000..3c1d72a8 --- /dev/null +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -0,0 +1,172 @@ + +# coding: utf-8 + +# In[ ]: + + +import pandas as pd +import sqlite3 + +from lxml import etree + + +# In[ ]: + + +db_path = "../tb3_demo_dds_discovery.db" + + +# In[ ]: + + +node_pub_query = """ +SELECT DISTINCT + DCPSParticipant.ParticipantData_key, + DCPSParticipant.ParticipantData_user_data, + DCPSPublication.PublicationData_topic_name +FROM DCPSParticipant +INNER JOIN DCPSPublication ON + DCPSParticipant.ParticipantData_key = DCPSPublication.PublicationData_participant_key +AND DCPSParticipant.ParticipantData_user_data IS NOT NULL; +""" + +node_sub_query = """ +SELECT DISTINCT + DCPSParticipant.ParticipantData_key, + DCPSParticipant.ParticipantData_user_data, + DCPSSubscription.SubscriptionData_topic_name +FROM DCPSParticipant +INNER JOIN DCPSSubscription ON + DCPSParticipant.ParticipantData_key=DCPSSubscription.SubscriptionData_participant_key +AND DCPSParticipant.ParticipantData_user_data IS NOT NULL; +""" + + +# In[ ]: + + +def user_bytes_to_dict(user_bytes): + user_string = user_bytes.decode('utf8') + key_value_list = user_string[:-2].split(';') + key_value_dict = dict() + for key_values in key_value_list: + key, value = key_values.split('=', 1) + key_value_dict[key] = value + return key_value_dict + + +# In[ ]: + + +def translate_df(df): + _df = df['ParticipantData_user_data'].apply(user_bytes_to_dict).apply(pd.Series) + df = pd.concat([df, _df], axis=1) + return df + + +# In[ ]: + + +db_uri = 'file:{}?mode=ro'.format(db_path) +with sqlite3.connect(db_uri, uri=True) as db: + pub_df = translate_df(pd.read_sql_query(node_pub_query, db)) + sub_df = translate_df(pd.read_sql_query(node_sub_query, db)) + +pub_df = pub_df.assign(mode='publish') +pub_df = pub_df.rename(columns={"PublicationData_topic_name": "dds_topic"}) +sub_df = sub_df.assign(mode='subscribe') +sub_df = sub_df.rename(columns={"SubscriptionData_topic_name": "dds_topic"}) + +df = pd.concat([pub_df, sub_df]) +df.set_index(['namespace', 'name', 'mode'], inplace=True) + + +# In[ ]: + + +foo = df.index[0] +foo + + +# In[ ]: + + +# with pd.option_context('display.max_rows', None, 'display.max_columns', None): # more options can be specified also +# display(df) + + +# In[ ]: + + +polciy = etree.Element('polciy') +profiles = etree.SubElement(polciy, 'profiles') + +for namespace in df.index.get_level_values('namespace').unique(): + for name in df.index.get_level_values('name').unique(): + profile = etree.SubElement(profiles, 'profile') + profile.set("ns", namespace) + profile.set("node", name) + _df = df['dds_topic'].loc[namespace, name, :] + for mode in _df.index.get_level_values('mode').unique(): + topics = etree.SubElement(profile, 'dds_topics') + topics.set(mode, "ALLOW") + for dds_topic in df['dds_topic'].loc[namespace, name, :]: + topic = etree.SubElement(topics, 'dds_topic') + topic.text = dds_topic + + +# In[ ]: + + +print(etree.tostring(polciy, pretty_print=True).decode()) + + +# In[ ]: + + +import os + +from lxml import etree + +from sros2.policy import ( + get_policy_schema, + get_transport_schema, + get_transport_template, +) + + +# In[ ]: + + +# Get paths +policy_xsd_path = get_policy_schema('policy.xsd') +permissions_xsl_path = get_transport_template('dds', 'permissions.xsl') +permissions_xsd_path = get_transport_schema('dds', 'permissions.xsd') +policy_xsl_path = 'sros2/sros2/policy/templates/dds/policy.xsl' + +# Parse files +policy_xsd = etree.XMLSchema(etree.parse(policy_xsd_path)) +policy_xsl = etree.XSLT(etree.parse(policy_xsl_path)) +permissions_xsl = etree.XSLT(etree.parse(permissions_xsl_path)) +permissions_xsd = etree.XMLSchema(etree.parse(permissions_xsd_path)) + +# Get policy +dds_policy_xml_path = 'tb3_raw_dds_policy.xml' +dds_policy_xml = etree.parse(dds_policy_xml_path) +dds_policy_xml.xinclude() + +# Validate policy schema +# policy_xsd.assertValid(policy_xml) + +# Transform policy +policy_xml = policy_xsl(dds_policy_xml) + +# Validate permissions schema +# policy_xsd.assertValid(policy_xml) + +# Output policy +# policy_xml_path = os.path.join('policy.xml') +# with open(policy_xml_path, 'w') as f: +# f.write(etree.tostring(policy_xml, pretty_print=True).decode()) +print(etree.tostring(policy_xml, pretty_print=True).decode()) + From 527bfc75bc8e3a85fa3755ba5808f2b08fa24f4b Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 10:25:29 -0700 Subject: [PATCH 07/18] Fix typo for execute Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/policy.xsl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index 6e22b7a1..235597ff 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -22,13 +22,13 @@ - + - + - - + + @@ -65,14 +65,14 @@ @@ -112,7 +112,7 @@ + @execute)"/> @@ -126,7 +126,7 @@ + @execute)))]"/> + following-sibling::services[@execute = current()/@execute]/node() " /> From 3f8c81c4420fdeb001d8335f4c2d968761c0bfac Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 10:28:44 -0700 Subject: [PATCH 08/18] Fix typo for policy Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 6 +++--- sros2/sros2/policy/templates/dds/policy.xsl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 3c1d72a8..8725cb78 100644 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -98,8 +98,8 @@ def translate_df(df): # In[ ]: -polciy = etree.Element('polciy') -profiles = etree.SubElement(polciy, 'profiles') +policy = etree.Element('policy') +profiles = etree.SubElement(policy, 'profiles') for namespace in df.index.get_level_values('namespace').unique(): for name in df.index.get_level_values('name').unique(): @@ -118,7 +118,7 @@ def translate_df(df): # In[ ]: -print(etree.tostring(polciy, pretty_print=True).decode()) +print(etree.tostring(policy, pretty_print=True).decode()) # In[ ]: diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/policy.xsl index 235597ff..7d2e8b39 100644 --- a/sros2/sros2/policy/templates/dds/policy.xsl +++ b/sros2/sros2/policy/templates/dds/policy.xsl @@ -168,7 +168,7 @@ - + From e06b247c0033aecdf8cf48322217759455af3bd0 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 10:41:41 -0700 Subject: [PATCH 09/18] Correct indexing to include mode To avoid provisioning topic to all modes instead of the topic's respective mode as intended Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 8725cb78..bda83c5b 100644 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -110,7 +110,7 @@ def translate_df(df): for mode in _df.index.get_level_values('mode').unique(): topics = etree.SubElement(profile, 'dds_topics') topics.set(mode, "ALLOW") - for dds_topic in df['dds_topic'].loc[namespace, name, :]: + for dds_topic in df['dds_topic'].loc[namespace, name, mode]: topic = etree.SubElement(topics, 'dds_topic') topic.text = dds_topic From 1fa8af49c895cb6e811bffb707ec4b6f291cc145 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 11:39:58 -0700 Subject: [PATCH 10/18] Rename to dds/demangle.xml Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/{policy.xsl => demangle.xsl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sros2/sros2/policy/templates/dds/{policy.xsl => demangle.xsl} (100%) diff --git a/sros2/sros2/policy/templates/dds/policy.xsl b/sros2/sros2/policy/templates/dds/demangle.xsl similarity index 100% rename from sros2/sros2/policy/templates/dds/policy.xsl rename to sros2/sros2/policy/templates/dds/demangle.xsl From 350e37b3281cb5c0a753a73617bbb2b8f5c7f33c Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 11:49:01 -0700 Subject: [PATCH 11/18] Clean up notebook into runnable script Borrowing cli petern from @mikaelarguedas Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 211 ++++++++++------------- 1 file changed, 91 insertions(+), 120 deletions(-) mode change 100644 => 100755 sros2/scripts/dds_sql_to_sros2_policy.py diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py old mode 100644 new mode 100755 index bda83c5b..24ede630 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -1,22 +1,19 @@ +#! /usr/bin/env python3 -# coding: utf-8 - -# In[ ]: - - +import argparse +import os import pandas as pd import sqlite3 +import sys from lxml import etree - -# In[ ]: - - -db_path = "../tb3_demo_dds_discovery.db" - - -# In[ ]: +from sros2.policy import ( + get_policy_schema, + get_transport_schema, + get_transport_template, + POLICY_VERSION, +) node_pub_query = """ @@ -42,9 +39,6 @@ """ -# In[ ]: - - def user_bytes_to_dict(user_bytes): user_string = user_bytes.decode('utf8') key_value_list = user_string[:-2].split(';') @@ -55,118 +49,95 @@ def user_bytes_to_dict(user_bytes): return key_value_dict -# In[ ]: - - def translate_df(df): - _df = df['ParticipantData_user_data'].apply(user_bytes_to_dict).apply(pd.Series) + _df = df['ParticipantData_user_data'] \ + .apply(user_bytes_to_dict) \ + .apply(pd.Series) df = pd.concat([df, _df], axis=1) return df -# In[ ]: - - -db_uri = 'file:{}?mode=ro'.format(db_path) -with sqlite3.connect(db_uri, uri=True) as db: +def db_to_df(db): pub_df = translate_df(pd.read_sql_query(node_pub_query, db)) sub_df = translate_df(pd.read_sql_query(node_sub_query, db)) -pub_df = pub_df.assign(mode='publish') -pub_df = pub_df.rename(columns={"PublicationData_topic_name": "dds_topic"}) -sub_df = sub_df.assign(mode='subscribe') -sub_df = sub_df.rename(columns={"SubscriptionData_topic_name": "dds_topic"}) - -df = pd.concat([pub_df, sub_df]) -df.set_index(['namespace', 'name', 'mode'], inplace=True) - - -# In[ ]: - - -foo = df.index[0] -foo - - -# In[ ]: - - -# with pd.option_context('display.max_rows', None, 'display.max_columns', None): # more options can be specified also -# display(df) - + pub_df = pub_df.assign(mode='publish') + pub_df = pub_df.rename(columns={"PublicationData_topic_name": "dds_topic"}) + sub_df = sub_df.assign(mode='subscribe') + sub_df = sub_df.rename(columns={"SubscriptionData_topic_name": "dds_topic"}) -# In[ ]: - - -policy = etree.Element('policy') -profiles = etree.SubElement(policy, 'profiles') - -for namespace in df.index.get_level_values('namespace').unique(): - for name in df.index.get_level_values('name').unique(): - profile = etree.SubElement(profiles, 'profile') - profile.set("ns", namespace) - profile.set("node", name) - _df = df['dds_topic'].loc[namespace, name, :] - for mode in _df.index.get_level_values('mode').unique(): - topics = etree.SubElement(profile, 'dds_topics') - topics.set(mode, "ALLOW") - for dds_topic in df['dds_topic'].loc[namespace, name, mode]: - topic = etree.SubElement(topics, 'dds_topic') - topic.text = dds_topic - - -# In[ ]: - - -print(etree.tostring(policy, pretty_print=True).decode()) - - -# In[ ]: - - -import os - -from lxml import etree - -from sros2.policy import ( - get_policy_schema, - get_transport_schema, - get_transport_template, -) - - -# In[ ]: - - -# Get paths -policy_xsd_path = get_policy_schema('policy.xsd') -permissions_xsl_path = get_transport_template('dds', 'permissions.xsl') -permissions_xsd_path = get_transport_schema('dds', 'permissions.xsd') -policy_xsl_path = 'sros2/sros2/policy/templates/dds/policy.xsl' - -# Parse files -policy_xsd = etree.XMLSchema(etree.parse(policy_xsd_path)) -policy_xsl = etree.XSLT(etree.parse(policy_xsl_path)) -permissions_xsl = etree.XSLT(etree.parse(permissions_xsl_path)) -permissions_xsd = etree.XMLSchema(etree.parse(permissions_xsd_path)) - -# Get policy -dds_policy_xml_path = 'tb3_raw_dds_policy.xml' -dds_policy_xml = etree.parse(dds_policy_xml_path) -dds_policy_xml.xinclude() - -# Validate policy schema -# policy_xsd.assertValid(policy_xml) - -# Transform policy -policy_xml = policy_xsl(dds_policy_xml) - -# Validate permissions schema -# policy_xsd.assertValid(policy_xml) + df = pd.concat([pub_df, sub_df]) + df.set_index(['namespace', 'name', 'mode'], inplace=True) + return df -# Output policy -# policy_xml_path = os.path.join('policy.xml') -# with open(policy_xml_path, 'w') as f: -# f.write(etree.tostring(policy_xml, pretty_print=True).decode()) -print(etree.tostring(policy_xml, pretty_print=True).decode()) +def df_to_dds_policy(df): + dds_policy = etree.Element('policy') + dds_policy.set("version", POLICY_VERSION,) + profiles = etree.SubElement(dds_policy, 'profiles') + + for namespace in df.index.get_level_values('namespace').unique(): + for name in df.index.get_level_values('name').unique(): + profile = etree.SubElement(profiles, 'profile') + profile.set("ns", namespace) + profile.set("node", name) + _df = df['dds_topic'].loc[namespace, name, :] + for mode in _df.index.get_level_values('mode').unique(): + topics = etree.SubElement(profile, 'dds_topics') + topics.set(mode, "ALLOW") + for dds_topic in df['dds_topic'].loc[namespace, name, mode]: + topic = etree.SubElement(topics, 'dds_topic') + topic.text = dds_topic + return dds_policy + + +def dds_policy_to_sros2_policy(dds_policy): + + # Parse files + policy_xsd = etree.XMLSchema( + etree.parse( + get_policy_schema('policy.xsd'))) + demangle_xsl = etree.XSLT( + etree.parse( + get_transport_template('dds', 'demangle.xsl'))) + + # TODO: update schema for dds_topics? + # Validate policy schema + # policy_xsd.assertValid(dds_policy) + + # Transform policy + sros2_policy = demangle_xsl(dds_policy) + + # Validate policy schema + policy_xsd.assertValid(sros2_policy) + + return sros2_policy + + +def main(argv=sys.argv[1:]): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '-i', '--input-db', required=True, + help='path to SQLite3 database with discovery data') + parser.add_argument( + '-o', '--output-policy', + help='path to XML policy file with generated output') + args = parser.parse_args(argv) + + db_uri = 'file:{}?mode=ro'.format(args.input_db) + with sqlite3.connect(db_uri, uri=True) as db: + df = db_to_df(db) + + dds_policy = df_to_dds_policy(df) + sros2_policy = dds_policy_to_sros2_policy(dds_policy) + + if args.output_policy is not None: + with open(args.policy_file, 'wb') as f: + f.write(etree.tostring(sros2_policy, pretty_print=True)) + else: + print(etree.tostring(sros2_policy, pretty_print=True).decode()) + + +if __name__ == '__main__': + main() \ No newline at end of file From c6e061dbd072e29b16a5eb199d8a294aafa329d1 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 16:14:29 -0700 Subject: [PATCH 12/18] Fix unintended node and namespace combinatorial node names should be limited to what exists with respect to the current iterative namespace Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 24ede630..e4eb0ca0 100755 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -77,12 +77,13 @@ def df_to_dds_policy(df): profiles = etree.SubElement(dds_policy, 'profiles') for namespace in df.index.get_level_values('namespace').unique(): - for name in df.index.get_level_values('name').unique(): + _df = df.loc[namespace, :, :] + for name in _df.index.get_level_values('name').unique(): profile = etree.SubElement(profiles, 'profile') profile.set("ns", namespace) profile.set("node", name) - _df = df['dds_topic'].loc[namespace, name, :] - for mode in _df.index.get_level_values('mode').unique(): + __df = df.loc[namespace, name, :] + for mode in __df.index.get_level_values('mode').unique(): topics = etree.SubElement(profile, 'dds_topics') topics.set(mode, "ALLOW") for dds_topic in df['dds_topic'].loc[namespace, name, mode]: From b0b6441b5509e101b1985547c449a2b8ba8fa7f1 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 17:53:39 -0700 Subject: [PATCH 13/18] Declare python dependency for pandas Signed-off-by: ruffsl --- sros2/package.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/sros2/package.xml b/sros2/package.xml index 92bf25ef..5772b3db 100644 --- a/sros2/package.xml +++ b/sros2/package.xml @@ -15,6 +15,7 @@ python3-lxml python3-cryptography + python3-pandas ament_copyright ament_flake8 From 054dd8062ae76e36c35ce359fd5fafc96cc8ca7e Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 20:02:32 -0700 Subject: [PATCH 14/18] Check for unintelligible or missing QoS data Drop rows with non-parcelable userdata payloads, as we can't infer the node name or namespace from participant Participant vs data discovery sent over separate QoS packets in DDS. Ensure a profile is created even if no topics are discovered. E.g. for the ros2 cli daemon only listing nodes on the network Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 55 +++++++++++++----------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index e4eb0ca0..3e8ff992 100755 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -15,38 +15,40 @@ POLICY_VERSION, ) - -node_pub_query = """ +node_query = """ SELECT DISTINCT DCPSParticipant.ParticipantData_key, - DCPSParticipant.ParticipantData_user_data, - DCPSPublication.PublicationData_topic_name -FROM DCPSParticipant -INNER JOIN DCPSPublication ON - DCPSParticipant.ParticipantData_key = DCPSPublication.PublicationData_participant_key -AND DCPSParticipant.ParticipantData_user_data IS NOT NULL; + DCPSParticipant.ParticipantData_user_data +FROM DCPSParticipant +WHERE DCPSParticipant.ParticipantData_user_data IS NOT NULL; """ -node_sub_query = """ +_query = """ SELECT DISTINCT DCPSParticipant.ParticipantData_key, DCPSParticipant.ParticipantData_user_data, - DCPSSubscription.SubscriptionData_topic_name + DCPS{mode}.{mode}Data_topic_name FROM DCPSParticipant -INNER JOIN DCPSSubscription ON - DCPSParticipant.ParticipantData_key=DCPSSubscription.SubscriptionData_participant_key +INNER JOIN DCPS{mode} ON + DCPSParticipant.ParticipantData_key = DCPS{mode}.{mode}Data_participant_key AND DCPSParticipant.ParticipantData_user_data IS NOT NULL; """ +node_pub_query = _query.format(mode="Publication") +node_sub_query = _query.format(mode="Subscription") + def user_bytes_to_dict(user_bytes): - user_string = user_bytes.decode('utf8') - key_value_list = user_string[:-2].split(';') - key_value_dict = dict() - for key_values in key_value_list: - key, value = key_values.split('=', 1) - key_value_dict[key] = value - return key_value_dict + try: + user_string = user_bytes.decode('utf8') + key_value_list = user_string[:-2].split(';') + key_value_dict = dict() + for key_values in key_value_list: + key, value = key_values.split('=', 1) + key_value_dict[key] = value + return key_value_dict + except: + return None def translate_df(df): @@ -58,6 +60,7 @@ def translate_df(df): def db_to_df(db): + node_df = translate_df(pd.read_sql_query(node_query, db)) pub_df = translate_df(pd.read_sql_query(node_pub_query, db)) sub_df = translate_df(pd.read_sql_query(node_sub_query, db)) @@ -66,7 +69,8 @@ def db_to_df(db): sub_df = sub_df.assign(mode='subscribe') sub_df = sub_df.rename(columns={"SubscriptionData_topic_name": "dds_topic"}) - df = pd.concat([pub_df, sub_df]) + df = pd.concat([node_df, pub_df, sub_df]) + df = df[df['namespace'].notnull()] df.set_index(['namespace', 'name', 'mode'], inplace=True) return df @@ -84,11 +88,12 @@ def df_to_dds_policy(df): profile.set("node", name) __df = df.loc[namespace, name, :] for mode in __df.index.get_level_values('mode').unique(): - topics = etree.SubElement(profile, 'dds_topics') - topics.set(mode, "ALLOW") - for dds_topic in df['dds_topic'].loc[namespace, name, mode]: - topic = etree.SubElement(topics, 'dds_topic') - topic.text = dds_topic + if not pd.isna(mode): + topics = etree.SubElement(profile, 'dds_topics') + topics.set(mode, "ALLOW") + for dds_topic in df['dds_topic'].loc[namespace, name, mode]: + topic = etree.SubElement(topics, 'dds_topic') + topic.text = dds_topic return dds_policy From c1534d2fbe0986d30909860fe7aabdf3bd18b51e Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 20:42:06 -0700 Subject: [PATCH 15/18] Fix typos in compression Signed-off-by: ruffsl --- sros2/sros2/policy/templates/dds/demangle.xsl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sros2/sros2/policy/templates/dds/demangle.xsl b/sros2/sros2/policy/templates/dds/demangle.xsl index 7d2e8b39..db800975 100644 --- a/sros2/sros2/policy/templates/dds/demangle.xsl +++ b/sros2/sros2/policy/templates/dds/demangle.xsl @@ -117,7 +117,7 @@ @reply, '|', @request)"/> @@ -133,7 +133,7 @@ @request)))]"/> @@ -142,9 +142,9 @@ + following-sibling::actions[@call = current()/@call]/node() " /> + following-sibling::actions[@execute = current()/@execute]/node() " /> @@ -160,9 +160,9 @@ + following-sibling::topics[@publish = current()/@publish]/node() " /> + following-sibling::topics[@subscribe = current()/@subscribe]/node() " /> From d71c75beca33ce537d75e997180ae200f0c5dc3d Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 21:57:38 -0700 Subject: [PATCH 16/18] Fix file output Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 3e8ff992..324b62ab 100755 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -139,7 +139,7 @@ def main(argv=sys.argv[1:]): sros2_policy = dds_policy_to_sros2_policy(dds_policy) if args.output_policy is not None: - with open(args.policy_file, 'wb') as f: + with open(args.output_policy, 'wb') as f: f.write(etree.tostring(sros2_policy, pretty_print=True)) else: print(etree.tostring(sros2_policy, pretty_print=True).decode()) From 5e03e8d4173bc5c73db5f93f4f748d1dea9f5905 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 6 Aug 2019 21:59:20 -0700 Subject: [PATCH 17/18] Add raw permission type Update schema and transforms as well as generating script Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 7 ++-- sros2/sros2/policy/schemas/policy.xsd | 11 +++++++ sros2/sros2/policy/templates/dds/demangle.xsl | 32 +++++++++--------- .../policy/templates/dds/permissions.xsl | 33 +++++++++++++++++++ 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 324b62ab..6c602cde 100755 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -89,10 +89,10 @@ def df_to_dds_policy(df): __df = df.loc[namespace, name, :] for mode in __df.index.get_level_values('mode').unique(): if not pd.isna(mode): - topics = etree.SubElement(profile, 'dds_topics') + topics = etree.SubElement(profile, 'raws') topics.set(mode, "ALLOW") for dds_topic in df['dds_topic'].loc[namespace, name, mode]: - topic = etree.SubElement(topics, 'dds_topic') + topic = etree.SubElement(topics, 'raw') topic.text = dds_topic return dds_policy @@ -107,9 +107,8 @@ def dds_policy_to_sros2_policy(dds_policy): etree.parse( get_transport_template('dds', 'demangle.xsl'))) - # TODO: update schema for dds_topics? # Validate policy schema - # policy_xsd.assertValid(dds_policy) + policy_xsd.assertValid(dds_policy) # Transform policy sros2_policy = demangle_xsl(dds_policy) diff --git a/sros2/sros2/policy/schemas/policy.xsd b/sros2/sros2/policy/schemas/policy.xsd index 8086516e..fa11b28e 100644 --- a/sros2/sros2/policy/schemas/policy.xsd +++ b/sros2/sros2/policy/schemas/policy.xsd @@ -24,6 +24,7 @@ + @@ -34,6 +35,16 @@ + + + + + + + + + + diff --git a/sros2/sros2/policy/templates/dds/demangle.xsl b/sros2/sros2/policy/templates/dds/demangle.xsl index db800975..3cbdb1b7 100644 --- a/sros2/sros2/policy/templates/dds/demangle.xsl +++ b/sros2/sros2/policy/templates/dds/demangle.xsl @@ -227,13 +227,13 @@ - + - + @@ -241,14 +241,14 @@ - + - - - - + + + + + substring($_raw, + string-length($_raw) - string-length(@suffix) +1)" /> - + @@ -282,11 +282,11 @@ + substring($_raw, + string-length($_raw) - string-length(@suffix) +1)" /> - + @@ -301,7 +301,7 @@ - + diff --git a/sros2/sros2/policy/templates/dds/permissions.xsl b/sros2/sros2/policy/templates/dds/permissions.xsl index 8a243da2..5b346843 100644 --- a/sros2/sros2/policy/templates/dds/permissions.xsl +++ b/sros2/sros2/policy/templates/dds/permissions.xsl @@ -73,6 +73,9 @@ + + + @@ -90,6 +93,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 850fedc9579f5db47bc78a8260efbc96938c9583 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 7 Aug 2019 21:24:26 -0700 Subject: [PATCH 18/18] Separate raw changes into different schema for now Signed-off-by: ruffsl --- sros2/scripts/dds_sql_to_sros2_policy.py | 17 ++++--- sros2/sros2/policy/schemas/dds/policy.xsd | 56 +++++++++++++++++++++++ sros2/sros2/policy/schemas/policy.xsd | 11 ----- 3 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 sros2/sros2/policy/schemas/dds/policy.xsd diff --git a/sros2/scripts/dds_sql_to_sros2_policy.py b/sros2/scripts/dds_sql_to_sros2_policy.py index 6c602cde..63d0aafb 100755 --- a/sros2/scripts/dds_sql_to_sros2_policy.py +++ b/sros2/scripts/dds_sql_to_sros2_policy.py @@ -100,21 +100,24 @@ def df_to_dds_policy(df): def dds_policy_to_sros2_policy(dds_policy): # Parse files - policy_xsd = etree.XMLSchema( + sros2_policy_xsd = etree.XMLSchema( etree.parse( get_policy_schema('policy.xsd'))) - demangle_xsl = etree.XSLT( + dds_policy_xsd = etree.XMLSchema( + etree.parse( + get_transport_schema('dds', 'policy.xsd'))) + dds_demangle_xsl = etree.XSLT( etree.parse( get_transport_template('dds', 'demangle.xsl'))) - # Validate policy schema - policy_xsd.assertValid(dds_policy) + # Validate input schema + dds_policy_xsd.assertValid(dds_policy) # Transform policy - sros2_policy = demangle_xsl(dds_policy) + sros2_policy = dds_demangle_xsl(dds_policy) - # Validate policy schema - policy_xsd.assertValid(sros2_policy) + # Validate output schema + sros2_policy_xsd.assertValid(sros2_policy) return sros2_policy diff --git a/sros2/sros2/policy/schemas/dds/policy.xsd b/sros2/sros2/policy/schemas/dds/policy.xsd new file mode 100644 index 00000000..96698c19 --- /dev/null +++ b/sros2/sros2/policy/schemas/dds/policy.xsd @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sros2/sros2/policy/schemas/policy.xsd b/sros2/sros2/policy/schemas/policy.xsd index fa11b28e..8086516e 100644 --- a/sros2/sros2/policy/schemas/policy.xsd +++ b/sros2/sros2/policy/schemas/policy.xsd @@ -24,7 +24,6 @@ - @@ -35,16 +34,6 @@ - - - - - - - - - -