Scripts to update the container journey db from pipelines (#5)

* Adding function to update hld pipeline

* Adding requirements.txt

* Updating requirements.txt

* Updating requirements.txt

* Updating requirements.txt

* Updating requirements.txt

* Adding new script for manifest pipeline

* Combining scripts into one

* Handle pipelines started in between

* Check for key correctly

* Support for adding a new entry

* Update task to entity

* Minor updates to script

* Attempting to reduce requirements

.gitignore

@@ -4,6 +4,7 @@
 /node_modules
 /.pnp
 .pnp.js
+/venv
 
 # testing
 /coverage

pipeline-scripts/requirements.txt

@@ -0,0 +1,17 @@
+asn1crypto==0.24.0
+azure-common==1.1.23
+azure-cosmosdb-nspkg==2.0.2
+azure-cosmosdb-table==1.0.5
+azure-nspkg==3.0.2
+azure-storage==0.36.0
+azure-storage-common==1.4.2
+certifi==2019.6.16
+cffi==1.12.3
+chardet==3.0.4
+cryptography==2.7
+idna==2.8
+pycparser==2.19
+python-dateutil==2.8.0
+requests==2.22.0
+six==1.12.0
+urllib3==1.25.3

pipeline-scripts/update_pipeline.py

@@ -0,0 +1,68 @@
+from azure.cosmosdb.table.tableservice import TableService
+from azure.cosmosdb.table.models import Entity
+import sys
+import uuid
+
+def generate_row_key():
+    return str(uuid.uuid4()).split('-')[-1]
+
+# Performs a look up based on filter_name:filter_value for the pipeline to update its details
+def update_pipeline(account_name, account_key, table_name, partition_name, filter_name, filter_value, name1, value1, name2=None, value2=None):
+    table_service = TableService(account_name=account_name, account_key=account_key)
+    entities = table_service.query_entities(table_name, filter=filter_name + " eq '"+ filter_value + "'")
+
+    count = 0
+    for entity in entities:
+        count = count + 1
+        add = False
+        if name1 in entity and entity[name1] != value1:
+            add = True
+        entity[name1] = value1
+
+        if name2 != None and value2 != None:
+            if name2 in entity and entity[name2] != value2:
+                add = True
+            entity[name2] = value2
+
+        if add == False:
+            table_service.update_entity(table_name, entity)
+            print("Updating existing entry")
+        else:
+            guid = generate_row_key()
+            entity["RowKey"] = guid
+            table_service.insert_entity(table_name, entity)
+            print("Adding new entry since one already existed")
+        print(entity)
+        break
+
+    if count == 0:
+        add_pipeline(account_name, account_key, table_name, partition_name, filter_name, filter_value, name1, value1, name2, value2)
+    print("Done")
+
+def add_pipeline(account_name, account_key, table_name, partition_name, filter_name, filter_value, name1, value1, name2=None, value2=None):
+    print("Adding a new entry")
+    new_entry = {}
+    new_entry["RowKey"] = generate_row_key()
+    new_entry["PartitionKey"] = partition_name
+    new_entry[filter_name] = filter_value
+    new_entry[name1] = value1
+    new_entry[name2] = value2
+    print(new_entry)
+    table_service = TableService(account_name=account_name, account_key=account_key)
+    table_service.insert_entity(table_name, new_entry)
+
+
+def list_all_entities(account_name, account_key, table_name):
+    table_service = TableService(account_name=account_name, account_key=account_key)
+    entities = table_service.query_entities(table_name)
+    for entity in entities:
+        print(entity)
+
+if __name__ == "__main__":
+    print(len(sys.argv))
+    if len(sys.argv) == 9:
+        update_pipeline(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6], sys.argv[7], sys.argv[8])
+    elif len(sys.argv) == 11:
+        update_pipeline(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6], sys.argv[7], sys.argv[8], sys.argv[9], sys.argv[10])
+    elif len(sys.argv) == 4:
+        list_all_entities(sys.argv[1], sys.argv[2], sys.argv[3])