From 1392dfbb9b1357d3bf677a23030b2d25e01627fb Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Wed, 7 Jun 2023 13:52:24 -0400 Subject: [PATCH 1/6] Code to delete test dataset --- clone_dataset/clone_dataset_tool.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/clone_dataset/clone_dataset_tool.py b/clone_dataset/clone_dataset_tool.py index 7ae50dd..31368a5 100644 --- a/clone_dataset/clone_dataset_tool.py +++ b/clone_dataset/clone_dataset_tool.py @@ -264,5 +264,12 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase 'name': new_dataset_name } # Clone dataset -new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=True, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only) -print('new dataset API ID:' + new_dataset_id) \ No newline at end of file +new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=clone_children, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only) +print('new dataset API ID:' + new_dataset_id) + + + +# Delete dataset (By default this will delete the new dataset you just created) +#dataset_to_delete = LMIPy.Dataset(new_dataset_id) +#delete_url = f'{dataset_to_delete.server}/v1/dataset/{dataset_to_delete.id}' +#requests.delete(delete_url, headers=create_headers()) \ No newline at end of file From 1e281e82355e8cfd7fab08ff67bbdd2e61a3c4c3 Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Wed, 7 Jun 2023 13:53:52 -0400 Subject: [PATCH 2/6] New script to edit wms datasets and associated layers --- edit_datasets/edit_wms_dataset.py | 124 ++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 edit_datasets/edit_wms_dataset.py diff --git a/edit_datasets/edit_wms_dataset.py b/edit_datasets/edit_wms_dataset.py new file mode 100644 index 0000000..eeb97c6 --- /dev/null +++ b/edit_datasets/edit_wms_dataset.py @@ -0,0 +1,124 @@ +import os +import requests +import json + +API_TOKEN = os.getenv('RW_API_KEY') + +def create_headers(): + return { + 'content-type': "application/json", + 'authorization': "{}".format(os.getenv('apiToken')), + } + +# Start with a dataset you cloned to make sure everything works +cloned_dataset = [''] +cloned_layer = [''] + +# Add your real datasets here. Don't uncomment until you have tested. +datasets = ['', ''] + +# Choose the field you want to find and replace in, the value to fine, and what to replace it with +dataset_field = 'connectorUrl' +layer_field = 'layerConfig' +find = '' +replace = '' + +# Don't actually to anything yet +dry_run = True + +# Use test dataset +test_run = True + +if test_run: + datasets = cloned_dataset + +# Print out what we are doing +print(f'Replacing "{find}" with "{replace}" in `{dataset_field}`.') + + +# Loop through datasets to find and replace values +for dataset_id in datasets: + current_url = f'https://api.resourcewatch.org/v1/dataset/{dataset_id}/?includes=layer' + print(f'DATASET URL: {current_url}') + with requests.get(current_url) as r: + if r.ok: + ds = json.loads(r.content) + else: + raise ValueError(f'API request failed: {current_url}') + assert 'data' in ds + + name = ds['data']['attributes']['name'] + print(f'DATASET NAME: {name}') + + old_value = ds['data']['attributes'][dataset_field] + new_value = old_value.replace(find, replace) + + find_count = old_value.count(find) + replace_count = new_value.count(replace) + + print(f'\nOLD VALUE: {old_value}') + print(f'\nNEW VALUE: {new_value}') + + print(f'\nStarted with {find_count} instances of "{find}" and ended up with {replace_count} instances of "{replace}".') + + update = json.dumps({dataset_field: new_value}) + + if dry_run == False: + with requests.patch(current_url, headers=create_headers(), data=update) as r: + if r.ok: + print('\nDONE') + else: + raise ValueError(f'API request failed: {current_url}') + assert 'data' in ds + else: + print('\nDRY RUN') + + + + + # Layers + layers = [item["id"] for item in ds['data']['attributes']['layer']] + if test_run: + layers = cloned_layer + + for layer_id in layers: + current_url = f'https://api.resourcewatch.org/v1/dataset/{dataset_id}/layer/{layer_id}' + print(f'LAYER URL: {current_url}') + with requests.get(current_url) as r: + if r.ok: + ly = json.loads(r.content) + else: + raise ValueError(f'API request failed: {current_url}') + assert 'data' in ly + + name = ly['data']['attributes']['name'] + print(f'LAYER NAME: {name}') + + old_value = ly['data']['attributes']['layerConfig'] + + #Convert to string and replace + old_str = json.dumps(old_value) + find_count = old_str.count(find) + + new_str = old_str.replace(find, replace) + replace_count = new_str.count(replace) + + #Get obj back with replacement + new_value = json.loads(new_str) + + print(f'\nOLD VALUE: {old_value}') + print(f'\nNEW VALUE: {new_value}') + + print(f'\nStarted with {find_count} instances of "{find}" and ended up with {replace_count} instances of "{replace}".') + + update = json.dumps({layer_field: new_value}) + + if dry_run == False: + with requests.patch(current_url, headers=create_headers(), data=update) as r: + if r.ok: + print('\nDONE') + else: + raise ValueError(f'API request failed: {current_url}') + assert 'data' in ds + else: + print('\nDRY RUN') From 60cb5c7e9b4debf9ffbd812ecd60df82dde4fcd0 Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Wed, 7 Jun 2023 13:55:40 -0400 Subject: [PATCH 3/6] clone_children=True + new line at end of file --- clone_dataset/clone_dataset_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clone_dataset/clone_dataset_tool.py b/clone_dataset/clone_dataset_tool.py index 31368a5..e14d7f0 100644 --- a/clone_dataset/clone_dataset_tool.py +++ b/clone_dataset/clone_dataset_tool.py @@ -264,7 +264,7 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase 'name': new_dataset_name } # Clone dataset -new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=clone_children, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only) +new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=True, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only) print('new dataset API ID:' + new_dataset_id) @@ -272,4 +272,4 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase # Delete dataset (By default this will delete the new dataset you just created) #dataset_to_delete = LMIPy.Dataset(new_dataset_id) #delete_url = f'{dataset_to_delete.server}/v1/dataset/{dataset_to_delete.id}' -#requests.delete(delete_url, headers=create_headers()) \ No newline at end of file +#requests.delete(delete_url, headers=create_headers()) From 896603cd25919b98d51e515abc534bbf40f4849a Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Wed, 7 Jun 2023 19:07:04 +0000 Subject: [PATCH 4/6] Use one env variable + update deletion code --- clone_dataset/clone_dataset_tool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clone_dataset/clone_dataset_tool.py b/clone_dataset/clone_dataset_tool.py index e14d7f0..2515ffb 100644 --- a/clone_dataset/clone_dataset_tool.py +++ b/clone_dataset/clone_dataset_tool.py @@ -46,7 +46,7 @@ def create_headers(): return { 'content-type': "application/json", - 'authorization': "{}".format(os.getenv('apiToken')), + 'authorization': f'Bearer {API_TOKEN}', } def clone_ds(self, token=None, enviro='preproduction', clone_server=None, dataset_params=None, clone_children=True, clone_first_layer_only=True, clone_default_widget_only=True, published = False): @@ -268,8 +268,8 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase print('new dataset API ID:' + new_dataset_id) - -# Delete dataset (By default this will delete the new dataset you just created) -#dataset_to_delete = LMIPy.Dataset(new_dataset_id) +# Delete dataset and all its layers/widgets (By default this will delete the new dataset you just created) +#delete_dataset_id = new_dataset_id +#dataset_to_delete = LMIPy.Dataset(delete_dataset_id) #delete_url = f'{dataset_to_delete.server}/v1/dataset/{dataset_to_delete.id}' #requests.delete(delete_url, headers=create_headers()) From 1e39e32d97a6cc3a784f54f89ab98aa7be4b21b0 Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Tue, 27 Jun 2023 16:31:06 +0000 Subject: [PATCH 5/6] Add jupyter extentions to devcontainer --- .devcontainer/devcontainer.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7fddde3..d9a33fb 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ { "name": "Python 3", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/python:0-3.11", + "image": "mcr.microsoft.com/devcontainers/python:0-3.7", // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, @@ -18,7 +18,13 @@ "customizations": { "vscode": { "extensions": [ - "ms-python.python" + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-slideshow" ] } } From 80c49cb24153f42ce4ce2662078b9a15a66e2708 Mon Sep 17 00:00:00 2001 From: Chris Rowe Date: Tue, 27 Jun 2023 16:31:34 +0000 Subject: [PATCH 6/6] Keep original app id --- clone_dataset/clone_dataset_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clone_dataset/clone_dataset_tool.py b/clone_dataset/clone_dataset_tool.py index 2515ffb..5003a37 100644 --- a/clone_dataset/clone_dataset_tool.py +++ b/clone_dataset/clone_dataset_tool.py @@ -68,7 +68,7 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase clone_dataset_attr[k] = dataset_params.get(k, '') payload = { 'dataset': { - 'application': ['rw'], + 'application': clone_dataset_attr['application'], 'connectorType': clone_dataset_attr['connectorType'], 'connectorUrl': clone_dataset_attr['connectorUrl'], 'tableName': clone_dataset_attr['tableName'],