From 296d0770ecf64fa344a8b42c7799a1235923a2cf Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 27 Jun 2018 22:47:43 +0200 Subject: [PATCH 1/7] Support retry on delete. Bump version to 0.0.15. --- cloudify_occi_plugin/tasks.py | 14 ++++++++++++-- plugin.yaml | 16 ++++++++++++++-- setup.py | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index 22d0b55..1c45c4c 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -114,13 +114,23 @@ def stop(client, **kwargs): @operation @with_client -def delete(client, **kwargs): +def delete(client, delete_retry_interval=30, **kwargs): ctx.logger.info('Deleting') url = ctx.instance.runtime_properties.get('occi_resource_url') if url: try: client.delete(url) - finally: + except Exception: + pass + + # check the resource is deleted + try: + client.describe(url) + + return ctx.operation.retry( + message='Waiting for resource to delete', + retry_after=delete_retry_interval) + except Exception: delete_runtime_properties(ctx) diff --git a/plugin.yaml b/plugin.yaml index 0cdca3b..dbba904 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,9 +1,9 @@ plugins: occi: executor: central_deployment_agent - source: https://github.com/ICS-MU/westlife-cloudify-occi-plugin/archive/0.0.14.zip + source: https://github.com/ICS-MU/westlife-cloudify-occi-plugin/archive/0.0.15.zip package_name: cloudify-occi-plugin - package_version: '0.0.14' + package_version: '0.0.15' data_types: cloudify.datatypes.occi.Config: @@ -184,6 +184,12 @@ node_types: implementation: occi.cloudify_occi_plugin.tasks.stop delete: implementation: occi.cloudify_occi_plugin.tasks.delete + inputs: + delete_retry_interval: + description: > + Polling interval until the server is deleted + type: integer + default: 30 cloudify.occi.nodes.Volume: derived_from: cloudify.nodes.Volume @@ -222,3 +228,9 @@ node_types: default: 30 delete: implementation: occi.cloudify_occi_plugin.tasks.delete + inputs: + delete_retry_interval: + description: > + Polling interval until the volume is deleted + type: integer + default: 30 diff --git a/setup.py b/setup.py index dba6295..268589a 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setuptools.setup( zip_safe=False, name='cloudify-occi-plugin', - version='0.0.14', + version='0.0.15', author='Vlastimil Holer', author_email='holer@ics.muni.cz', packages=['cloudify_occi_plugin', From 80da97adec14a455de4050618154edc809b52848 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Sat, 30 Jun 2018 00:11:31 +0200 Subject: [PATCH 2/7] Default image name with random string --- cloudify_occi_plugin/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index 1c45c4c..26da8fb 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -1,4 +1,6 @@ # import cloudify +import random +import string from cloudify import ctx # from cloudify.exceptions import NonRecoverableError, RecoverableError @@ -141,7 +143,8 @@ def create_volume(client, **kwargs): size = ctx.node.properties.get('size', dict()) name = ctx.node.properties.get('name') if not name: - name = 'cfy-disk-%s' % ctx.instance.id + rand = ''.join(random.sample((string.letters+string.digits)*6, 6)) + name = 'cfy-disk-%s-%s' % (ctx.instance.id, rand) availability_zone = ctx.node.properties.get('availability_zone') url = client.create_volume(name, size, availability_zone) From 3718e84253a9c28ff10331081ec658bad6a034ee Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Sat, 30 Jun 2018 02:26:03 +0200 Subject: [PATCH 3/7] Parameter wait_finish for delete/stop/unlink --- cloudify_occi_plugin/tasks.py | 33 +++++++++++++++++++-------------- plugin.yaml | 26 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index 26da8fb..2133c0e 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -109,14 +109,15 @@ def stop(client, **kwargs): state = get_instance_state(ctx, client) # check again for suspended instance or retry - if state not in ['suspended', 'inactive']: + if (state not in ['suspended', 'inactive']) and kwargs['wait_finish']: return ctx.operation.retry( - message='Waiting for server to stop (state: %s)' % (state,)) + message='Waiting for server to stop (state: %s)' % (state,), + retry_after=kwargs['stop_retry_interval']) @operation @with_client -def delete(client, delete_retry_interval=30, **kwargs): +def delete(client, **kwargs): ctx.logger.info('Deleting') url = ctx.instance.runtime_properties.get('occi_resource_url') if url: @@ -129,9 +130,12 @@ def delete(client, delete_retry_interval=30, **kwargs): try: client.describe(url) - return ctx.operation.retry( - message='Waiting for resource to delete', - retry_after=delete_retry_interval) + if kwargs['wait_finish']: + return ctx.operation.retry( + message='Waiting for resource to delete', + retry_after=kwargs['delete_retry_interval']) + else: + raise Exception except Exception: delete_runtime_properties(ctx) @@ -158,8 +162,8 @@ def start_volume(client, start_retry_interval, **kwargs): state = get_instance_state(ctx, client) if (state != 'online'): return ctx.operation.retry( - message='Waiting for volume to start (state: %s)' % (state,), - retry_after=start_retry_interval) + message='Waiting for volume to start (state: %s)' % (state,), + retry_after=start_retry_interval) @operation @@ -179,11 +183,11 @@ def attach_volume(client, attach_retry_interval, **kwargs): if state == 'active': ctx.source.instance.runtime_properties['device'] = \ - desc[0]['attributes']['occi']['storagelink']['deviceid'] + desc[0]['attributes']['occi']['storagelink']['deviceid'] else: return ctx.operation.retry( - message='Waiting for volume to attach (state: %s)' % (state,), - retry_after=attach_retry_interval) + message='Waiting for volume to attach (state: %s)' % (state,), + retry_after=attach_retry_interval) @operation @@ -198,9 +202,10 @@ def detach_volume(client, detach_retry_interval, **kwargs): if state == 'active': client.delete(url) - return ctx.operation.retry( - message='Waiting for volume to detach (state: %s)' % (state,), - retry_after=detach_retry_interval) + if kwargs['wait_finish']: + return ctx.operation.retry( + message='Waiting for volume to detach (state: %s)' % (state,), + retry_after=detach_retry_interval) except Exception: if 'occi_link_url' in ctx.source.instance.runtime_properties: del ctx.source.instance.runtime_properties['occi_link_url'] diff --git a/plugin.yaml b/plugin.yaml index dbba904..e2e7e76 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -143,6 +143,11 @@ relationships: Polling interval until volum is detached in seconds type: integer default: 60 + wait_finish: + description: > + Wait until volume is sucessfully detached. + type: boolean + default: true node_types: cloudify.occi.nodes.Server: @@ -182,6 +187,17 @@ node_types: default: 30 stop: implementation: occi.cloudify_occi_plugin.tasks.stop + inputs: + stop_retry_interval: + description: > + Polling interval until the server is stopped + type: integer + default: 30 + wait_finish: + description: > + Wait until server is successfully stopped + type: boolean + default: true delete: implementation: occi.cloudify_occi_plugin.tasks.delete inputs: @@ -190,6 +206,11 @@ node_types: Polling interval until the server is deleted type: integer default: 30 + wait_finish: + description: > + Wait until server is successfully deleted + type: boolean + default: true cloudify.occi.nodes.Volume: derived_from: cloudify.nodes.Volume @@ -234,3 +255,8 @@ node_types: Polling interval until the volume is deleted type: integer default: 30 + wait_finish: + description: > + Wait until volume is successfully deleted + type: boolean + default: true From 77f1f84067e8cb673245478d5ca86e7cbc6a5a56 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Sat, 30 Jun 2018 15:46:20 +0200 Subject: [PATCH 4/7] Clean volume OCCI metadata if not wait_finish --- cloudify_occi_plugin/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index 2133c0e..a0d9fc9 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -206,6 +206,8 @@ def detach_volume(client, detach_retry_interval, **kwargs): return ctx.operation.retry( message='Waiting for volume to detach (state: %s)' % (state,), retry_after=detach_retry_interval) + else: + raise Exception except Exception: if 'occi_link_url' in ctx.source.instance.runtime_properties: del ctx.source.instance.runtime_properties['occi_link_url'] From 6d600b5168c08bb49d1787478b7316fd80c6cb4a Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Sat, 30 Jun 2018 18:43:57 +0200 Subject: [PATCH 5/7] Support linked resources cleanup on delete --- cloudify_occi_plugin/tasks.py | 34 +++++++++++++++++++++++++++++++++- cloudify_occi_plugin/utils.py | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index a0d9fc9..2398aba 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -119,8 +119,25 @@ def stop(client, **kwargs): @with_client def delete(client, **kwargs): ctx.logger.info('Deleting') + + cln = ctx.instance.runtime_properties.get('occi_cleanup_urls') url = ctx.instance.runtime_properties.get('occi_resource_url') + if url: + # store linked resources for post-delete cleanup + if cln is None: + cln = [] + + try: + desc = client.describe(url) + for link in desc[0]['links']: + if link['rel'].endswith('#storage'): + cln.append(link['target']) + except Exception: + pass + finally: + ctx.instance.runtime_properties['occi_cleanup_urls'] = cln + try: client.delete(url) except Exception: @@ -137,7 +154,22 @@ def delete(client, **kwargs): else: raise Exception except Exception: - delete_runtime_properties(ctx) + if cln: + del ctx.instance.runtime_properties['occi_resource_url'] + + return ctx.operation.retry( + message='Waiting for linked resources to delete', + retry_after=kwargs['delete_retry_interval']) + + # cleanup linked resources + elif cln: + for link in cln: + try: + client.delete(link) + except Exception: + pass + + delete_runtime_properties(ctx) @operation diff --git a/cloudify_occi_plugin/utils.py b/cloudify_occi_plugin/utils.py index 3227912..289810c 100644 --- a/cloudify_occi_plugin/utils.py +++ b/cloudify_occi_plugin/utils.py @@ -7,7 +7,7 @@ 'ip', 'networks', 'device', 'user', 'port', 'password', 'key', 'occi_resource_id', 'occi_resource_url', 'occi_resource_title', - 'occi_network_link_url', + 'occi_network_link_url', 'occi_cleanup_urls' ] From 24643a7a26ff478de7c678802955ad4847a7d6ba Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 3 Jul 2018 23:05:26 +0200 Subject: [PATCH 6/7] Support volume unlink skip_action. Wait_finish cleanups. --- cloudify_occi_plugin/tasks.py | 14 ++++++++++---- plugin.yaml | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index 2398aba..bdfbf7e 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -109,7 +109,8 @@ def stop(client, **kwargs): state = get_instance_state(ctx, client) # check again for suspended instance or retry - if (state not in ['suspended', 'inactive']) and kwargs['wait_finish']: + if ((state not in ['suspended', 'inactive']) and + kwargs.get('wait_finish', True)): return ctx.operation.retry( message='Waiting for server to stop (state: %s)' % (state,), retry_after=kwargs['stop_retry_interval']) @@ -131,7 +132,8 @@ def delete(client, **kwargs): try: desc = client.describe(url) for link in desc[0]['links']: - if link['rel'].endswith('#storage'): + if (link['rel'].endswith('#storage') and + not link['id'].endswith('_disk_0')): cln.append(link['target']) except Exception: pass @@ -147,7 +149,7 @@ def delete(client, **kwargs): try: client.describe(url) - if kwargs['wait_finish']: + if kwargs.get('wait_finish', True): return ctx.operation.retry( message='Waiting for resource to delete', retry_after=kwargs['delete_retry_interval']) @@ -225,6 +227,10 @@ def attach_volume(client, attach_retry_interval, **kwargs): @operation @with_client def detach_volume(client, detach_retry_interval, **kwargs): + if kwargs.get('skip_action'): + ctx.logger.info('Volume detach skipped by configuration') + return + ctx.logger.info('Detaching volume') url = ctx.source.instance.runtime_properties['occi_link_url'] @@ -234,7 +240,7 @@ def detach_volume(client, detach_retry_interval, **kwargs): if state == 'active': client.delete(url) - if kwargs['wait_finish']: + if kwargs.get('wait_finish', True): return ctx.operation.retry( message='Waiting for volume to detach (state: %s)' % (state,), retry_after=detach_retry_interval) diff --git a/plugin.yaml b/plugin.yaml index e2e7e76..334d040 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -123,6 +123,16 @@ relationships: Polling interval until volum is detached in seconds type: integer default: 60 + wait_finish: + description: > + Wait until volume is sucessfully detached. + type: boolean + default: true + skip_action: + description: > + Skip this action, don't do anything. + type: boolean + default: false cloudify.occi.relationships.volume_contained_in_server: derived_from: cloudify.relationships.contained_in target_interfaces: @@ -148,6 +158,11 @@ relationships: Wait until volume is sucessfully detached. type: boolean default: true + skip_action: + description: > + Skip this action, don't do anything. + type: boolean + default: false node_types: cloudify.occi.nodes.Server: From 2ad86e18317cb34038bbdcfffa3bcb820ff87ebb Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Jul 2018 00:18:09 +0200 Subject: [PATCH 7/7] Clean metadata on skipped volume unlink --- cloudify_occi_plugin/tasks.py | 20 +++++++++++--------- cloudify_occi_plugin/utils.py | 3 ++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cloudify_occi_plugin/tasks.py b/cloudify_occi_plugin/tasks.py index bdfbf7e..66a7984 100644 --- a/cloudify_occi_plugin/tasks.py +++ b/cloudify_occi_plugin/tasks.py @@ -151,8 +151,8 @@ def delete(client, **kwargs): if kwargs.get('wait_finish', True): return ctx.operation.retry( - message='Waiting for resource to delete', - retry_after=kwargs['delete_retry_interval']) + message='Waiting for resource to delete', + retry_after=kwargs['delete_retry_interval']) else: raise Exception except Exception: @@ -160,8 +160,8 @@ def delete(client, **kwargs): del ctx.instance.runtime_properties['occi_resource_url'] return ctx.operation.retry( - message='Waiting for linked resources to delete', - retry_after=kwargs['delete_retry_interval']) + message='Waiting for linked resources to delete', + retry_after=kwargs['delete_retry_interval']) # cleanup linked resources elif cln: @@ -205,12 +205,12 @@ def start_volume(client, start_retry_interval, **kwargs): def attach_volume(client, attach_retry_interval, **kwargs): ctx.logger.info('Attaching volume') - url = ctx.source.instance.runtime_properties.get('occi_link_url') + url = ctx.source.instance.runtime_properties.get('occi_storage_link_url') if not url: srv_url = ctx.target.instance.runtime_properties['occi_resource_url'] vol_url = ctx.source.instance.runtime_properties['occi_resource_url'] url = client.link(vol_url, srv_url) - ctx.source.instance.runtime_properties['occi_link_url'] = url + ctx.source.instance.runtime_properties['occi_storage_link_url'] = url desc = client.describe(url) state = desc[0]['attributes']['occi']['storagelink']['state'] @@ -229,10 +229,12 @@ def attach_volume(client, attach_retry_interval, **kwargs): def detach_volume(client, detach_retry_interval, **kwargs): if kwargs.get('skip_action'): ctx.logger.info('Volume detach skipped by configuration') + del ctx.source.instance.runtime_properties['occi_storage_link_url'] + del ctx.source.instance.runtime_properties['device'] return ctx.logger.info('Detaching volume') - url = ctx.source.instance.runtime_properties['occi_link_url'] + url = ctx.source.instance.runtime_properties['occi_storage_link_url'] try: desc = client.describe(url) @@ -247,7 +249,7 @@ def detach_volume(client, detach_retry_interval, **kwargs): else: raise Exception except Exception: - if 'occi_link_url' in ctx.source.instance.runtime_properties: - del ctx.source.instance.runtime_properties['occi_link_url'] + if 'occi_storage_link_url' in ctx.source.instance.runtime_properties: + del ctx.source.instance.runtime_properties['occi_storage_link_url'] if 'device' in ctx.source.instance.runtime_properties: del ctx.source.instance.runtime_properties['device'] diff --git a/cloudify_occi_plugin/utils.py b/cloudify_occi_plugin/utils.py index 289810c..305b18f 100644 --- a/cloudify_occi_plugin/utils.py +++ b/cloudify_occi_plugin/utils.py @@ -7,7 +7,8 @@ 'ip', 'networks', 'device', 'user', 'port', 'password', 'key', 'occi_resource_id', 'occi_resource_url', 'occi_resource_title', - 'occi_network_link_url', 'occi_cleanup_urls' + 'occi_storage_link_url', 'occi_network_link_url', + 'occi_cleanup_urls' ]