-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement new feature copy-links (#23)
- Loading branch information
Showing
8 changed files
with
252 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
from toolbox.commands.copy_links import copy_links | ||
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace | ||
from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token | ||
import pytest as pytest | ||
|
||
|
||
# Mocks | ||
|
||
def mock_list_links_on_source_workspace(self, workspace_name): | ||
if workspace_name == 'workspace_source': | ||
return ['link1', 'link2', 'link3'] | ||
return [] | ||
|
||
|
||
# Scenarios | ||
|
||
def test_copy_links_when_workspace_source_does_not_exist(mocker): | ||
# GIVEN | ||
client_space_mock = mocker.patch.object(Token, 'get_client_space_id', autospec=True) | ||
client_space_mock.return_value = 'cid' | ||
api_authenticate_mock = mocker.patch.object(DataGalaxyApiAuthentication, 'authenticate', autospec=True) | ||
api_authenticate_mock.return_value = 'token' | ||
workspaces = mocker.patch.object(DataGalaxyApiWorkspace, 'list_workspaces', autospec=True) | ||
workspaces.return_value = ['workspace_source'] | ||
workspace_source_mock = mocker.patch.object(DataGalaxyApiWorkspace, 'get_workspace', autospec=True) | ||
workspace_source_mock.return_value = None | ||
|
||
# ASSERT / VERIFY | ||
with pytest.raises(Exception, match='workspace workspace_source does not exist'): | ||
copy_links( | ||
url_source='url_source', | ||
token_source='token_source', | ||
url_target='url_target', | ||
token_target='token_target', | ||
workspace_source_name='workspace_source', | ||
workspace_target_name='workspace_target' | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
import requests as requests | ||
from toolbox.api.datagalaxy_api import DataGalaxyBulkResult | ||
|
||
|
||
class DataGalaxyApiLinks: | ||
def __init__(self, url: str, access_token: str, workspace: dict): | ||
self.url = url | ||
self.access_token = access_token | ||
self.workspace = workspace | ||
|
||
def bulk_create_links(self, workspace_name: str, links: list) -> DataGalaxyBulkResult: | ||
# Creating links between entities based on their path | ||
if self.workspace["isVersioningEnabled"]: | ||
raise Exception('Workspace with versioning enabled are currently not supported.') | ||
|
||
version_id = self.workspace['defaultVersionId'] | ||
headers = {'Authorization': f"Bearer {self.access_token}"} | ||
response = requests.post(f"{self.url}/links/bulktree/{version_id}", json=links, | ||
headers=headers) | ||
code = response.status_code | ||
body_json = response.json() | ||
if code != 201: | ||
raise Exception(body_json['error']) | ||
|
||
result = DataGalaxyBulkResult(total=body_json["total"], | ||
created=body_json["created"], | ||
deleted=body_json["deleted"], | ||
unchanged=body_json["unchanged"], | ||
updated=body_json["updated"]) | ||
return result |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
import logging | ||
from typing import Optional | ||
|
||
from toolbox.api.datagalaxy_api import get_access_token, Token, DataGalaxyBulkResult | ||
from toolbox.api.datagalaxy_api_usages import DataGalaxyApiUsages | ||
from toolbox.api.datagalaxy_api_glossary import DataGalaxyApiGlossary | ||
from toolbox.api.datagalaxy_api_dictionary import DataGalaxyApiDictionary | ||
from toolbox.api.datagalaxy_api_links import DataGalaxyApiLinks | ||
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace | ||
|
||
|
||
def copy_links(url_source: str, | ||
url_target: Optional[str], | ||
token_source: str, | ||
token_target: Optional[str], | ||
workspace_source_name: str, | ||
workspace_target_name: str) -> DataGalaxyBulkResult: | ||
if token_target is None: | ||
token_target = token_source | ||
|
||
if url_target is None: | ||
url_target = url_source | ||
|
||
integration_token_source = Token(token_source) | ||
integration_token_target = Token(token_target) | ||
source_access_token = get_access_token(url_source, integration_token_source) | ||
target_access_token = get_access_token(url_target, integration_token_target) | ||
workspaces_api_on_source_env = DataGalaxyApiWorkspace( | ||
url=url_source, | ||
access_token=source_access_token) | ||
source_workspace = workspaces_api_on_source_env.get_workspace(workspace_source_name) | ||
if source_workspace is None: | ||
raise Exception(f'workspace {workspace_source_name} does not exist') | ||
|
||
workspaces_api_on_target_env = DataGalaxyApiWorkspace( | ||
url=url_target, | ||
access_token=target_access_token | ||
) | ||
target_workspace = workspaces_api_on_target_env.get_workspace(workspace_target_name) | ||
if target_workspace is None: | ||
raise Exception(f'workspace {workspace_target_name} does not exist') | ||
|
||
source_usages_api = DataGalaxyApiUsages( | ||
url=url_source, | ||
access_token=source_access_token, | ||
workspace=source_workspace | ||
) | ||
source_glossary_api = DataGalaxyApiGlossary( | ||
url=url_source, | ||
access_token=source_access_token, | ||
workspace=source_workspace | ||
) | ||
source_dictionary_api = DataGalaxyApiDictionary( | ||
url=url_source, | ||
access_token=source_access_token, | ||
workspace=source_workspace | ||
) | ||
|
||
# Find all links in source workspace | ||
source_usages = source_usages_api.list_usages(workspace_source_name, include_links=True) | ||
source_properties = source_glossary_api.list_properties(workspace_source_name, include_links=True) | ||
source_sources = source_dictionary_api.list_sources(workspace_source_name, include_links=True) | ||
source_containers = source_dictionary_api.list_containers(workspace_source_name, include_links=True) | ||
source_structures = source_dictionary_api.list_structures(workspace_source_name, include_links=True) | ||
source_fields = source_dictionary_api.list_fields(workspace_source_name, include_links=True) | ||
|
||
# Collecting all links | ||
links = parse_links(source_usages) | ||
links += parse_links(source_properties) | ||
links += parse_links(source_sources) | ||
links += parse_links(source_containers) | ||
links += parse_links(source_structures) | ||
links += parse_links(source_fields) | ||
logging.info(f'copy-links - {len(links)} links found') | ||
|
||
target_links_api = DataGalaxyApiLinks( | ||
url=url_target, | ||
access_token=target_access_token, | ||
workspace=target_workspace | ||
) | ||
|
||
# Creating links in target workspace | ||
return target_links_api.bulk_create_links(workspace_name=workspace_target_name, links=links) | ||
|
||
|
||
def parse_links(objs: list) -> list: | ||
links = [] | ||
for obj in objs: | ||
# DPI are ignored since they are handled differently | ||
if "DataProcessingItem" in obj["typePath"]: | ||
continue | ||
for key in obj["links"]: | ||
for dest in obj["links"][key]: | ||
if "DataProcessingItem" in dest["typePath"]: | ||
continue | ||
logging.info(f'copy-links - {obj["path"]} {key} {dest["path"]}') | ||
link = { | ||
'fromPath': obj["path"], | ||
'fromType': obj["typePath"], | ||
'linkType': key, | ||
'toPath': dest["path"], | ||
'toType': dest["typePath"] | ||
} | ||
links.append(link) | ||
return links | ||
|
||
|
||
def copy_links_parse(subparsers): | ||
# create the parser for the "copy_links" command | ||
copy_links_parse = subparsers.add_parser('copy-links', help='copy-links help') | ||
copy_links_parse.add_argument( | ||
'--url-source', | ||
type=str, | ||
help='url source environnement', | ||
required=True) | ||
copy_links_parse.add_argument( | ||
'--token-source', | ||
type=str, | ||
help='integration token source environnement', | ||
required=True) | ||
copy_links_parse.add_argument( | ||
'--url-target', | ||
type=str, | ||
help='url target environnement (if undefined, use url source)') | ||
copy_links_parse.add_argument( | ||
'--token-target', | ||
type=str, | ||
help='integration token target environnement (if undefined, use token source)') | ||
copy_links_parse.add_argument( | ||
'--workspace-source', | ||
type=str, | ||
help='workspace source name', | ||
required=True) | ||
copy_links_parse.add_argument( | ||
'--workspace-target', | ||
type=str, | ||
help='workspace target name', | ||
required=True) |