Add edges _key _from _to test to d3 json uploader

multinet/api/tests/data/miserables-key-to-from.json

@@ -0,0 +1,183 @@
+{
+  "nodes": [
+    { "_key": "Myriel", "group": 1 },
+    { "_key": "Napoleon", "group": 1 },
+    { "_key": "Mlle.Baptistine", "group": 1 },
+    { "_key": "Mme.Magloire", "group": 1 },
+    { "_key": "CountessdeLo", "group": 1 },
+    { "_key": "Geborand", "group": 1 },
+    { "_key": "Champtercier", "group": 1 },
+    { "_key": "Cravatte", "group": 1 },
+    { "_key": "Count", "group": 1 },
+    { "_key": "OldMan", "group": 1 },
+    { "_key": "Labarre", "group": 2 },
+    { "_key": "Valjean", "group": 2 },
+    { "_key": "Marguerite", "group": 3 },
+    { "_key": "Mme.deR", "group": 2 },
+    { "_key": "Isabeau", "group": 2 },
+    { "_key": "Gervais", "group": 2 },
+    { "_key": "Tholomyes", "group": 3 },
+    { "_key": "Listolier", "group": 3 },
+    { "_key": "Fameuil", "group": 3 },
+    { "_key": "Blacheville", "group": 3 },
+    { "_key": "Favourite", "group": 3 },
+    { "_key": "Dahlia", "group": 3 },
+    { "_key": "Zephine", "group": 3 },
+    { "_key": "Fantine", "group": 3 },
+    { "_key": "Mme.Thenardier", "group": 4 },
+    { "_key": "Thenardier", "group": 4 },
+    { "_key": "Cosette", "group": 5 },
+    { "_key": "Javert", "group": 4 },
+    { "_key": "Fauchelevent", "group": 0 },
+    { "_key": "Bamatabois", "group": 2 },
+    { "_key": "Perpetue", "group": 3 },
+    { "_key": "Simplice", "group": 2 },
+    { "_key": "Scaufflaire", "group": 2 },
+    { "_key": "Woman1", "group": 2 },
+    { "_key": "Judge", "group": 2 },
+    { "_key": "Champmathieu", "group": 2 },
+    { "_key": "Brevet", "group": 2 },
+    { "_key": "Chenildieu", "group": 2 },
+    { "_key": "Cochepaille", "group": 2 },
+    { "_key": "Pontmercy", "group": 4 },
+    { "_key": "Boulatruelle", "group": 6 },
+    { "_key": "Eponine", "group": 4 },
+    { "_key": "Anzelma", "group": 4 },
+    { "_key": "Woman2", "group": 5 },
+    { "_key": "MotherInnocent", "group": 0 },
+    { "_key": "Gribier", "group": 0 },
+    { "_key": "Jondrette", "group": 7 },
+    { "_key": "Mme.Burgon", "group": 7 },
+    { "_key": "Gavroche", "group": 8 },
+    { "_key": "Gillenormand", "group": 5 },
+    { "_key": "Magnon", "group": 5 },
+    { "_key": "Mlle.Gillenormand", "group": 5 },
+    { "_key": "Mme.Pontmercy", "group": 5 },
+    { "_key": "Mlle.Vaubois", "group": 5 },
+    { "_key": "Lt.Gillenormand", "group": 5 },
+    { "_key": "Marius", "group": 8 },
+    { "_key": "BaronessT", "group": 5 },
+    { "_key": "Mabeuf", "group": 8 },
+    { "_key": "Enjolras", "group": 8 },
+    { "_key": "Combeferre", "group": 8 },
+    { "_key": "Prouvaire", "group": 8 },
+    { "_key": "Feuilly", "group": 8 },
+    { "_key": "Courfeyrac", "group": 8 },
+    { "_key": "Bahorel", "group": 8 },
+    { "_key": "Bossuet", "group": 8 },
+    { "_key": "Joly", "group": 8 },
+    { "_key": "Grantaire", "group": 8 },
+    { "_key": "MotherPlutarch", "group": 9 },
+    { "_key": "Gueulemer", "group": 4 },
+    { "_key": "Babet", "group": 4 },
+    { "_key": "Claquesous", "group": 4 },
+    { "_key": "Montparnasse", "group": 4 },
+    { "_key": "Toussaint", "group": 5 },
+    { "_key": "Child1", "group": 10 },
+    { "_key": "Child2", "group": 10 },
+    { "_key": "Brujon", "group": 4 },
+    { "_key": "Mme.Hucheloup", "group": 8 }
+  ],
+  "edges": [
+    { "_from": "Napoleon", "_to": "Myriel", "value": 1 },
+    { "_from": "Mlle.Baptistine", "_to": "Myriel", "value": 8 },
+    { "_from": "Mme.Magloire", "_to": "Myriel", "value": 10 },
+    { "_from": "Mme.Magloire", "_to": "Mlle.Baptistine", "value": 6 },
+    { "_from": "CountessdeLo", "_to": "Myriel", "value": 1 },
+    { "_from": "Geborand", "_to": "Myriel", "value": 1 },
+    { "_from": "Champtercier", "_to": "Myriel", "value": 1 },
+    { "_from": "Cravatte", "_to": "Myriel", "value": 1 },
+    { "_from": "Count", "_to": "Myriel", "value": 2 },
+    { "_from": "OldMan", "_to": "Myriel", "value": 1 },
+    { "_from": "Valjean", "_to": "Labarre", "value": 1 },
+    { "_from": "Valjean", "_to": "Mme.Magloire", "value": 3 },
+    { "_from": "Valjean", "_to": "Mlle.Baptistine", "value": 3 },
+    { "_from": "Valjean", "_to": "Myriel", "value": 5 },
+    { "_from": "Marguerite", "_to": "Valjean", "value": 1 },
+    { "_from": "Mme.deR", "_to": "Valjean", "value": 1 },
+    { "_from": "Isabeau", "_to": "Valjean", "value": 1 },
+    { "_from": "Gervais", "_to": "Valjean", "value": 1 },
+    { "_from": "Listolier", "_to": "Tholomyes", "value": 4 },
+    { "_from": "Fameuil", "_to": "Tholomyes", "value": 4 },
+    { "_from": "Fameuil", "_to": "Listolier", "value": 4 },
+    { "_from": "Blacheville", "_to": "Tholomyes", "value": 4 },
+    { "_from": "Blacheville", "_to": "Listolier", "value": 4 },
+    { "_from": "Blacheville", "_to": "Fameuil", "value": 4 },
+    { "_from": "Favourite", "_to": "Tholomyes", "value": 3 },
+    { "_from": "Favourite", "_to": "Listolier", "value": 3 },
+    { "_from": "Favourite", "_to": "Fameuil", "value": 3 },
+    { "_from": "Favourite", "_to": "Blacheville", "value": 4 },
+    { "_from": "Dahlia", "_to": "Tholomyes", "value": 3 },
+    { "_from": "Dahlia", "_to": "Listolier", "value": 3 },
+    { "_from": "Dahlia", "_to": "Fameuil", "value": 3 },
+    { "_from": "Dahlia", "_to": "Blacheville", "value": 3 },
+    { "_from": "Dahlia", "_to": "Favourite", "value": 5 },
+    { "_from": "Zephine", "_to": "Tholomyes", "value": 3 },
+    { "_from": "Zephine", "_to": "Listolier", "value": 3 },
+    { "_from": "Zephine", "_to": "Fameuil", "value": 3 },
+    { "_from": "Zephine", "_to": "Blacheville", "value": 3 },
+    { "_from": "Zephine", "_to": "Favourite", "value": 4 },
+    { "_from": "Zephine", "_to": "Dahlia", "value": 4 },
+    { "_from": "Fantine", "_to": "Tholomyes", "value": 3 },
+    { "_from": "Fantine", "_to": "Listolier", "value": 3 },
+    { "_from": "Fantine", "_to": "Fameuil", "value": 3 },
+    { "_from": "Fantine", "_to": "Blacheville", "value": 3 },
+    { "_from": "Fantine", "_to": "Favourite", "value": 4 },
+    { "_from": "Fantine", "_to": "Dahlia", "value": 4 },
+    { "_from": "Fantine", "_to": "Zephine", "value": 4 },
+    { "_from": "Fantine", "_to": "Marguerite", "value": 2 },
+    { "_from": "Fantine", "_to": "Valjean", "value": 9 },
+    { "_from": "Mme.Thenardier", "_to": "Fantine", "value": 2 },
+    { "_from": "Mme.Thenardier", "_to": "Valjean", "value": 7 },
+    { "_from": "Thenardier", "_to": "Mme.Thenardier", "value": 13 },
+    { "_from": "Thenardier", "_to": "Fantine", "value": 1 },
+    { "_from": "Thenardier", "_to": "Valjean", "value": 12 },
+    { "_from": "Cosette", "_to": "Mme.Thenardier", "value": 4 },
+    { "_from": "Cosette", "_to": "Valjean", "value": 31 },
+    { "_from": "Cosette", "_to": "Tholomyes", "value": 1 },
+    { "_from": "Cosette", "_to": "Thenardier", "value": 1 },
+    { "_from": "Javert", "_to": "Valjean", "value": 17 },
+    { "_from": "Javert", "_to": "Fantine", "value": 5 },
+    { "_from": "Javert", "_to": "Thenardier", "value": 5 },
+    { "_from": "Javert", "_to": "Mme.Thenardier", "value": 1 },
+    { "_from": "Javert", "_to": "Cosette", "value": 1 },
+    { "_from": "Fauchelevent", "_to": "Valjean", "value": 8 },
+    { "_from": "Fauchelevent", "_to": "Javert", "value": 1 },
+    { "_from": "Bamatabois", "_to": "Fantine", "value": 1 },
+    { "_from": "Bamatabois", "_to": "Javert", "value": 1 },
+    { "_from": "Bamatabois", "_to": "Valjean", "value": 2 },
+    { "_from": "Perpetue", "_to": "Fantine", "value": 1 },
+    { "_from": "Simplice", "_to": "Perpetue", "value": 2 },
+    { "_from": "Simplice", "_to": "Valjean", "value": 3 },
+    { "_from": "Simplice", "_to": "Fantine", "value": 2 },
+    { "_from": "Simplice", "_to": "Javert", "value": 1 },
+    { "_from": "Scaufflaire", "_to": "Valjean", "value": 1 },
+    { "_from": "Woman1", "_to": "Valjean", "value": 2 },
+    { "_from": "Woman1", "_to": "Javert", "value": 1 },
+    { "_from": "Judge", "_to": "Valjean", "value": 3 },
+    { "_from": "Judge", "_to": "Bamatabois", "value": 2 },
+    { "_from": "Champmathieu", "_to": "Valjean", "value": 3 },
+    { "_from": "Champmathieu", "_to": "Judge", "value": 3 },
+    { "_from": "Champmathieu", "_to": "Bamatabois", "value": 2 },
+    { "_from": "Brevet", "_to": "Judge", "value": 2 },
+    { "_from": "Brevet", "_to": "Champmathieu", "value": 2 },
+    { "_from": "Brevet", "_to": "Valjean", "value": 2 },
+    { "_from": "Brevet", "_to": "Bamatabois", "value": 1 },
+    { "_from": "Chenildieu", "_to": "Judge", "value": 2 },
+    { "_from": "Chenildieu", "_to": "Champmathieu", "value": 2 },
+    { "_from": "Chenildieu", "_to": "Brevet", "value": 2 },
+    { "_from": "Chenildieu", "_to": "Valjean", "value": 2 },
+    { "_from": "Chenildieu", "_to": "Bamatabois", "value": 1 },
+    { "_from": "Cochepaille", "_to": "Judge", "value": 2 },
+    { "_from": "Cochepaille", "_to": "Champmathieu", "value": 2 },
+    { "_from": "Cochepaille", "_to": "Brevet", "value": 2 },
+    { "_from": "Cochepaille", "_to": "Chenildieu", "value": 2 },
+    { "_from": "Cochepaille", "_to": "Valjean", "value": 2 },
+    { "_from": "Cochepaille", "_to": "Bamatabois", "value": 1 },
+    { "_from": "Pontmercy", "_to": "Thenardier", "value": 1 },
+    { "_from": "Boulatruelle", "_to": "Thenardier", "value": 1 },
+    { "_from": "Eponine", "_to": "Mme.Thenardier", "value": 2 },
+    { "_from": "Eponine", "_to": "Thenardier", "value": 3 },
+    { "_from": "Anzelma", "_to": "Eponine", "value": 2 }
+  ]
+}

multinet/api/tests/test_upload_d3_json.py

@@ -31,6 +31,7 @@
 
 data_dir = pathlib.Path(__file__).parent / 'data'
 miserables_json_file = data_dir / 'miserables.json'
+miserables_key_from_to_json_file = data_dir / 'miserables-key-from-to.json'
 
 
 def json_upload(obj: IO[bytes], name: str, workspace, user):
@@ -78,6 +79,37 @@ def miserables_json(
     }
 
 
+@pytest.fixture
+def miserables_json_key_from_to_field_value(s3ff_field_value_factory, workspace, user) -> str:
+    upload = json_file_upload(miserables_key_from_to_json_file, workspace, user)
+    return s3ff_field_value_factory(upload.blob)
+
+
+@pytest.fixture
+def miserables_json_key_from_to(
+    workspace: Workspace,
+    user: User,
+    authenticated_api_client: APIClient,
+    miserables_json_field_value,
+) -> Dict:
+    # Model creation request
+    workspace.set_user_permission(user, WorkspaceRoleChoice.WRITER)
+    network_name = f't{uuid.uuid4().hex}'
+    r: Response = authenticated_api_client.post(
+        f'/api/workspaces/{workspace.name}/uploads/d3_json/',
+        {
+            'field_value': miserables_json_key_from_to_field_value,
+            'network_name': network_name,
+        },
+        format='json',
+    )
+    WorkspaceRole.objects.filter(workspace=workspace, user=user).delete()
+    return {
+        'response': r,
+        'network_name': network_name,
+    }
+
+
 @pytest.mark.django_db
 def test_create_upload_model(workspace: Workspace, user: User, miserables_json):
     """Test just the response of the model creation, not the task itself."""
@@ -259,6 +291,83 @@ def test_valid_d3_json_task_response(
         assert results[i] == dict_to_fuzzy_arango_doc(link)
 
 
+@pytest.mark.django_db
+def test_valid_d3_json_task_response_key_from_to(
+    workspace: Workspace, user: User, authenticated_api_client: APIClient, miserables_json_key_from_to
+):
+    """Test just the response of the model creation, not the task itself."""
+    # Get upload info
+    workspace.set_user_permission(user, WorkspaceRoleChoice.WRITER)
+    r = miserables_json_key_from_to['response']
+    network_name = miserables_json_key_from_to['network_name']
+    node_table_name = f'{network_name}_nodes'
+    edge_table_name = f'{network_name}_edges'
+
+    # Since we're running with celery_task_always_eager=True, this job is finished
+    r: Response = authenticated_api_client.get(
+        f'/api/workspaces/{workspace.name}/uploads/{r.json()["id"]}/'
+    )
+
+    r_json = r.json()
+    assert r.status_code == 200
+    assert r_json['status'] == Upload.Status.FINISHED
+    assert r_json['error_messages'] is None
+
+    # Check that tables are created
+    for table_name in (node_table_name, edge_table_name):
+        r: Response = authenticated_api_client.get(
+            f'/api/workspaces/{workspace.name}/tables/{table_name}/'
+        )
+        assert r.status_code == 200
+
+    # Check that network was created
+    r: Response = authenticated_api_client.get(
+        f'/api/workspaces/{workspace.name}/networks/{network_name}/'
+    )
+    assert r.status_code == 200
+
+    # Get source data
+    with open(miserables_key_from_to_json_file) as file_stream:
+        loaded_miserables_key_from_to_json_file = json.load(file_stream)
+        nodes = sorted(
+            (d3_node_to_arango_doc(node) for node in loaded_miserables_key_from_to_json_file['nodes']),
+            key=operator.itemgetter('_key'),
+        )
+        edges = sorted(
+            (
+                d3_link_to_arango_doc(link, node_table_name)
+                for link in loaded_miserables_key_from_to_json_file['edges']
+            ),
+            key=operator.itemgetter('_from'),
+        )
+
+    # Check that nodes were ingested correctly
+    r: Response = authenticated_api_client.get(
+        f'/api/workspaces/{workspace.name}/networks/{network_name}/nodes/'
+    )
+
+    r_json = r.json()
+    assert r.status_code == 200
+    assert r_json['count'] == len(nodes)
+
+    results = sorted(r_json['results'], key=operator.itemgetter('_key'))
+    for i, node in enumerate(nodes):
+        assert results[i] == dict_to_fuzzy_arango_doc(node, exclude=['_key'])
+
+    # Check that links were ingested correctly
+    r: Response = authenticated_api_client.get(
+        f'/api/workspaces/{workspace.name}/networks/{network_name}/edges/'
+    )
+
+    r_json = r.json()
+    assert r.status_code == 200
+    assert r_json['count'] == len(edges)
+
+    results = sorted(r_json['results'], key=operator.itemgetter('_from'))
+    for i, link in enumerate(edges):
+        assert results[i] == dict_to_fuzzy_arango_doc(link)
+
+
 @pytest.mark.django_db
 def test_d3_json_task_filter_missing(
     workspace: Workspace,