diff --git a/README.md b/README.md index a607c476bb..40f890b938 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

-![Kedro-Viz Pipeline Visualisation](./.github/img/banner.png) +![Kedro-Viz Pipeline Visualisation](https://raw.githubusercontent.com/kedro-org/kedro-viz/main/.github/img/banner.png)

diff --git a/RELEASE.md b/RELEASE.md index 658b9bea0d..1990e9f633 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -6,6 +6,14 @@ Please follow the established format: - Include the ID number for the related PR (or PRs) in parentheses --> +# Release 6.4.1 + +## Major features and improvements + +## Bug fixes and other changes + +- Fix to search for a ' Dict: - """Returns the dataset statistics for the data node if found else returns an - empty dictionary + def get_stats_for_data_node(self, data_node_name: str) -> Union[Dict, None]: + """Returns the dataset statistics for the data node if found Args: - The data node for which we need the statistics + The data node name for which we need the statistics """ - return self.dataset_stats.get(data_node.name, {}) + return self.dataset_stats.get(data_node_name, None) def add_pipeline(self, registered_pipeline_id: str, pipeline: KedroPipeline): """Iterate through all the nodes and datasets in a "registered" pipeline @@ -278,6 +276,7 @@ def add_dataset( layer=layer, tags=set(), dataset=obj, + stats=self.get_stats_for_data_node(_strip_transcoding(dataset_name)), is_free_input=is_free_input, ) graph_node = self.nodes.add_node(graph_node) diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index a2638851a1..a59d11f4c5 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -178,6 +178,7 @@ def create_data_node( layer: Optional[str], tags: Set[str], dataset: AbstractDataset, + stats: Optional[Dict], is_free_input: bool = False, ) -> Union["DataNode", "TranscodedDataNode"]: """Create a graph node of type DATA for a given Kedro DataSet instance. @@ -188,6 +189,8 @@ def create_data_node( tags: The set of tags assigned to assign to the graph representation of this dataset. N.B. currently it's derived from the node's tags. dataset: A dataset in a Kedro pipeline. + stats: The dictionary of dataset statistics, e.g. + {"rows":2, "columns":3, "file_size":100} is_free_input: Whether the dataset is a free input in the pipeline Returns: An instance of DataNode. @@ -201,6 +204,7 @@ def create_data_node( tags=tags, layer=layer, is_free_input=is_free_input, + stats=stats, ) return DataNode( @@ -210,6 +214,7 @@ def create_data_node( layer=layer, kedro_obj=dataset, is_free_input=is_free_input, + stats=stats, ) @classmethod @@ -434,6 +439,9 @@ class DataNode(GraphNode): # the type of this graph node, which is DATA type: str = GraphNodeType.DATA.value + # statistics for the data node + stats: Optional[Dict] = field(default=None) + def __post_init__(self): self.dataset_type = get_dataset_type(self.kedro_obj) @@ -517,6 +525,9 @@ class TranscodedDataNode(GraphNode): # the type of this graph node, which is DATA type: str = GraphNodeType.DATA.value + # statistics for the data node + stats: Optional[Dict] = field(default=None) + def has_metadata(self) -> bool: return True @@ -541,7 +552,6 @@ class DataNodeMetadata(GraphNodeMetadata): # the underlying data node to which this metadata belongs data_node: InitVar[DataNode] - dataset_stats: InitVar[Dict] # the optional plot data if the underlying dataset has a plot. # currently only applicable for PlotlyDataSet @@ -561,12 +571,12 @@ class DataNodeMetadata(GraphNodeMetadata): stats: Optional[Dict] = field(init=False, default=None) # TODO: improve this scheme. - def __post_init__(self, data_node: DataNode, dataset_stats: Dict): + def __post_init__(self, data_node: DataNode): self.type = data_node.dataset_type dataset = cast(AbstractDataset, data_node.kedro_obj) dataset_description = dataset._describe() self.filepath = _parse_filepath(dataset_description) - self.stats = dataset_stats + self.stats = data_node.stats # Run command is only available if a node is an output, i.e. not a free input if not data_node.is_free_input: @@ -625,11 +635,8 @@ class TranscodedDataNodeMetadata(GraphNodeMetadata): # the underlying data node to which this metadata belongs transcoded_data_node: InitVar[TranscodedDataNode] - dataset_stats: InitVar[Dict] - def __post_init__( - self, transcoded_data_node: TranscodedDataNode, dataset_stats: Dict - ): + def __post_init__(self, transcoded_data_node: TranscodedDataNode): original_version = transcoded_data_node.original_version self.original_type = get_dataset_type(original_version) @@ -640,7 +647,7 @@ def __post_init__( dataset_description = original_version._describe() self.filepath = _parse_filepath(dataset_description) - self.stats = dataset_stats + self.stats = transcoded_data_node.stats if not transcoded_data_node.is_free_input: self.run_command = ( diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 09c343074c..2321cb4097 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -44,9 +44,12 @@ def populate_data( data_access_manager.set_db_session(session_class) data_access_manager.add_catalog(catalog) - data_access_manager.add_pipelines(pipelines) + + # add dataset stats before adding pipelines data_access_manager.add_dataset_stats(stats_dict) + data_access_manager.add_pipelines(pipelines) + def run_server( host: str = DEFAULT_HOST, diff --git a/package/test_requirements.txt b/package/test_requirements.txt index 2ba35ada35..95de341b0b 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -12,7 +12,7 @@ fastapi[all]>=0.73.0, <0.96.0 isort~=5.11 matplotlib~=3.5 mypy~=1.0 -moto~=4.1.12 +moto~=4.1.14 psutil==5.9.5 # same as Kedro for now pylint~=2.17 pytest~=7.3 diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 16b7a1446b..83c3c955a1 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -108,6 +108,7 @@ def assert_example_data(response_data): "type": "data", "layer": "raw", "dataset_type": "pandas.csv_dataset.CSVDataSet", + "stats": None, }, { "id": "f0ebef01", @@ -118,6 +119,7 @@ def assert_example_data(response_data): "type": "parameters", "layer": None, "dataset_type": None, + "stats": None, }, { "id": "0ecea0de", @@ -128,6 +130,7 @@ def assert_example_data(response_data): "type": "data", "layer": "model_inputs", "dataset_type": "pandas.csv_dataset.CSVDataSet", + "stats": {"columns": 12, "rows": 29768}, }, { "id": "7b140b3f", @@ -150,6 +153,7 @@ def assert_example_data(response_data): "type": "parameters", "layer": None, "dataset_type": None, + "stats": None, }, { "id": "d5a8b994", @@ -160,6 +164,7 @@ def assert_example_data(response_data): "type": "data", "layer": None, "dataset_type": "io.memory_dataset.MemoryDataset", + "stats": None, }, { "id": "uk.data_processing", @@ -170,6 +175,7 @@ def assert_example_data(response_data): "modular_pipelines": None, "layer": None, "dataset_type": None, + "stats": None, }, { "id": "uk.data_science", @@ -180,6 +186,7 @@ def assert_example_data(response_data): "modular_pipelines": None, "layer": None, "dataset_type": None, + "stats": None, }, { "id": "uk", @@ -190,6 +197,7 @@ def assert_example_data(response_data): "modular_pipelines": None, "layer": None, "dataset_type": None, + "stats": None, }, ] assert_nodes_equal(response_data.pop("nodes"), expected_nodes) @@ -480,6 +488,7 @@ def assert_example_transcoded_data(response_data): "modular_pipelines": [], "layer": None, "dataset_type": "io.memory_dataset.MemoryDataset", + "stats": None, }, { "id": "f0ebef01", @@ -490,6 +499,7 @@ def assert_example_transcoded_data(response_data): "modular_pipelines": ["uk", "uk.data_processing"], "layer": None, "dataset_type": None, + "stats": None, }, { "id": "0ecea0de", @@ -500,6 +510,7 @@ def assert_example_transcoded_data(response_data): "modular_pipelines": [], "layer": None, "dataset_type": None, + "stats": None, }, { "id": "2302ea78", @@ -519,6 +530,7 @@ def assert_example_transcoded_data(response_data): "modular_pipelines": [], "layer": None, "dataset_type": None, + "stats": None, }, { "id": "1d06a0d7", @@ -529,6 +541,7 @@ def assert_example_transcoded_data(response_data): "modular_pipelines": [], "layer": None, "dataset_type": "io.memory_dataset.MemoryDataset", + "stats": None, }, ] @@ -572,7 +585,6 @@ def test_transcoded_data_node_metadata(self, example_transcoded_api): "pandas.parquet_dataset.ParquetDataSet", ], "run_command": "kedro run --to-outputs=model_inputs@pandas2", - "stats": {}, } @@ -614,7 +626,6 @@ def test_data_node_metadata_for_free_input(self, client): assert response.json() == { "filepath": "raw_data.csv", "type": "pandas.csv_dataset.CSVDataSet", - "stats": {}, } def test_parameters_node_metadata(self, client): @@ -664,6 +675,7 @@ def test_get_pipeline(self, client): "type": "data", "layer": "model_inputs", "dataset_type": "pandas.csv_dataset.CSVDataSet", + "stats": {"columns": 12, "rows": 29768}, }, { "id": "7b140b3f", @@ -686,6 +698,7 @@ def test_get_pipeline(self, client): "type": "parameters", "layer": None, "dataset_type": None, + "stats": None, }, { "id": "d5a8b994", @@ -696,6 +709,7 @@ def test_get_pipeline(self, client): "type": "data", "layer": None, "dataset_type": "io.memory_dataset.MemoryDataset", + "stats": None, }, { "id": "uk", @@ -706,6 +720,7 @@ def test_get_pipeline(self, client): "modular_pipelines": None, "layer": None, "dataset_type": None, + "stats": None, }, { "id": "uk.data_science", @@ -716,6 +731,7 @@ def test_get_pipeline(self, client): "modular_pipelines": None, "layer": None, "dataset_type": None, + "stats": None, }, ] assert_nodes_equal(response_data.pop("nodes"), expected_nodes) diff --git a/package/tests/test_data_access/test_repositories/test_modular_pipelines.py b/package/tests/test_data_access/test_repositories/test_modular_pipelines.py index 718ff19e80..97a4fe0f4f 100644 --- a/package/tests/test_data_access/test_repositories/test_modular_pipelines.py +++ b/package/tests/test_data_access/test_repositories/test_modular_pipelines.py @@ -47,6 +47,7 @@ def test_add_input(self): layer="model", tags=set(), dataset=kedro_dataset, + stats=None, ) modular_pipelines.add_input("data_science", data_node) assert data_node.id in data_science_pipeline.inputs @@ -62,6 +63,7 @@ def test_add_output(self): layer="model", tags=set(), dataset=kedro_dataset, + stats=None, ) modular_pipelines.add_output("data_science", data_node) assert data_node.id in data_science_pipeline.outputs diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index e66a411357..0caa179e1f 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -98,7 +98,7 @@ def test_create_task_node(self, namespace, expected_modular_pipelines): assert task_node.modular_pipelines == expected_modular_pipelines @pytest.mark.parametrize( - "dataset_name,expected_modular_pipelines", + "dataset_name, expected_modular_pipelines", [ ("dataset", []), ( @@ -118,6 +118,7 @@ def test_create_data_node(self, dataset_name, expected_modular_pipelines): layer="raw", tags=set(), dataset=kedro_dataset, + stats={"rows": 10, "columns": 5, "file_size": 1024}, ) assert isinstance(data_node, DataNode) assert data_node.kedro_obj is kedro_dataset @@ -127,6 +128,9 @@ def test_create_data_node(self, dataset_name, expected_modular_pipelines): assert data_node.tags == set() assert data_node.pipelines == set() assert data_node.modular_pipelines == expected_modular_pipelines + assert data_node.stats["rows"] == 10 + assert data_node.stats["columns"] == 5 + assert data_node.stats["file_size"] == 1024 assert not data_node.is_plot_node() assert not data_node.is_metric_node() assert not data_node.is_image_node() @@ -150,6 +154,7 @@ def test_create_transcoded_data_node(self, transcoded_dataset_name, original_nam layer="raw", tags=set(), dataset=kedro_dataset, + stats={"rows": 10, "columns": 2, "file_size": 1048}, ) assert isinstance(data_node, TranscodedDataNode) assert data_node.id == GraphNode._hash(original_name) @@ -157,6 +162,9 @@ def test_create_transcoded_data_node(self, transcoded_dataset_name, original_nam assert data_node.layer == "raw" assert data_node.tags == set() assert data_node.pipelines == set() + assert data_node.stats["rows"] == 10 + assert data_node.stats["columns"] == 2 + assert data_node.stats["file_size"] == 1048 def test_create_parameters_all_parameters(self): parameters_dataset = MemoryDataset( @@ -252,6 +260,7 @@ def test_add_node_to_pipeline(self): layer="raw", tags=set(), dataset=kedro_dataset, + stats={"rows": 10, "columns": 2, "file_size": 1048}, ) assert data_node.pipelines == set() data_node.add_pipeline(default_pipeline.id) @@ -265,7 +274,7 @@ class TestGraphNodeMetadata: ) def test_node_has_metadata(self, dataset, has_metadata): data_node = GraphNode.create_data_node( - "test_dataset", layer=None, tags=set(), dataset=dataset + "test_dataset", layer=None, tags=set(), dataset=dataset, stats=None ) assert data_node.has_metadata() == has_metadata @@ -354,10 +363,9 @@ def test_data_node_metadata(self): layer="raw", tags=set(), dataset=dataset, + stats={"rows": 10, "columns": 2}, ) - data_node_metadata = DataNodeMetadata( - data_node=data_node, dataset_stats={"rows": 10, "columns": 2} - ) + data_node_metadata = DataNodeMetadata(data_node=data_node) assert data_node_metadata.type == "pandas.csv_dataset.CSVDataSet" assert data_node_metadata.filepath == "/tmp/dataset.csv" assert data_node_metadata.run_command == "kedro run --to-outputs=dataset" @@ -368,10 +376,7 @@ def test_preview_args_not_exist(self): metadata = {"kedro-viz": {"something": 3}} dataset = CSVDataSet(filepath="test.csv", metadata=metadata) data_node = GraphNode.create_data_node( - dataset_name="dataset", - tags=set(), - layer=None, - dataset=dataset, + dataset_name="dataset", tags=set(), layer=None, dataset=dataset, stats=None ) assert not data_node.is_preview_node() @@ -379,10 +384,7 @@ def test_get_preview_args(self): metadata = {"kedro-viz": {"preview_args": {"nrows": 3}}} dataset = CSVDataSet(filepath="test.csv", metadata=metadata) data_node = GraphNode.create_data_node( - dataset_name="dataset", - tags=set(), - layer=None, - dataset=dataset, + dataset_name="dataset", tags=set(), layer=None, dataset=dataset, stats=None ) assert data_node.is_preview_node() assert data_node.get_preview_args() == {"nrows": 3} @@ -405,9 +407,7 @@ def test_preview_data_node_metadata(self): preview_data_node.is_tracking_node.return_value = False preview_data_node.is_preview_node.return_value = True preview_data_node.kedro_obj._preview.return_value = mock_preview_data - preview_node_metadata = DataNodeMetadata( - data_node=preview_data_node, dataset_stats={} - ) + preview_node_metadata = DataNodeMetadata(data_node=preview_data_node) assert preview_node_metadata.preview == mock_preview_data def test_preview_data_node_metadata_not_exist(self): @@ -418,9 +418,7 @@ def test_preview_data_node_metadata_not_exist(self): preview_data_node.is_tracking_node.return_value = False preview_data_node.is_preview_node.return_value = True preview_data_node.kedro_obj._preview.return_value = False - preview_node_metadata = DataNodeMetadata( - data_node=preview_data_node, dataset_stats={} - ) + preview_node_metadata = DataNodeMetadata(data_node=preview_data_node) assert preview_node_metadata.plot is None def test_transcoded_data_node_metadata(self): @@ -430,13 +428,13 @@ def test_transcoded_data_node_metadata(self): layer="raw", tags=set(), dataset=dataset, + stats={"rows": 10, "columns": 2}, ) transcoded_data_node.original_name = "dataset" transcoded_data_node.original_version = ParquetDataSet(filepath="foo.parquet") transcoded_data_node.transcoded_versions = [CSVDataSet(filepath="foo.csv")] transcoded_data_node_metadata = TranscodedDataNodeMetadata( - transcoded_data_node=transcoded_data_node, - dataset_stats={"rows": 10, "columns": 2}, + transcoded_data_node=transcoded_data_node ) assert ( transcoded_data_node_metadata.original_type @@ -456,8 +454,9 @@ def test_partitioned_data_node_metadata(self): layer="raw", tags=set(), dataset=dataset, + stats=None, ) - data_node_metadata = DataNodeMetadata(data_node=data_node, dataset_stats={}) + data_node_metadata = DataNodeMetadata(data_node=data_node) assert data_node_metadata.filepath == "partitioned/" # TODO: these test should ideally use a "real" catalog entry to create actual rather @@ -479,9 +478,7 @@ def test_plotly_data_node_metadata(self): plotly_data_node.is_tracking_node.return_value = False plotly_data_node.is_preview_node.return_value = False plotly_data_node.kedro_obj.load.return_value = mock_plot_data - plotly_node_metadata = DataNodeMetadata( - data_node=plotly_data_node, dataset_stats={} - ) + plotly_node_metadata = DataNodeMetadata(data_node=plotly_data_node) assert plotly_node_metadata.plot == mock_plot_data def test_plotly_data_node_dataset_not_exist(self): @@ -491,9 +488,7 @@ def test_plotly_data_node_dataset_not_exist(self): plotly_data_node.is_tracking_node.return_value = False plotly_data_node.kedro_obj.exists.return_value = False plotly_data_node.is_preview_node.return_value = False - plotly_node_metadata = DataNodeMetadata( - data_node=plotly_data_node, dataset_stats={} - ) + plotly_node_metadata = DataNodeMetadata(data_node=plotly_data_node) assert plotly_node_metadata.plot is None def test_plotly_json_dataset_node_metadata(self): @@ -512,9 +507,7 @@ def test_plotly_json_dataset_node_metadata(self): plotly_json_dataset_node.is_tracking_node.return_value = False plotly_json_dataset_node.is_preview_node.return_value = False plotly_json_dataset_node.kedro_obj.load.return_value = mock_plot_data - plotly_node_metadata = DataNodeMetadata( - data_node=plotly_json_dataset_node, dataset_stats={} - ) + plotly_node_metadata = DataNodeMetadata(data_node=plotly_json_dataset_node) assert plotly_node_metadata.plot == mock_plot_data # @patch("base64.b64encode") @@ -529,9 +522,7 @@ def test_image_data_node_metadata(self): image_dataset_node.is_tracking_node.return_value = False image_dataset_node.is_preview_node.return_value = False image_dataset_node.kedro_obj.load.return_value = mock_image_data - image_node_metadata = DataNodeMetadata( - data_node=image_dataset_node, dataset_stats={} - ) + image_node_metadata = DataNodeMetadata(data_node=image_dataset_node) assert image_node_metadata.image == mock_image_data def test_image_data_node_dataset_not_exist(self): @@ -540,9 +531,7 @@ def test_image_data_node_dataset_not_exist(self): image_dataset_node.is_plot_node.return_value = False image_dataset_node.kedro_obj.exists.return_value = False image_dataset_node.is_preview_node.return_value = False - image_node_metadata = DataNodeMetadata( - data_node=image_dataset_node, dataset_stats={} - ) + image_node_metadata = DataNodeMetadata(data_node=image_dataset_node) assert image_node_metadata.image is None def test_json_data_node_metadata(self): @@ -559,9 +548,7 @@ def test_json_data_node_metadata(self): json_data_node.is_metric_node.return_value = False json_data_node.is_preview_node.return_value = False json_data_node.kedro_obj.load.return_value = mock_json_data - json_node_metadata = DataNodeMetadata( - data_node=json_data_node, dataset_stats={} - ) + json_node_metadata = DataNodeMetadata(data_node=json_data_node) assert json_node_metadata.tracking_data == mock_json_data assert json_node_metadata.plot is None @@ -572,9 +559,7 @@ def test_metrics_data_node_metadata_dataset_not_exist(self): metrics_data_node.is_metric_node.return_value = True metrics_data_node.is_preview_node.return_value = False metrics_data_node.kedro_obj.exists.return_value = False - metrics_node_metadata = DataNodeMetadata( - data_node=metrics_data_node, dataset_stats={} - ) + metrics_node_metadata = DataNodeMetadata(data_node=metrics_data_node) assert metrics_node_metadata.plot is None def test_data_node_metadata_latest_tracking_data_not_exist(self): @@ -584,9 +569,7 @@ def test_data_node_metadata_latest_tracking_data_not_exist(self): plotly_data_node.is_tracking_node.return_value = False plotly_data_node.kedro_obj.exists.return_value = False plotly_data_node.kedro_obj.exists.return_value = False - plotly_node_metadata = DataNodeMetadata( - data_node=plotly_data_node, dataset_stats={} - ) + plotly_node_metadata = DataNodeMetadata(data_node=plotly_data_node) assert plotly_node_metadata.plot is None def test_parameters_metadata_all_parameters(self): diff --git a/package/tests/test_services/test_layers.py b/package/tests/test_services/test_layers.py index 7724187b2d..b4dd7cd326 100644 --- a/package/tests/test_services/test_layers.py +++ b/package/tests/test_services/test_layers.py @@ -163,6 +163,7 @@ def test_sort_layers(graph_schema, nodes, node_dependencies, expected): layer=node_dict.get("layer"), tags=None, dataset=None, + stats=None, ) for node_id, node_dict in nodes.items() } @@ -183,6 +184,7 @@ def test_sort_layers_should_return_empty_list_on_cyclic_layers(mocker): layer=node_dict.get("layer"), tags=None, dataset=None, + stats=None, ) for node_id, node_dict in data.items() } diff --git a/src/components/metadata/metadata-stats.js b/src/components/metadata/metadata-stats.js index ffff03be6c..e15a614010 100644 --- a/src/components/metadata/metadata-stats.js +++ b/src/components/metadata/metadata-stats.js @@ -1,6 +1,6 @@ import React, { useState, useRef, useLayoutEffect } from 'react'; import { formatFileSize, formatNumberWithCommas } from '../../utils'; -import { datasetStatLabels } from '../../config'; +import { datasetStatLabels, statsRowLen } from '../../config'; import './styles/metadata-stats.css'; const MetaDataStats = ({ stats }) => { @@ -14,14 +14,13 @@ const MetaDataStats = ({ stats }) => { return; } - const containerWidth = statsContainer.clientWidth; - const totalItemsWidth = Array.from(statsContainer.children).reduce( - (total, item) => total + item.offsetWidth, + const statsLen = Array.from(statsContainer.children).reduce( + (total, item) => total + item.outerText?.length, 0 ); - setHasOverflow(totalItemsWidth > containerWidth); - }, []); + setHasOverflow(statsLen > statsRowLen); + }, [stats]); return (