From b2f57188614c8fa4dafe04747fa22cf5d997ecfe Mon Sep 17 00:00:00 2001 From: Vishal Kumar Date: Thu, 12 Oct 2023 14:26:55 +0530 Subject: [PATCH 1/3] updated query for table_size --- vertica_sqlalchemy_dialect/base.py | 108 ++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 34 deletions(-) diff --git a/vertica_sqlalchemy_dialect/base.py b/vertica_sqlalchemy_dialect/base.py index c628d03..de92bc9 100644 --- a/vertica_sqlalchemy_dialect/base.py +++ b/vertica_sqlalchemy_dialect/base.py @@ -628,22 +628,46 @@ def fetch_table_properties(self,connection, schema): % {"schema": schema.lower()} ) ) + + + table_size_query = sql.text( + dedent( + """ + select p.projection_schema, anchor_table_name, sum(used_bytes) + from projections p join storage_containers sc + on p.projection_name = sc.projection_name + and p.projection_schema = sc.schema_name + where lower(p.projection_schema)='%(schema)s' + group by 1,2; + """ + % {"schema": schema.lower()} + ) + ) + properties = [] for row in connection.execute(sct): properties.append({"create_time": str(row[0]), "table_name": row[1]}) - + + # Dictionary to store table sizes table_size_dict = {} - for table_size in connection.execute(sts): - if table_size[1] not in table_size_dict: - table_size_dict[table_size[1]] = table_size[0] / 1024 + + # Second loop to fetch table sizes + for table_size in connection.execute(table_size_query): + table_name, size = table_size[1], table_size[2] + + if table_name not in table_size_dict: + table_size_dict[table_name] = str(size) + " KB" else: - table_size_dict[table_size[1]] += table_size[0] / 1024 + table_size_dict[table_name] += str(size) + " KB" + + for a in properties: if a["table_name"] in table_size_dict: - a["table_size"] = str(int(table_size_dict[a["table_name"]])) + " KB" + a["table_size"] = table_size_dict[a["table_name"]] else: a["table_size"] = "0 KB" - a["table_name"] = a["table_name"].lower() + + return properties def get_table_comment(self, connection, table_name, schema=None, **kw): @@ -661,20 +685,20 @@ def get_table_comment(self, connection, table_name, schema=None, **kw): # below code tracks it function is called from a view or a table # if called from table it return table_size , if called from view it only returns create_time . - stack = traceback.extract_stack(limit=-10) - function_names = [frame.name for frame in stack] - called_from = function_names[-1] + # stack = traceback.extract_stack(limit=-10) + # function_names = [frame.name for frame in stack] + # called_from = function_names[-1] - if called_from == "loop_tables": + # if called_from == "loop_tables": - table_properties = { - "create_time": filtered_properties[0]['create_time'], - "table_size": filtered_properties[0]['table_size'], - } - else: - table_properties = { - "create_time": filtered_properties[0]['create_time'], - } + table_properties = { + "create_time": filtered_properties[0]['create_time'], + "table_size": filtered_properties[0]['table_size'], + } + # else: + # table_properties = { + # "create_time": filtered_properties[0]['create_time'], + # } return { @@ -1488,14 +1512,27 @@ def fetch_projection_comments(self,connection,schema): ) ) - partition_num = sql.text( + # partition_num = sql.text( + # dedent( + # """ + # SELECT COUNT(ros_id) as Partition_Size , LOWER(projection_name) + # FROM v_monitor.partitions + # WHERE table_schema = '%(schema)s' + # GROUP BY projection_name + # """ + # % {"schema": schema} + # ) + # ) + + num_of_partition = sql.text( dedent( """ - SELECT COUNT(ros_id) as Partition_Size , LOWER(projection_name) - FROM v_monitor.partitions - WHERE table_schema = '%(schema)s' - GROUP BY projection_name - """ + SELECT LOWER(projection_name) , count(partition_key) as Partition_Size + FROM v_monitor.partitions WHERE lower(table_schema) = '%(schema)s' + group by 1; + + + """ % {"schema": schema} ) ) @@ -1580,10 +1617,13 @@ def fetch_projection_comments(self,connection,schema): else: a["projection_size"] = "0 KB" - for partition_number in connection.execute(partition_num): + for partition_number in connection.execute(num_of_partition): + for a in projection_comment: - if a["projection_name"].lower() == partition_number[1]: - a["Partition_Size"] = str(partition_number["Partition_Size"]) + if a["projection_name"].lower() == partition_number[0].lower(): + a["Partition_Size"] = str(partition_number[1]) + + for projection_cached in connection.execute(projection_cache): for a in projection_comment: @@ -1614,9 +1654,9 @@ def get_projection_comment(self, connection, projection, schema=None, **kw): projection_properties['Projection_Type']= "Not Available" if 'is_segmented' in projection_comments[0]: - projection_properties['is_segmented'] = str(projection_comments[0]['is_segmented']) + projection_properties['Is_Segmented'] = str(projection_comments[0]['is_segmented']) else: - projection_properties['is_segmented'] = "Not Available" + projection_properties['Is_Segmented'] = "Not Available" if 'Segmentation_key' in projection_comments[0]: projection_properties['Segmentation_key'] = str(projection_comments[0]['Segmentation_key']) @@ -1624,9 +1664,9 @@ def get_projection_comment(self, connection, projection, schema=None, **kw): projection_properties['Segmentation_key'] = "Not Available" if 'projection_size' in projection_comments[0]: - projection_properties['projection_size'] = str(projection_comments[0]['projection_size']) + projection_properties['Projection_size'] = str(projection_comments[0]['projection_size']) else: - projection_properties['projection_size'] = "0 KB" + projection_properties['Projection_size'] = "0 KB" if 'Partition_Key' in projection_comments[0]: projection_properties['Partition_Key'] = str(projection_comments[0]['Partition_Key']) @@ -1634,9 +1674,9 @@ def get_projection_comment(self, connection, projection, schema=None, **kw): projection_properties['Partition_Key'] = "Not Available" if 'Partition_Size' in projection_comments[0]: - projection_properties['Partition_Size'] = str(projection_comments[0]['Partition_Size']) + projection_properties['Number_Of_Partitions'] = str(projection_comments[0]['Partition_Size']) else: - projection_properties['Partition_Size'] = "0" + projection_properties['Number_Of_Partitions'] = "0" if 'Projection_Cached' in projection_comments[0]: projection_properties['Projection_Cached'] = str(projection_comments[0]['Projection_Cached']) From 7944b20421bdefccb092a25e1d1091fbd0a17bb7 Mon Sep 17 00:00:00 2001 From: Vishal Kumar Date: Thu, 12 Oct 2023 14:32:54 +0530 Subject: [PATCH 2/3] modief projection properties in sample --- test/sample_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sample_objects.py b/test/sample_objects.py index bc1b3cd..b82c861 100644 --- a/test/sample_objects.py +++ b/test/sample_objects.py @@ -169,5 +169,5 @@ sample_oauth_name = "v_oauth" -sample_projection_properties = {'text': 'Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs', 'properties': {'ROS_Count': '1', 'Projection_Type': 'is_super_projection', 'is_segmented': 'True', 'Segmentation_key': 'hash(product_dimension.product_key, product_dimension.product_version)', 'projection_size': '19 KB', 'Partition_Key': 'Not Available', 'Partition_Size': '0', 'Projection_Cached': 'False'}} +sample_projection_properties = {'text': 'Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs', 'properties': {'ROS_Count': '1', 'Projection_Type': 'is_super_projection', 'Is_Segmented': 'True', 'Segmentation_key': 'hash(product_dimension.product_key, product_dimension.product_version)', 'Projection_size': '19 KB', 'Partition_Key': 'Not Available', 'Number_Of_Partitions': '0', 'Projection_Cached': 'False'}} From adbc709f81e4f63cf186aa3a59893c18577b15a7 Mon Sep 17 00:00:00 2001 From: Vishal Kumar Date: Fri, 13 Oct 2023 13:47:42 +0530 Subject: [PATCH 3/3] converted table size from byte to kilo byte --- vertica_sqlalchemy_dialect/base.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/vertica_sqlalchemy_dialect/base.py b/vertica_sqlalchemy_dialect/base.py index de92bc9..b38d544 100644 --- a/vertica_sqlalchemy_dialect/base.py +++ b/vertica_sqlalchemy_dialect/base.py @@ -618,16 +618,6 @@ def fetch_table_properties(self,connection, schema): % {"schema": schema.lower()} ) ) - sts = sql.text( - dedent( - """ - SELECT used_bytes ,anchor_table_name - FROM v_monitor.column_storage - where lower(anchor_table_schema) = '%(schema)s' - """ - % {"schema": schema.lower()} - ) - ) table_size_query = sql.text( @@ -654,11 +644,11 @@ def fetch_table_properties(self,connection, schema): # Second loop to fetch table sizes for table_size in connection.execute(table_size_query): table_name, size = table_size[1], table_size[2] - + TableSize = int(size / 1024) if table_name not in table_size_dict: - table_size_dict[table_name] = str(size) + " KB" + table_size_dict[table_name] = str(TableSize) + " KB" else: - table_size_dict[table_name] += str(size) + " KB" + table_size_dict[table_name] += str(TableSize) + " KB" for a in properties: