Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated query for table_size and partition size #13

Merged
merged 3 commits into from
Oct 13, 2023
Merged

updated query for table_size and partition size #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2f5718
updated query for table_size
vishalkSimplify Oct 12, 2023
7944b20
modief projection properties in sample
vishalkSimplify Oct 12, 2023
adbc709
converted table size from byte to kilo byte
vishalkSimplify Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/sample_objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,5 +169,5 @@

sample_oauth_name = "v_oauth"

sample_projection_properties = {'text': 'Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs', 'properties': {'ROS_Count': '1', 'Projection_Type': 'is_super_projection', 'is_segmented': 'True', 'Segmentation_key': 'hash(product_dimension.product_key, product_dimension.product_version)', 'projection_size': '19 KB', 'Partition_Key': 'Not Available', 'Partition_Size': '0', 'Projection_Cached': 'False'}}
sample_projection_properties = {'text': 'Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs', 'properties': {'ROS_Count': '1', 'Projection_Type': 'is_super_projection', 'Is_Segmented': 'True', 'Segmentation_key': 'hash(product_dimension.product_key, product_dimension.product_version)', 'Projection_size': '19 KB', 'Partition_Key': 'Not Available', 'Number_Of_Partitions': '0', 'Projection_Cached': 'False'}}

106 changes: 68 additions & 38 deletions vertica_sqlalchemy_dialect/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,32 +618,46 @@ def fetch_table_properties(self,connection, schema):
% {"schema": schema.lower()}
)
)
sts = sql.text(


table_size_query = sql.text(
dedent(
"""
SELECT used_bytes ,anchor_table_name
FROM v_monitor.column_storage
where lower(anchor_table_schema) = '%(schema)s'
select p.projection_schema, anchor_table_name, sum(used_bytes)
from projections p join storage_containers sc
on p.projection_name = sc.projection_name
and p.projection_schema = sc.schema_name
where lower(p.projection_schema)='%(schema)s'
group by 1,2;
"""
% {"schema": schema.lower()}
)
)

properties = []
for row in connection.execute(sct):
properties.append({"create_time": str(row[0]), "table_name": row[1]})


# Dictionary to store table sizes
table_size_dict = {}
for table_size in connection.execute(sts):
if table_size[1] not in table_size_dict:
table_size_dict[table_size[1]] = table_size[0] / 1024

# Second loop to fetch table sizes
for table_size in connection.execute(table_size_query):
table_name, size = table_size[1], table_size[2]
TableSize = int(size / 1024)
if table_name not in table_size_dict:
table_size_dict[table_name] = str(TableSize) + " KB"
else:
table_size_dict[table_size[1]] += table_size[0] / 1024
table_size_dict[table_name] += str(TableSize) + " KB"


for a in properties:
if a["table_name"] in table_size_dict:
a["table_size"] = str(int(table_size_dict[a["table_name"]])) + " KB"
a["table_size"] = table_size_dict[a["table_name"]]
else:
a["table_size"] = "0 KB"
a["table_name"] = a["table_name"].lower()


return properties

def get_table_comment(self, connection, table_name, schema=None, **kw):
Expand All @@ -661,20 +675,20 @@ def get_table_comment(self, connection, table_name, schema=None, **kw):
# below code tracks it function is called from a view or a table
# if called from table it return table_size , if called from view it only returns create_time .

stack = traceback.extract_stack(limit=-10)
function_names = [frame.name for frame in stack]
called_from = function_names[-1]
# stack = traceback.extract_stack(limit=-10)
# function_names = [frame.name for frame in stack]
# called_from = function_names[-1]

if called_from == "loop_tables":
# if called_from == "loop_tables":

table_properties = {
"create_time": filtered_properties[0]['create_time'],
"table_size": filtered_properties[0]['table_size'],
}
else:
table_properties = {
"create_time": filtered_properties[0]['create_time'],
}
table_properties = {
"create_time": filtered_properties[0]['create_time'],
"table_size": filtered_properties[0]['table_size'],
}
# else:
# table_properties = {
# "create_time": filtered_properties[0]['create_time'],
# }


return {
Expand Down Expand Up @@ -1488,14 +1502,27 @@ def fetch_projection_comments(self,connection,schema):
)
)

partition_num = sql.text(
# partition_num = sql.text(
# dedent(
# """
# SELECT COUNT(ros_id) as Partition_Size , LOWER(projection_name)
# FROM v_monitor.partitions
# WHERE table_schema = '%(schema)s'
# GROUP BY projection_name
# """
# % {"schema": schema}
# )
# )

num_of_partition = sql.text(
dedent(
"""
SELECT COUNT(ros_id) as Partition_Size , LOWER(projection_name)
FROM v_monitor.partitions
WHERE table_schema = '%(schema)s'
GROUP BY projection_name
"""
SELECT LOWER(projection_name) , count(partition_key) as Partition_Size
FROM v_monitor.partitions WHERE lower(table_schema) = '%(schema)s'
group by 1;


"""
% {"schema": schema}
)
)
Expand Down Expand Up @@ -1580,10 +1607,13 @@ def fetch_projection_comments(self,connection,schema):
else:
a["projection_size"] = "0 KB"

for partition_number in connection.execute(partition_num):
for partition_number in connection.execute(num_of_partition):

for a in projection_comment:
if a["projection_name"].lower() == partition_number[1]:
a["Partition_Size"] = str(partition_number["Partition_Size"])
if a["projection_name"].lower() == partition_number[0].lower():
a["Partition_Size"] = str(partition_number[1])



for projection_cached in connection.execute(projection_cache):
for a in projection_comment:
Expand Down Expand Up @@ -1614,29 +1644,29 @@ def get_projection_comment(self, connection, projection, schema=None, **kw):
projection_properties['Projection_Type']= "Not Available"

if 'is_segmented' in projection_comments[0]:
projection_properties['is_segmented'] = str(projection_comments[0]['is_segmented'])
projection_properties['Is_Segmented'] = str(projection_comments[0]['is_segmented'])
else:
projection_properties['is_segmented'] = "Not Available"
projection_properties['Is_Segmented'] = "Not Available"

if 'Segmentation_key' in projection_comments[0]:
projection_properties['Segmentation_key'] = str(projection_comments[0]['Segmentation_key'])
else:
projection_properties['Segmentation_key'] = "Not Available"

if 'projection_size' in projection_comments[0]:
projection_properties['projection_size'] = str(projection_comments[0]['projection_size'])
projection_properties['Projection_size'] = str(projection_comments[0]['projection_size'])
else:
projection_properties['projection_size'] = "0 KB"
projection_properties['Projection_size'] = "0 KB"

if 'Partition_Key' in projection_comments[0]:
projection_properties['Partition_Key'] = str(projection_comments[0]['Partition_Key'])
else:
projection_properties['Partition_Key'] = "Not Available"

if 'Partition_Size' in projection_comments[0]:
projection_properties['Partition_Size'] = str(projection_comments[0]['Partition_Size'])
projection_properties['Number_Of_Partitions'] = str(projection_comments[0]['Partition_Size'])
else:
projection_properties['Partition_Size'] = "0"
projection_properties['Number_Of_Partitions'] = "0"

if 'Projection_Cached' in projection_comments[0]:
projection_properties['Projection_Cached'] = str(projection_comments[0]['Projection_Cached'])
Expand Down