diff --git a/include/storage.h b/include/storage.h index bef6a52..e0b6922 100644 --- a/include/storage.h +++ b/include/storage.h @@ -51,9 +51,11 @@ EXTERNC int statRelationSpaceUsage(Relation aorel, int segno, int64 modcount, size_t *local_commited_bytes, size_t *external_bytes); -EXTERNC int statRelationSpaceUsagePerExternalChunk( - Relation aorel, int segno, int64 modcount, int64 logicalEof, +EXTERNC int statRelationChunksSpaceUsage( + Relation aorel, size_t *local_bytes, size_t *local_commited_bytes, yezzeyChunkMeta **list, size_t *cnt_chunks); +EXTERNC int yezzey_get_block_from_file_path(const char* path); + #endif /* YEZZEY_STORAGE_H */ \ No newline at end of file diff --git a/src/storage.cpp b/src/storage.cpp index 5c6d60e..5003358 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -468,12 +468,11 @@ int statRelationSpaceUsage(Relation aorel, int segno, int64 modcount, return 0; } -int statRelationSpaceUsagePerExternalChunk(Relation aorel, int segno, - int64 modcount, int64 logicalEof, - size_t *local_bytes, - size_t *local_commited_bytes, - yezzeyChunkMeta **list, - size_t *cnt_chunks) { +int statRelationChunksSpaceUsage(Relation aorel, + size_t *local_bytes, + size_t *local_commited_bytes, + yezzeyChunkMeta **list, + size_t *cnt_chunks) { auto rnode = aorel->rd_node; /* rnode.spcNode == YEZZEYTABLESPACEOID here. we need @@ -482,7 +481,7 @@ int statRelationSpaceUsagePerExternalChunk(Relation aorel, int segno, auto spcNode = resolveTablespaceOidByName( YezzeyGetRelationOriginTablespace(NULL, NULL, RelationGetRelid(aorel))); - auto coords = relnodeCoord(spcNode, rnode.dbNode, rnode.relNode, segno); + auto coords = relnodeCoord(spcNode, rnode.dbNode, rnode.relNode, 0); auto tp = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(aorel->rd_rel->relnamespace)); @@ -540,3 +539,15 @@ int statRelationSpaceUsagePerExternalChunk(Relation aorel, int segno, *local_commited_bytes = 0; return 0; } + +int yezzey_get_block_from_file_path(const char* path) { + std::string pathstr = path; + int i = 0; + int previ = 0; + for (int n = 0; n < 7; ++n) { + previ = i; + i = pathstr.find('_', i+1); + } + auto blkno = pathstr.substr(previ+1, i-previ); + return atoi(blkno.c_str()); +} diff --git a/src/yproxy.cpp b/src/yproxy.cpp index 46c2618..0000234 100644 --- a/src/yproxy.cpp +++ b/src/yproxy.cpp @@ -3,7 +3,9 @@ #include "meta.h" #include "url.h" #include "util.h" +#include "virtual_index.h" #include +#include #include #include @@ -673,8 +675,7 @@ std::vector YProxyLister::list_relation_chunks() { return res; } - auto msg = ConstructListRequest(yezzey_block_file_path( - adv_->nspname, adv_->relname, adv_->coords_, segindx_)); + auto msg = ConstructListRequest(yezzey_block_db_file_path(adv_->nspname, adv_->relname, adv_->coords_, segindx_)); size_t rc = ::write(client_fd_, msg.data(), msg.size()); if (rc <= 0) { // throw? diff --git a/yezzey.c b/yezzey.c index 54b0743..533d070 100644 --- a/yezzey.c +++ b/yezzey.c @@ -775,13 +775,9 @@ Datum yezzey_relation_describe_external_storage_structure_internal( Oid reloid; Relation aorel; int i; - int segno; - int pseudosegno; int total_segfiles; int nvp; int inat; - int64 modcount; - int64 logicalEof; FileSegInfo **segfile_array; AOCSFileSegInfo **segfile_array_cs; Snapshot appendOnlyMetaDataSnapshot; @@ -825,13 +821,10 @@ Datum yezzey_relation_describe_external_storage_structure_internal( */ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - size_t total_row = 0; size_t local_bytes = 0; size_t external_bytes = 0; size_t local_commited_bytes = 0; - if (RelationIsAoRows(aorel)) { - /* ao rows relation */ #if IsModernYezzey segfile_array = GetAllFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles, &segrelid); @@ -840,118 +833,45 @@ Datum yezzey_relation_describe_external_storage_structure_internal( GetAllFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles); #endif - for (i = 0; i < total_segfiles; ++i) { - - segno = segfile_array[i]->segno; - modcount = segfile_array[i]->modcount; - logicalEof = segfile_array[i]->eof; - - elog(yezzey_log_level, - "stat segment no %d, modcount %ld with to logial eof %ld", segno, - modcount, logicalEof); - size_t curr_local_bytes = 0; - size_t curr_external_bytes = 0; - size_t curr_local_commited_bytes = 0; - yezzeyChunkMeta *list; - size_t cnt_chunks; - - if (statRelationSpaceUsagePerExternalChunk( - aorel, segno, modcount, logicalEof, &curr_local_bytes, - &curr_local_commited_bytes, &list, &cnt_chunks) < 0) { - elog(ERROR, "failed to stat segment %d usage", segno); - } - - local_bytes = curr_local_bytes; - external_bytes = curr_external_bytes; - local_commited_bytes = curr_local_commited_bytes; - - chunkInfo = realloc(chunkInfo, sizeof(yezzeyChunkMetaInfo) * - (total_row + cnt_chunks)); - - for (size_t chunk_index = 0; chunk_index < cnt_chunks; ++chunk_index) { - chunkInfo[total_row + chunk_index].reloid = reloid; - chunkInfo[total_row + chunk_index].segindex = GpIdentity.segindex; - chunkInfo[total_row + chunk_index].segfileindex = i; - chunkInfo[total_row + chunk_index].external_storage_filepath = - list[chunk_index].chunkName; - chunkInfo[total_row + chunk_index].local_bytes = local_bytes; - chunkInfo[total_row + chunk_index].local_commited_bytes = - local_commited_bytes; - chunkInfo[total_row + chunk_index].external_bytes = - list[chunk_index].chunkSize; - } - total_row += cnt_chunks; - } - - /* - * Build a tuple descriptor for our result type - * The number and type of attributes have to match the definition of the - * view yezzey_offload_relation_status_internal - */ - - } else if (RelationIsAoCols(aorel)) { - -#if IsGreenplum6 - segfile_array_cs = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, - &total_segfiles); -#else - segfile_array_cs = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, - &total_segfiles, &segrelid); -#endif + size_t curr_local_bytes = 0; + size_t curr_external_bytes = 0; + size_t curr_local_commited_bytes = 0; + yezzeyChunkMeta *list; + size_t cnt_chunks; - for (inat = 0; inat < nvp; ++inat) { - for (i = 0; i < total_segfiles; i++) { - segno = segfile_array_cs[i]->segno; - /* in AOCS case actual *segno* differs from segfile_array_cs[i]->segno - * whis is logical number of segment. On physical level, each logical - * segno (segfile_array_cs[i]->segno) is represented by - * AOTupleId_MultiplierSegmentFileNum in storage (1 file per - * attribute) - */ - pseudosegno = (inat * AOTupleId_MultiplierSegmentFileNum) + segno; - modcount = segfile_array_cs[i]->modcount; - logicalEof = segfile_array_cs[i]->vpinfo.entry[inat].eof; - - size_t curr_local_bytes = 0; - size_t curr_external_bytes = 0; - size_t curr_local_commited_bytes = 0; - yezzeyChunkMeta *list; - size_t cnt_chunks; - - if (statRelationSpaceUsagePerExternalChunk( - aorel, pseudosegno, modcount, logicalEof, &curr_local_bytes, - &curr_local_commited_bytes, &list, &cnt_chunks) < 0) { - elog(ERROR, "failed to stat segment %d usage", segno); - } + if (statRelationChunksSpaceUsage( + aorel, &curr_local_bytes, + &curr_local_commited_bytes, &list, &cnt_chunks) < 0) { + elog(ERROR, "failed to stat relation chunks space usage"); + } - local_bytes = curr_local_bytes; - external_bytes = curr_external_bytes; - local_commited_bytes = curr_local_commited_bytes; - - chunkInfo = realloc(chunkInfo, sizeof(yezzeyChunkMetaInfo) * - (total_row + cnt_chunks)); - - for (size_t chunk_index = 0; chunk_index < cnt_chunks; - ++chunk_index) { - chunkInfo[total_row + chunk_index].reloid = reloid; - chunkInfo[total_row + chunk_index].segindex = GpIdentity.segindex; - chunkInfo[total_row + chunk_index].segfileindex = i; - chunkInfo[total_row + chunk_index].external_storage_filepath = - list[chunk_index].chunkName; - chunkInfo[total_row + chunk_index].local_bytes = local_bytes; - chunkInfo[total_row + chunk_index].local_commited_bytes = - local_commited_bytes; - chunkInfo[total_row + chunk_index].external_bytes = - list[chunk_index].chunkSize; - } - total_row += cnt_chunks; - } - } - } else { + local_bytes = curr_local_bytes; + external_bytes = curr_external_bytes; + local_commited_bytes = curr_local_commited_bytes; - elog(ERROR, "yezzey: wrong relation storage type: not AO/AOCS relation"); + chunkInfo = realloc(chunkInfo, sizeof(yezzeyChunkMetaInfo) * + (cnt_chunks)); + + for (size_t chunk_index = 0; chunk_index < cnt_chunks; ++chunk_index) { + chunkInfo[chunk_index].reloid = reloid; + chunkInfo[chunk_index].segindex = GpIdentity.segindex; + chunkInfo[chunk_index].segfileindex = yezzey_get_block_from_file_path( + list[chunk_index].chunkName); + chunkInfo[chunk_index].external_storage_filepath = + list[chunk_index].chunkName; + chunkInfo[chunk_index].local_bytes = local_bytes; + chunkInfo[chunk_index].local_commited_bytes = + local_commited_bytes; + chunkInfo[chunk_index].external_bytes = + list[chunk_index].chunkSize; } + /* + * Build a tuple descriptor for our result type + * The number and type of attributes have to match the definition of the + * view yezzey_offload_relation_status_internal + */ + #if IsGreenplum6 funcctx->tuple_desc = CreateTemplateTupleDesc( NUM_USED_OFFLOAD_PER_SEGMENT_STATUS_STRUCT, false); @@ -985,8 +905,8 @@ Datum yezzey_relation_describe_external_storage_structure_internal( attinmeta = TupleDescGetAttInMetadata(funcctx->tuple_desc); funcctx->attinmeta = attinmeta; - if (total_row > 0) { - funcctx->max_calls = total_row; + if (cnt_chunks > 0) { + funcctx->max_calls = cnt_chunks; funcctx->user_fctx = chunkInfo; /* funcctx->user_fctx */ } else {