From df2f250d278ca457eb8983ca80ebfa8c082a6891 Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Sat, 16 Sep 2023 10:31:55 +0800 Subject: [PATCH 1/6] feat: full inspect --- docs/zh/maintain/diagnose.md | 100 ++++- .../openmldb_tool/diagnostic_tool/diagnose.py | 45 +- .../openmldb_tool/diagnostic_tool/inspect.py | 405 ++++++++++++++++++ python/openmldb_tool/diagnostic_tool/pb.py | 92 ++++ python/openmldb_tool/diagnostic_tool/rpc.py | 109 ++--- .../diagnostic_tool/table_checker.py | 116 +++-- python/openmldb_tool/setup.py | 6 +- python/openmldb_tool/tests/inspect_test.py | 340 +++++++++++++++ 8 files changed, 1054 insertions(+), 159 deletions(-) create mode 100644 python/openmldb_tool/diagnostic_tool/inspect.py create mode 100644 python/openmldb_tool/diagnostic_tool/pb.py create mode 100644 python/openmldb_tool/tests/inspect_test.py diff --git a/docs/zh/maintain/diagnose.md b/docs/zh/maintain/diagnose.md index eef7db5b5a1..a5d126f09df 100644 --- a/docs/zh/maintain/diagnose.md +++ b/docs/zh/maintain/diagnose.md @@ -8,15 +8,68 @@ 安装方式与使用： ```bash -pip install openmldb-tool # openmldb-tool[rpc] +pip install openmldb-tool # openmldb-tool[pb] openmldb_tool # 注意下划线 ``` 有以下几个子命令可选择执行： ```bash -usage: openmldb_tool [-h] [--helpfull] {status,inspect,test,static-check} ... +usage: openmldb_tool [-h] [--helpfull] {status,inspect,rpc,test,static-check} ... ``` 只有`static-check`静态检查命令需要指定`--dist_conf`参数，该参数指定OpenMLDB节点分布的配置文件。其他命令只需要`--cluster`参数，格式为`/`，默认为镜像中的OpenMLDB集群地址`127.0.0.1:2181/openmldb`。如果是自行设置的OpenMLDB集群，请配置此参数。 +### 一键inspect + +`openmldb_tool inspect [--cluster=0.0.0.0:2181/openmldb]`可以一键查询，得到完整的集群状态报告。如果需要局部视角或额外的诊断功能，才需要其他子命令。 + +报告分为几个板块，其中如果所有表都是健康的，不会展示Ops和Partitions板块。用户首先看报告末尾的总结 summary & hint，如果存在server offline（红色），需先重启server，保证server尤其是TabletServer都在线。server重启后，集群可能会尝试自动修复，自动修复也可能会失败，所以，用户有必要等待一定时间后再次inspect。此时如果仍然有不健康的表，可以检查它们的状态，Fatal表需要尽快修复，它们可能会读写失败，Warn表，用户可以考虑推迟修复。修复方式见报告末尾提供的文档。 + +`inspect`可配置参数除了`--cluster/-c`，还可配置不显示彩色`--nocolor/-noc`方便复制，以及`--table_width/-tw n`配置表格宽度，`--offset_diff_thresh/-od n`配置offset diff的报警阈值。 + +``` +diagnosing cluster xxx + + +Server Detail +{server map} +{server online/offline report} + + +Table Healthy Detail +summary: {total table cnt} +{unhealthy tables db.name} + + +Ops Detail +> failed ops do not mean cluster is unhealthy, just for reference +last one op(check time): {} +last 10 unfinished ops: +{op list} + + +Table Partitions Detail +tablet server order: {tablet ip -> idx} +{partition tables of unhealthy tables} +Example: +{a detailed description of partition table} + + +================== +Summary & Hint +================== +Server: + +{online | offline servers ['[tablet]xxx'], restart them first} + +Table: +{all healthy | unhealthy tables desc} +[]Fatal/Warn table, {read/write may fail or still work}, {repair immediatly or not} +{partition detail: if leader healthy, if has unhealthy replicas, if offset too large, related ops} + + Make sure all servers online, and no ops for the table is running. + Repair table manually, run recoverdata, check https://openmldb.ai/docs/zh/main/maintain/openmldb_ops.html. + Check 'Table Partitions Detail' above for detail. +``` + ## 子命令详情 ### status 状态 @@ -29,7 +82,7 @@ usage: openmldb_tool status [-h] [--helpfull] [--diff] optional arguments: -h, --help show this help message and exit --helpfull show full help message and exit - --diff check if all endpoints in conf are in cluster. If set, need to set `--conf_file` + --diff check if all endpoints in conf are in cluster. If set, need to set `--conf_file/-f` ``` - 简单查询集群状态： @@ -57,7 +110,8 @@ openmldb_tool status --diff -f=/work/openmldb/conf/hosts ### inspect 检查 -`inspect`用于检查集群的在线和离线两个部分是否正常工作，可以选择单独检查`online`或`offline`，不指定则都检查。可以定期执行检查，以便及时发现异常。 +如果是为了检查集群状态，更推荐一键`inspect`获取集群完整检查报告，`inspect`子命令是更具有针对性的检查。 + ``` openmldb_tool inspect -h usage: openmldb_tool inspect [-h] [--helpfull] {online,offline,job} ... @@ -68,19 +122,26 @@ positional arguments: offline only inspect offline jobs. job show jobs by state, show joblog or parse joblog by id. ``` -在线检查会检查集群中的表状态（包括系统表），并输出有异常的表，包括表的状态，分区信息，副本信息等，等价于`SHOW TABLE STATUS`并筛选出有异常的表。如果发现集群表现不正常，请先检查下是否有异常表。例如，`SHOW JOBS`无法正常输出历史任务时，可以`inspect online`检查一下是否是job系统表出现问题。 + +#### online在线检查 + +`inspect online`检查在线表的健康状态，并输出有异常的表，包括表的状态，分区信息，副本信息等，等价于`SHOW TABLE STATUS`并筛选出有异常的表。 ##### 检查在线数据分布 -在线检查中，可以使用`inspect online --dist`检查在线数据分布，默认检查所有数据库，可以使用`--db`指定要检查的数据库。若要查询多个数据库，请使用 ',' 分隔数据库名称。会输出数据库在各个节点上的数据分布情况。 +可以使用`inspect online --dist`检查在线数据分布，默认检查所有数据库，可以使用`--db`指定要检查的数据库。若要查询多个数据库，请使用 ',' 分隔数据库名称。会输出数据库在各个节点上的数据分布情况。 -#### 离线检查 +#### offline离线检查 -离线检查会输出最终状态为失败的任务（不检查“运行中”的任务），等价于`SHOW JOBS`并筛选出失败任务。 +`inspect offline`离线检查会输出最终状态为失败的任务（不检查“运行中”的任务），等价于`SHOW JOBS`并筛选出失败任务。 #### JOB 检查 -JOB 检查会检查集群中的离线任务，可以使用`inspect job`或`inspect job --state all`查询所有任务，等价于`SHOW JOBS`并按job_id排序。使用`inspect job --state `可以筛选出特定状态的日志，可以使用 ',' 分隔，同时查询不同状态的日志。例如：`inspect offline` 相当于`inspect job --state failed,killed,lost`即筛选出所有失败的任务。 +JOB 检查是更灵活的离线任务检查命令，可以按条件筛选job，或针对单个job日志进行分析。 + +##### 按state筛选 + +可以使用`inspect job`或`inspect job --state all`查询所有任务，等价于`SHOW JOBS`并按job_id排序。使用`inspect job --state `可以筛选出特定状态的日志，可以使用 ',' 分隔，同时查询不同状态的日志。例如：`inspect offline` 相当于`inspect job --state failed,killed,lost`即筛选出所有失败的任务。 以下是一些常见的state: @@ -93,8 +154,13 @@ JOB 检查会检查集群中的离线任务，可以使用`inspect job`或`inspe 更多state信息详见[Spark State]( https://spark.apache.org/docs/3.2.1/api/java/org/apache/spark/launcher/SparkAppHandle.State.html)，[Yarn State](https://hadoop.apache.org/docs/current/api/org/apache/hadoop/yarn/api/records/YarnApplicationState.html) +##### 解析单个JOB日志 + +使用`inspect job --id `查询指定任务的log日志，其结果会使用配置文件筛选出主要错误信息。 -使用`inspect job --id `查询指定任务的log日志，其结果会使用配置文件筛选出主要错误信息。如需更新配置文件，可以添加`--conf-update`，并且可以使用`--conf-url`配置镜像源，例如使用`--conf-url https://openmldb.ai/download/diag/common_err.yml`配置国内镜像。如果需要完整的日志信息，可以添加`--detail`获取详细信息。 +解析依靠配置文件，默认情况会自动下载。如需更新配置文件，可以`--conf-update`，它将会在解析前强制下载一次配置文件。如果默认下载源不合适，可以同时配置`--conf-url`配置镜像源，例如使用`--conf-url https://openmldb.ai/download/diag/common_err.yml`配置国内镜像。 + +如果只需要完整的日志信息而不是解析日志的结果，可以使用`--detail`获取详细信息，不会打印解析结果。 ### test 测试 @@ -185,7 +251,6 @@ nameserver: 如果检查配置文件或日志，将会把收集到的文件保存在`--collect_dir`中，默认为`/tmp/diag_collect`。你也也可以访问此目录查看收集到的配置或日志，进行更多的分析。 - #### 检查示例 在镜像容器中可以这样静态检查： @@ -193,14 +258,15 @@ nameserver: openmldb_tool static-check --conf_file=/work/openmldb/conf/hosts -VCL --local ``` -### rpc +### RPC 接口 + +`openmldb_tool`还提供了一个RPC接口，它可以让我们发送RPC更容易，不需要定位Server的IP，拼接RPC方法URL路径，也可以提示所有RPC方法和RPC方法的输入结构。使用方式是`openmldb_tool rpc`，例如，`openmldb_tool rpc ns ShowTable --field '{"show_all":true}'`可以调用`nameserver`的`ShowTable`接口，获取表的状态信息。 -`openmldb_tool`还提供了一个RPC接口，但它是一个额外组件，需要通过`pip install openmldb-tool[rpc]`安装。使用方式是`openmldb_tool rpc`，例如，`openmldb_tool rpc ns ShowTable --field '{"show_all":true}'`可以调用`nameserver`的`ShowTable`接口，获取表的状态信息。 +其中组件不使用ip，可以直接使用角色名。NameServer与TaskManager只有一个活跃，所以我们用ns和tm来代表这两个组件。而TabletServer有多个，我们用`tablet1`，`tablet2`等来指定某个TabletServer，从1开始，顺序可通过`openmldb_tool rpc`或`openmldb_tool status`来查看。 -NameServer与TaskManager只有一个活跃，所以我们用ns和tm来代表这两个组件。 -而TabletServer有多个，我们用`tablet1`，`tablet2`等来指定某个TabletServer，顺序可通过`openmldb_tool rpc`或`openmldb_tool status`来查看。 +如果对RPC服务的方法或者输入参数不熟悉，可以通过`openmldb_tool rpc [method] --hint`查看帮助信息。但它是一个额外组件，需要通过`pip install openmldb-tool[pb]`安装。hint还需要额外的pb文件，帮助解析输入参数，默认是从`/tmp/diag_cache`中读取，如果不存在则自动下载。如果你已有相应的文件，或者已经手动下载，可以通过`--pbdir`指定该目录。 -如果对RPC服务的方法或者输入参数不熟悉，可以通过`openmldb_tool rpc [method] --hint`查看帮助信息。例如： +例如： ```bash $ openmldb_tool rpc ns ShowTable --hint ... @@ -212,9 +278,7 @@ You should input json like this, ignore round brackets in the key and double quo "(optional)show_all": "bool" }' ``` -hint还需要额外的pb文件，帮助解析输入参数，默认是从`/tmp/diag_cache`中读取，如果不存在则自动下载。如果你已有相应的文件，或者已经手动下载，可以通过`--pbdir`指定该目录。 ## 附加 可使用`openmldb_tool --helpfull`查看所有配置项。例如，`--sdk_log`可以打印sdk的日志（zk，glog），可用于调试。 - \ No newline at end of file diff --git a/python/openmldb_tool/diagnostic_tool/diagnose.py b/python/openmldb_tool/diagnostic_tool/diagnose.py index 8bd67719489..80639036981 100644 --- a/python/openmldb_tool/diagnostic_tool/diagnose.py +++ b/python/openmldb_tool/diagnostic_tool/diagnose.py @@ -31,6 +31,8 @@ import diagnostic_tool.server_checker as checker from diagnostic_tool.table_checker import TableChecker from diagnostic_tool.parser import LogParser +from .inspect import server_ins, table_ins, partition_ins, ops_ins, inspect_hint +from .rpc import RPC from absl import app from absl import flags @@ -81,7 +83,7 @@ def status(args): # --diff with dist conf file, conf_file is required if args.diff: - assert flags.FLAGS.conf_file, "need --conf_file" + assert flags.FLAGS.conf_file, "need --conf_file/-f" print( "only check components in conf file, if cluster has more components, ignore them" ) @@ -96,8 +98,31 @@ def status(args): def inspect(args): - insepct_online(args) - inspect_offline(args) + # report all + # 1. server level + connect = Connector() + status_checker = checker.StatusChecker(connect) + server_map = status_checker._get_components() + offlines = server_ins(server_map) + + # 2. table level + # we use `show table status` instead of partition inspection, cuz it's simple and quick + warn_tables = table_ins(connect) + + # if has unhealthy tables, do partition and ops check, otherwise skip + hints = [] + if warn_tables: + # 3. ns ops + related_ops = ops_ins(connect) + # 4. partition level and get some hint about table + hints = partition_ins(server_map, related_ops) + + # 5. hint + # let user know what to do + # 1) start offline servers + # 2) let user know the warning table is fatal or not, related ops, warn if offset is too large + # 3) if table not healthy and no related ops, use recoverdata + inspect_hint(offlines, hints) def insepct_online(args): @@ -121,7 +146,9 @@ def insepct_online(args): if getattr(args, "dist", False): table_checker = TableChecker(conn) - table_checker.check_distribution(dbs=flags.FLAGS.db.split(",")) + dbs = flags.FLAGS.db + db_list = dbs.split(",") if dbs else None + table_checker.check_distribution(dbs=db_list) def inspect_offline(args): @@ -241,7 +268,6 @@ def rpc(args): tm: taskmanager""" ) return - from diagnostic_tool.rpc import RPC # use status connction to get version conns_with_version = { @@ -313,18 +339,19 @@ def parse_arg(argv): # sub inspect inspect_parser = subparsers.add_parser( "inspect", - help="Inspect online and offline. Use `inspect [online/offline]` to inspect one.", + help="Get full inspect report, --nocolor for batch mode, --table_width for partition tables display", ) - # inspect online & offline + # TODO inspect_parser.set_defaults(command=inspect) + inspect_sub = inspect_parser.add_subparsers() - # inspect online + # inspect online TODO online = inspect_sub.add_parser("online", help="only inspect online table.") online.set_defaults(command=insepct_online) online.add_argument( "--dist", action="store_true", help="Inspect online distribution." ) - # inspect offline + # inspect offline TODO offline = inspect_sub.add_parser("offline", help="only inspect offline jobs.") offline.set_defaults(command=inspect_offline) # inspect job diff --git a/python/openmldb_tool/diagnostic_tool/inspect.py b/python/openmldb_tool/diagnostic_tool/inspect.py new file mode 100644 index 00000000000..6f9401f3b66 --- /dev/null +++ b/python/openmldb_tool/diagnostic_tool/inspect.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2021 4Paradigm +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" gen multi-level readable reports for cluster devops """ +from absl import flags +import json +from collections import defaultdict +from prettytable import PrettyTable + +from .rpc import RPC + +# ANSI escape codes +flags.DEFINE_bool("nocolor", False, "disable color output", short_name="noc") +flags.DEFINE_integer( + "table_width", + 12, + "max columns in one row, 1 partition use r+1 cols, set k*(r+1)", + short_name="tw", +) +flags.DEFINE_integer( + "offset_diff_thresh", 100, "offset diff threshold", short_name="od" +) + +# color: red, green +RED = "\033[31m" +GREEN = "\033[32m" +BLUE = "\033[34m" +YELLOW = "\033[1;33m" +RESET = "\033[0m" + + +# switch by nocolor flag +def cr_print(color, obj): + if flags.FLAGS.nocolor or color == None: + print(obj) + else: + print(f"{color}{obj}{RESET}") + + +def server_ins(server_map): + print("\n\nServer Detail") + print(server_map) + offlines = [] + for component, value_list in server_map.items(): + for endpoint, status in value_list: + if status != "online": + offlines.append(f"[{component}]{endpoint}") + continue # offline tablet is needlessly to rpc + if offlines: + s = "\n".join(offlines) + cr_print(RED, f"offline servers:\n{s}") + else: + cr_print(GREEN, "all servers online (no backup tm and apiserver)") + return offlines + + +# support nocolor +def light(color, symbol, detail): + if flags.FLAGS.nocolor: + return f"{symbol} {detail}" + else: + return f"{color}{symbol}{RESET} {detail}" + + +def state2light(state): + state = state.ljust(15) # state str should be less than 15 + if not state.startswith("k"): + # meta mismatch status, all red + return light(RED, "X", state) + else: + # meta match, get the real state + state = state[1:] + if state.startswith("TableNormal"): + # green + return light(GREEN, "O", state) + else: + # ref https://github.com/4paradigm/OpenMLDB/blob/0462f8a9682f8d232e8d44df7513cff66870d686/tools/tool.py#L291 + # undefined is loading too: state == "kTableLoading" or state == "kTableUndefined" + # TODO what about state == "kTableLoadingSnapshot" or state == "kSnapshotPaused" + return light(YELLOW, "=", state) + + +# similar with `show table status` warnings field, but easier to read +# prettytable.colortable just make table border and header lines colorful, so we color the value +def check_table_info(t, replicas_on_tablet, tablet2idx): + pnum, rnum = t["partition_num"], t["replica_num"] + assert pnum == len(t["table_partition"]) + # multi-line for better display, max display columns in one row + valuable_cols = pnum * (rnum + 1) + display_width = min(flags.FLAGS.table_width, valuable_cols) + # if real multi-line, the last line may < width, padding with empty string + rest = valuable_cols % display_width + total_cols = valuable_cols + (0 if rest == 0 else display_width - rest) + + idx_row = [""] * total_cols + leader_row = [""] * total_cols + followers_row = [""] * total_cols + + table_mark = 0 + hint = "" + for i, p in enumerate(t["table_partition"]): + # each partition add 3 row, and rnum + 1 columns + # tablet idx pid | 1 | 4 | 5 + # leader 1 o + # followers o o + pid = p["pid"] + assert pid == i + + # sort by list tablets + replicas = [] + for r in p["partition_meta"]: + tablet = r["endpoint"] + # tablet_has_partition useless + # print(r["endpoint"], r["is_leader"], r["tablet_has_partition"]) + replicas_on_t = replicas_on_tablet[t["tid"]][p["pid"]] + # may can't find replica on tablet, e.g. tablet server is not ready + info_on_tablet = {} + if r["endpoint"] not in replicas_on_t: + info_on_tablet = {"state": "Miss", "mode": "Miss", "offset": -1} + else: + info_on_tablet = replicas_on_t[r["endpoint"]] + # print(info_on_tablet) + m = { + "role": "leader" if r["is_leader"] else "follower", + "state": info_on_tablet["state"], + "acrole": info_on_tablet["mode"], + "offset": info_on_tablet["offset"], + } + replicas.append((tablet2idx[tablet], m)) + + assert len(replicas) == rnum + replicas.sort(key=lambda x: x[0]) + leader_ind = [i for i, r in enumerate(replicas) if r[1]["role"] == "leader"] + # replica on offline tablet is still in the ns meta, so leader may > 1 + # assert len(ind) <= 1, f"should be only one leader or miss leader in {replicas}" + + # show partition idx and tablet server idx + cursor = i * (rnum + 1) + idx_row[cursor : cursor + rnum + 1] = ["p" + str(pid)] + [ + r[0] for r in replicas + ] + + # fulfill leader line + if leader_ind: + for leader in leader_ind: + # leader state + lrep = replicas[leader][1] + if lrep["state"] != "Miss" and lrep["acrole"] != "kTableLeader": + lrep["state"] = "NotLeaderOnT" # modify the state + leader_row[cursor + leader + 1] = state2light(lrep["state"]) + else: + # can't find leader in nameserver metadata TODO: is it ok to set in the first column? + leader_row[cursor] = state2light("NotFound") + + # fulfill follower line + for i, r in enumerate(replicas): + idx = cursor + i + 1 + if i in leader_ind: + continue + frep = r[1] + if frep["state"] != "Miss" and frep["acrole"] != "kTableFollower": + frep["state"] = "NotFollowerOnT" + followers_row[idx] = state2light(frep["state"]) + + # after state adjust, diag table + replicas = [r[1] for r in replicas] # tablet server is needless now + # fatal: leader replica is not normal, may read/write fail + # get one normal leader, the partition can work + if not leader_ind or not any( + [replicas[i]["state"] == "kTableNormal" for i in leader_ind] + ): + table_mark = max(4, table_mark) + hint += f"partition {pid} leader replica is not normal\n" + # warn: need repair(may auto repair by auto_failover), but not in emergency + # follower replica is not normal + if any([r["state"] != "kTableNormal" for r in replicas]): + table_mark = max(3, table_mark) + hint += f"partition {pid} has unhealthy replicas\n" + + # offset is not consistent, only check normal replicas + offsets = [r["offset"] for r in replicas if r["state"] == "kTableNormal"] + if offsets and max(offsets) - min(offsets) > flags.FLAGS.offset_diff_thresh: + table_mark = max(3, table_mark) + hint += ( + f"partition {pid} has offset diff > {flags.FLAGS.offset_diff_thresh}\n" + ) + + x = PrettyTable(align="l") + + x.field_names = [i for i in range(display_width)] + step = display_width + for i in range(0, len(idx_row), step): + x.add_row(idx_row[i : i + step]) + x.add_row(leader_row[i : i + step]) + x.add_row(followers_row[i : i + step], divider=True) + + table_summary = "" + if table_mark >= 4: + table_summary = light( + RED, + "X", + f"Fatal table {t['db']}.{t['name']}, read/write may fail, need repair immediately", + ) + elif table_mark >= 3: + table_summary = light( + YELLOW, "=", f"Warn table {t['db']}.{t['name']}, still work, but need repair" + ) + if table_summary: + table_summary += "\n" + hint + return x, table_summary + + +def show_table_info(t, replicas_on_tablet, tablet2idx): + """check table info and display for ut""" + print( + f"Table {t['tid']} {t['db']}.{t['name']} {t['partition_num']} partitions {t['replica_num']} replicas" + ) + table, _ = check_table_info(t, replicas_on_tablet, tablet2idx) + print(table.get_string(border=True, header=False)) + + +def table_ins(connect): + print("\n\nTable Healthy Detail") + rs = connect.execfetch("show table status like '%';") + rs.sort(key=lambda x: x[0]) + print(f"summary: {len(rs)} tables(including system tables)") + warn_tables = [] + for t in rs: + # any warning means unhealthy, partition_unalive may be 0 but already unhealthy, warnings is accurate? + if t[13]: + warn_tables.append(f"{t[2]}.{t[1]}") + if warn_tables: + s = "\n".join(warn_tables) + cr_print(RED, f"unhealthy tables:\n{s}") + else: + cr_print(GREEN, "all tables are healthy") + return warn_tables + + +def partition_ins(server_map, related_ops): + print("\n\nTable Partition Detail") + # ns table info + rpc = RPC("ns") + res = rpc.rpc_exec("ShowTable", {"show_all": True}) + if not res: + cr_print(RED, "get table info failed or empty from nameserver") + return + res = json.loads(res) + all_table_info = res["table_info"] + # TODO split system tables and user tables + + # get table info from tablet server + # >> + replicas = defaultdict(lambda: defaultdict(dict)) + tablets = server_map["tablet"] # has status + invalid_tablets = set() + for tablet, status in tablets: + if status == "offline": + invalid_tablets.add(tablet) + continue + # GetTableStatusRequest empty field means get all + rpc = RPC(tablet) + res = None + try: + res = json.loads(rpc.rpc_exec("GetTableStatus", {})) + except Exception as e: + print(f"rpc {tablet} failed") + # may get empty when tablet server is not ready + if not res or res["code"] != 0: + cr_print(RED, f"get table status failed or empty from {tablet}(online)") + invalid_tablets.add(tablet) + continue + if "all_table_status" not in res: + # just empty replica on tablet, skip + continue + for rep in res["all_table_status"]: + rep["tablet"] = tablet + # tid, pid are int + tid, pid = rep["tid"], rep["pid"] + replicas[tid][pid][tablet] = rep + + tablet2idx = {tablet[0]: i + 1 for i, tablet in enumerate(tablets)} + print(f"tablet server order: {tablet2idx}") + if invalid_tablets: + cr_print( + RED, + f"some tablet servers are offline/bad, can't get table info(exclude empty table server): {invalid_tablets}", + ) + + # display, depends on table info, replicas are used to check + all_table_info.sort(key=lambda x: x["tid"]) + # related op map + related_ops_map = {} + for op in related_ops: + db = op[9] + table = op[10] + if db not in related_ops_map: + related_ops_map[db] = {} + if table not in related_ops_map[db]: + related_ops_map[db][table] = [] + related_ops_map[db][table].append(op) + # print(f"related ops: {related_ops_map}") + print("") # for better display + diag_result = [] + for t in all_table_info: + # no need to print healthy table + table, diag_hint = check_table_info(t, replicas, tablet2idx) + if diag_hint: + print( + f"Table {t['tid']} {t['db']}.{t['name']} {t['partition_num']} partitions {t['replica_num']} replicas" + ) + print(table.get_string(header=False)) + if t["db"] in related_ops_map and t["name"] in related_ops_map[t["db"]]: + diag_hint += f"related op: {sorted(related_ops_map[t['db']][t['name']], key=lambda x: x[11])}" # 11 is pid + diag_result.append(diag_hint) + # comment for table info display, only for unhealthy table TODO: draw a example + if diag_result: + print( + """ +Example: +tablet server order: {'xxx': 1, 'xxx': 2, 'xxx': 3} -> get real tablet addr by idx ++----+-------------------+------------------+------------------+ +| p0 | 1 | 2 | 3 | -> p0: partition 0, 1-3: tablet server idx +| | [light] status | | | -> leader replica is on tablet 1 +| | | [light] status | [light] status | -> follower replicas are on tablet 2, 3 ++----+-------------------+------------------+------------------+ +light: +Green O -> OK +Yellow = -> replica meta is ok but state is not normal +Red X -> NotFound/Miss/NotFollowerOnT/NotLeaderOnT""" + ) + return diag_result + + +def ops_ins(connect): + # op sorted by id TODO: detail to show all? + print("\n\nOps Detail") + print("> failed ops do not mean cluster is unhealthy, just for reference") + rs = connect.execfetch("show jobs from NameServer;") + should_warn = [] + from datetime import datetime + # already in order + ops = [list(op) for op in rs] + for i in range(len(ops)): + op = ops[i] + op[3] = str(datetime.fromtimestamp(int(op[3]) / 1000)) if op[4] else "..." + op[4] = str(datetime.fromtimestamp(int(op[4]) / 1000)) if op[4] else "..." + if op[2] != "FINISHED": + should_warn.append(op) + # peek last one to let user know if cluster has tried to recover, or we should wait + print("last one op(check time): ", ops[-1]) + if not should_warn: + print("all nameserver ops are finished") + else: + print("last 10 unfinished ops:") + print(*should_warn[-10:], sep="\n") + recover_type = ["kRecoverTableOP", "kChangeLeaderOP", "kReAddReplicaOP", "kOfflineReplicaOP"] + related_ops = [ + op + for op in should_warn + if op[1] in recover_type and op[2] in ["Submitted", "RUNNING"] + ] + return related_ops + + +def inspect_hint(server_hint, table_hints): # TODO: + print( + """ + +================== +Summary & Hint +================== +Server: +""" + ) + if server_hint: + cr_print(RED, f"offline servers {server_hint}, restart them first") + else: + cr_print(GREEN, "all servers online") + print("\nTable:\n") + for h in table_hints: + print(h) + if table_hints: + print( + """ + Make sure all servers online, and no ops for the table is running. + Repair table manually, run recoverdata, check https://openmldb.ai/docs/zh/main/maintain/openmldb_ops.html. + Check 'Table Partitions Detail' above for detail. + """ + ) + else: + cr_print(GREEN, "all tables are healthy") diff --git a/python/openmldb_tool/diagnostic_tool/pb.py b/python/openmldb_tool/diagnostic_tool/pb.py new file mode 100644 index 00000000000..de0b6877471 --- /dev/null +++ b/python/openmldb_tool/diagnostic_tool/pb.py @@ -0,0 +1,92 @@ +# Copyright 2021 4Paradigm +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.protobuf.descriptor import FieldDescriptor +from absl import flags + +class DescriptorHelper: + def __init__(self, service): + # TODO(hw): symbol_database is useful? + # lazy import + assert flags.FLAGS.pbdir, "pbdir not set" + import sys + from pathlib import Path + + sys.path.append(Path(flags.FLAGS.pbdir).as_posix()) + import tablet_pb2 + import name_server_pb2 + import taskmanager_pb2 + + pb_map = { + "TabletServer": tablet_pb2, + "NameServer": name_server_pb2, + "TaskManagerServer": taskmanager_pb2, + # "ApiServer": api_server_pb2, + # "DataSync": data_sync_pb2, + } + self.descriptor = pb_map[service].DESCRIPTOR.services_by_name[service] + + def get_input_json(self, method): + inp = self.descriptor.FindMethodByName(method).input_type + return Field.to_json(inp) + + +class Field: + def to_str(typ): + typ2str = { + FieldDescriptor.TYPE_DOUBLE: "double", + FieldDescriptor.TYPE_FLOAT: "float", + FieldDescriptor.TYPE_INT64: "int64", + FieldDescriptor.TYPE_UINT64: "uint64", + FieldDescriptor.TYPE_INT32: "int32", + FieldDescriptor.TYPE_FIXED64: "fixed64", + FieldDescriptor.TYPE_FIXED32: "fixed32", + FieldDescriptor.TYPE_BOOL: "bool", + FieldDescriptor.TYPE_STRING: "string", + FieldDescriptor.TYPE_GROUP: "group", + FieldDescriptor.TYPE_MESSAGE: "message", + FieldDescriptor.TYPE_BYTES: "bytes", + FieldDescriptor.TYPE_UINT32: "uint32", + } + return typ2str[typ] + + def to_json(field): + # label optional, required, or repeated. + label = {1: "optional", 2: "required", 3: "repeated"} + if isinstance(field, FieldDescriptor): + key = f"({label[field.label]})" + field.name + if field.type == FieldDescriptor.TYPE_MESSAGE: + value = Field.to_json(field.message_type) + elif field.type == FieldDescriptor.TYPE_ENUM: + value = "/".join([n.name for n in field.enum_type.values]) + else: + value = Field.to_str(field.type) + if field.label == 3: + # json list style + return {key: [value, "..."]} + else: + return {key: value} + else: + # field is a message + if field.containing_type and [f.name for f in field.fields] == [ + "key", + "value", + ]: + # treat key-value as map type, can't figure out custom type + # TODO(hw): it's ok to pass a json list to proto map? + return {"k": "v", "...": "..."} + d = {} + for f in field.fields: + d.update(Field.to_json(f)) + return d diff --git a/python/openmldb_tool/diagnostic_tool/rpc.py b/python/openmldb_tool/diagnostic_tool/rpc.py index 686734e7641..b0615ca6f81 100644 --- a/python/openmldb_tool/diagnostic_tool/rpc.py +++ b/python/openmldb_tool/diagnostic_tool/rpc.py @@ -12,103 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -from absl import flags import json import requests -from bs4 import BeautifulSoup -from google.protobuf.descriptor import FieldDescriptor from .server_checker import StatusChecker from .connector import Connector +from absl import flags + +# used by pb.py but set here for simplicity, we will check pbdir before call hint(import pb) flags.DEFINE_string( "pbdir", "/tmp/diag_cache", "pb2 root dir, if not set, will use the /pb2 directory in the same directory as this script", ) +def validate_ip_address(ip_string): + return not any(c.isalpha() for c in ip_string) -class DescriptorHelper: - def __init__(self, service): - # TODO(hw): symbol_database is useful? - # lazy import - assert flags.FLAGS.pbdir, "pbdir not set" - import sys - from pathlib import Path - sys.path.append(Path(flags.FLAGS.pbdir).as_posix()) - import tablet_pb2 - import name_server_pb2 - import taskmanager_pb2 - - pb_map = { - "TabletServer": tablet_pb2, - "NameServer": name_server_pb2, - "TaskManagerServer": taskmanager_pb2, - # "ApiServer": api_server_pb2, - # "DataSync": data_sync_pb2, - } - self.descriptor = pb_map[service].DESCRIPTOR.services_by_name[service] - - def get_input_json(self, method): - inp = self.descriptor.FindMethodByName(method).input_type - return Field.to_json(inp) - - -class Field: - def to_str(typ): - typ2str = { - FieldDescriptor.TYPE_DOUBLE: "double", - FieldDescriptor.TYPE_FLOAT: "float", - FieldDescriptor.TYPE_INT64: "int64", - FieldDescriptor.TYPE_UINT64: "uint64", - FieldDescriptor.TYPE_INT32: "int32", - FieldDescriptor.TYPE_FIXED64: "fixed64", - FieldDescriptor.TYPE_FIXED32: "fixed32", - FieldDescriptor.TYPE_BOOL: "bool", - FieldDescriptor.TYPE_STRING: "string", - FieldDescriptor.TYPE_GROUP: "group", - FieldDescriptor.TYPE_MESSAGE: "message", - FieldDescriptor.TYPE_BYTES: "bytes", - FieldDescriptor.TYPE_UINT32: "uint32", - } - return typ2str[typ] - - def to_json(field): - # label optional, required, or repeated. - label = {1: "optional", 2: "required", 3: "repeated"} - if isinstance(field, FieldDescriptor): - key = f"({label[field.label]})" + field.name - if field.type == FieldDescriptor.TYPE_MESSAGE: - value = Field.to_json(field.message_type) - elif field.type == FieldDescriptor.TYPE_ENUM: - value = "/".join([n.name for n in field.enum_type.values]) - else: - value = Field.to_str(field.type) - if field.label == 3: - # json list style - return {key: [value, "..."]} - else: - return {key: value} - else: - # field is a message - if field.containing_type and [f.name for f in field.fields] == [ - "key", - "value", - ]: - # treat key-value as map type, can't figure out custom type - # TODO(hw): it's ok to pass a json list to proto map? - return {"k": "v", "...": "..."} - d = {} - for f in field.fields: - d.update(Field.to_json(f)) - return d + +host2service = { + "nameserver": "NameServer", + "taskmanager": "openmldb.taskmanager.TaskManagerServer", + "tablet": "TabletServer", +} class RPC: """rpc service""" def __init__(self, host) -> None: - self.host, self.endpoint, self.service = RPC.get_endpoint_service(host.lower()) + if validate_ip_address(host): + self.endpoint = host + self.host = "tablet" # TODO: you can get ns/tm by name, it's not necessary to input ip + self.service = host2service[self.host] + else: + self.host, self.endpoint, self.service = RPC.get_endpoint_service( + host.lower() + ) def rpc_help(self): if self.host == "taskmanager": @@ -129,16 +70,16 @@ def hint(self, info): print(self.rpc_help()) return - # input message to json style - # if taskmanager, service in pb2 is TaskManagerServer service = ( self.service if not self.service.endswith("TaskManagerServer") else "TaskManagerServer" ) + from .pb import DescriptorHelper helper = DescriptorHelper(service) + # input message to json style json_str = json.dumps(helper.get_input_json(info), indent=4) print( f"You should input json like this, ignore round brackets in the key and double quotation marks in the value: --field '{json_str}'" @@ -168,14 +109,12 @@ def get_endpoint_service(host): host = "nameserver" if host == "ns" else "taskmanager" assert host in components_map, f"{host} not found in cluster" endpoint = components_map[host][num][0] - host2service = { - "nameserver": "NameServer", - "taskmanager": "openmldb.taskmanager.TaskManagerServer", - "tablet": "TabletServer", - } + service = host2service[host] return host, endpoint, service def parse_html(html): + from bs4 import BeautifulSoup + soup = BeautifulSoup(html, "html.parser") return soup.get_text("\n") diff --git a/python/openmldb_tool/diagnostic_tool/table_checker.py b/python/openmldb_tool/diagnostic_tool/table_checker.py index 969e7d110e4..6c4a2cf934e 100644 --- a/python/openmldb_tool/diagnostic_tool/table_checker.py +++ b/python/openmldb_tool/diagnostic_tool/table_checker.py @@ -24,11 +24,11 @@ class TableChecker: def __init__(self, conn: Connector): self.conn = conn - def check_distribution(self, dbs: list): + def check_distribution(self, dbs: list = None): exist_dbs = [db[0] for db in self.conn.execfetch("SHOW DATABASES")] if not exist_dbs: return - if dbs == ['']: + if not dbs or len(dbs) == 0: dbs = exist_dbs assert all([db in exist_dbs for db in dbs]), "some databases are not exist" @@ -36,67 +36,87 @@ def check_distribution(self, dbs: list): url = f"http://{ns_leader}/NameServer/ShowTable" res = requests.get(url, json={"show_all": True}) tables = res.json()["table_info"] - + if not tables or len(tables) == 0: + print("no table") + return tablet2partition = {} tablet2count = {} tablet2mem = {} tablet2dused = {} table_infos = [] - max_values = {'mp': 0, 'mc': 0, 'mm': 0, 'md': 0} + max_values = {"mp": 0, "mc": 0, "mm": 0, "md": 0} for table in tables: - if table['db'] not in dbs: + if table["db"] not in dbs: continue t = {} - t['name'] = table['db'] + "." + table['name'] - parts = table['table_partition'] - part_dist = self._collect(parts, '') - count_dist = self._collect(parts, 'record_cnt') - mem_dist = self._collect(parts, 'record_byte_size') - dused_dist = self._collect(parts, 'diskused') - max_values['mp'] = max(max_values['mp'], *part_dist.values()) - max_values['mc'] = max(max_values['mc'], *count_dist.values()) - max_values['mm'] = max(max_values['mm'], *mem_dist.values()) - max_values['md'] = max(max_values['md'], *dused_dist.values()) - t['part_size'] = len(parts) - t['part_dist'] = part_dist - t['count_dist'] = count_dist - t['mem_dist'] = mem_dist - t['dused_dist'] = dused_dist + t["name"] = table["db"] + "." + table["name"] + parts = table["table_partition"] + part_dist = self._collect(parts, "") + count_dist = self._collect(parts, "record_cnt") + mem_dist = self._collect(parts, "record_byte_size") + dused_dist = self._collect(parts, "diskused") + t["part_size"] = len(parts) + t["part_dist"] = part_dist + t["count_dist"] = count_dist + t["mem_dist"] = mem_dist + t["dused_dist"] = dused_dist table_infos.append(t) self._add_merge(tablet2partition, part_dist) self._add_merge(tablet2count, count_dist) self._add_merge(tablet2mem, mem_dist) self._add_merge(tablet2dused, dused_dist) - max_values['mm'] = round(max_values['mm'] / 1024 / 1024, 4) - max_values['md'] = round(max_values['md'] / 1024 / 1024, 4) + def get_max(di): + return max(di.values()) + + max_values["mp"] = get_max(tablet2partition) + max_values["mc"] = get_max(tablet2count) + max_values["mm"] = round(get_max(tablet2mem) / 1024 / 1024, 4) + max_values["md"] = round(get_max(tablet2dused) / 1024 / 1024, 4) + max_width = 40 for t in table_infos: print() - print(t['name']) - print('partition size:', t['part_size']) - print('partition dist(include replica)') - self._show_dist(t['part_dist'], max_width=max_width * max(*t['part_dist'].values()) / max_values['mp']) - print('record count dist(include replica)') - self._show_dist(t['count_dist'], max_width=0 if max_values['mc'] == 0 else max_width * max(*t['count_dist'].values()) / max_values['mc']) - print('mem dist(include replica)(MB)') - self._byte2mb(t['mem_dist']) - self._show_dist(t['mem_dist'], max_width=0 if max_values['mm'] == 0 else max_width * max(*t['mem_dist'].values()) / max_values['mm']) - print('diskused dist(include replica)(MB)') - self._byte2mb(t['dused_dist']) - self._show_dist(t['dused_dist'], max_width=max_width * max(*t['dused_dist'].values()) / max_values['md']) + print(t["name"], "distribution") + print("partition size:", t["part_size"]) + print("partition dist(include replica)") + self._show_dist( + t["part_dist"], + max_width=max_width * get_max(t["part_dist"]) / max_values["mp"], + ) + print("record count dist(include replica)") + self._show_dist( + t["count_dist"], + max_width=0 + if max_values["mc"] == 0 + else max_width * get_max(t["count_dist"]) / max_values["mc"], + ) + print("mem dist(include replica)(MB)") + self._byte2mb(t["mem_dist"]) + self._show_dist( + t["mem_dist"], + max_width=0 + if max_values["mm"] == 0 + else max_width * get_max(t["mem_dist"]) / max_values["mm"], + ) + print("diskused dist(include replica)(MB)") + self._byte2mb(t["dused_dist"]) + self._show_dist( + t["dused_dist"], + max_width=max_width * get_max(t["dused_dist"]) / max_values["md"], + ) print() - print('total') - print('tablet2partition') + print("tablet server load distribution") + print("tablet2partition") self._show_dist(tablet2partition) - print('tablet2count') + print("tablet2count") self._show_dist(tablet2count) - print('tablet2mem(MB)') + print("tablet2mem(MB)") self._byte2mb(tablet2mem) self._show_dist(tablet2mem) - print('tablet2diskused(MB)') + print("tablet2diskused(MB)") self._byte2mb(tablet2dused) self._show_dist(tablet2dused) @@ -106,16 +126,24 @@ def _byte2mb(self, dist: dict): def _show_dist(self, dist: dict, max_width=40): figc = tpl.figure() - figc.barh(list(dist.values()), labels=list(dist.keys()), max_width=max_width) + if not dist: # protect barh args + print("no data") + return + figc.barh( + list(dist.values()), + labels=list(dist.keys()), + max_width=max_width, + force_ascii=True, + ) figc.show() def _collect(self, parts, field): dist = {} for part in parts: - for replica in part['partition_meta']: - if replica['endpoint'] not in dist: - dist[replica['endpoint']] = 0 - dist[replica['endpoint']] += replica[field] if field else 1 + for replica in part["partition_meta"]: + if replica["endpoint"] not in dist: + dist[replica["endpoint"]] = 0 + dist[replica["endpoint"]] += replica[field] if field else 1 return dist def _add_merge(self, dist, dist2): diff --git a/python/openmldb_tool/setup.py b/python/openmldb_tool/setup.py index 7b9a8dcf27f..555e5b51153 100644 --- a/python/openmldb_tool/setup.py +++ b/python/openmldb_tool/setup.py @@ -28,7 +28,7 @@ "Programming Language :: Python :: 3", ], install_requires=[ - "openmldb >= 0.6.9", + "openmldb >= 0.8.1", "absl-py", "pyyaml", "paramiko", @@ -36,12 +36,12 @@ "requests", ], extras_require={ - "rpc": [ + "pb": [ "protobuf==3.6.1", "beautifulsoup4", ], "test": [ - "openmldb-tool[rpc]", + "openmldb-tool[pb]", "pytest", ], }, diff --git a/python/openmldb_tool/tests/inspect_test.py b/python/openmldb_tool/tests/inspect_test.py new file mode 100644 index 00000000000..a548a76f81c --- /dev/null +++ b/python/openmldb_tool/tests/inspect_test.py @@ -0,0 +1,340 @@ +import pytest +from diagnostic_tool.inspect import show_table_info +from absl import flags + +flags.FLAGS["nocolor"].parse(False) +flags.FLAGS["table_width"].parse(12) + + +def test_show(): + # assume 3 tablet server + tablets = ["0.0.0.0:1111", "0.0.0.0:2222", "0.0.0.0:3333"] + tablet2idx = {tablet: i + 1 for i, tablet in enumerate(tablets)} + # simple + t_info = { + "name": "TABLE_A", + "table_partition": [ + { + "pid": 0, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + # "offset": 0, + # "record_cnt": 0, + # "record_byte_size": 0, + # "tablet_has_partition": true, + # "diskused": 9025, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + ], + # "term_offset": [{"term": 1, "offset": 0}], + # "record_cnt": 0, + # "record_byte_size": 0, + # "diskused": 9025, + } + ], + "tid": 0, + "partition_num": 1, + "replica_num": 2, + "db": "DB_A", + } + + replicas = { + 0: { + 0: { + tablets[0]: { + "tid": 0, # actually not used in show_table_info + "pid": 0, # not used in show_table_info + # "offset": 5, + "mode": "kTableLeader", + "state": "kTableNormal", + # "is_expire": True, + # "record_cnt": 1, + # "idx_cnt": 1, + # "ts_idx_status": [ + # {"idx_name": "id", "seg_cnts": [0, 0, 0, 0, 0, 0, 1, 0]} + # ], + "name": "Foo", + # "record_byte_size": 127, + # "record_idx_byte_size": 177, + # "record_pk_cnt": 1, + # "compress_type": "kNoCompress", + # "skiplist_height": 1, + # "diskused": 10074, + # "storage_mode": "kMemory", + "tablet": tablets[0], + }, + tablets[1]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[1], + }, + } + } + } + + show_table_info(t_info, replicas, tablet2idx) + + print("healthy ns meta, but replicas on tablet are all follower") + t_info = { + "name": "TABLE_A", + "table_partition": [ + { + "pid": 0, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + { + "endpoint": tablets[2], + "is_leader": False, + }, + ], + } + ], + "tid": 0, + "partition_num": 1, + "replica_num": 3, + "db": "DB_A", + } + replicas = { + 0: { + 0: { + tablets[0]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[0], + }, + tablets[1]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[1], + }, + tablets[2]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[2], + }, + } + } + } + show_table_info(t_info, replicas, tablet2idx) + + print("ns meta all followers, no leader") + t_info = { + "name": "TABLE_A", + "table_partition": [ + { + "pid": 0, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": False, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + { + "endpoint": tablets[2], + "is_leader": False, + }, + ], + } + ], + "tid": 0, + "partition_num": 1, + "replica_num": 3, + "db": "DB_A", + } + replicas = { + 0: { + 0: { + tablets[0]: { + "mode": "kTableLeader", + "state": "kTableNormal", + "tablet": tablets[0], + }, + tablets[1]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[1], + }, + tablets[2]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[2], + }, + } + } + } + show_table_info(t_info, replicas, tablet2idx) + + print("no corresponding replica on tablet server") + t_info = { + "name": "TABLE_A", + "table_partition": [ + { + "pid": 0, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + { + "endpoint": tablets[2], + "is_leader": False, + }, + ], + } + ], + "tid": 0, + "partition_num": 1, + "replica_num": 3, + "db": "DB_A", + } + replicas = { + 0: { + 0: { + tablets[1]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[1], + }, + tablets[2]: { + "mode": "kTableFollower", + "state": "kTableNormal", + "tablet": tablets[2], + }, + } + } + } + show_table_info(t_info, replicas, tablet2idx) + + print("meta match, but state is not normal") + t_info = { + "name": "TABLE_A", + "table_partition": [ + { + "pid": 0, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + { + "endpoint": tablets[2], + "is_leader": False, + }, + ], + }, + { + "pid": 1, + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + }, + { + "endpoint": tablets[1], + "is_leader": False, + }, + { + "endpoint": tablets[2], + "is_leader": False, + }, + ], + }, + ], + "tid": 0, + "partition_num": 2, + "replica_num": 3, + "db": "DB_A", + } + replicas = { + 0: { + 0: { + tablets[0]: { + "mode": "kTableFollower", + "state": "kTableLoading", + "tablet": tablets[0], + }, + tablets[1]: { + "mode": "kTableFollower", + "state": "kMakingSnapshot", + "tablet": tablets[1], + }, + tablets[2]: { + "mode": "kTableFollower", + "state": "kSnapshotPaused", + "tablet": tablets[2], + }, + }, + 1: { + tablets[1]: { + "mode": "kTableFollower", + "state": "kTableUndefined", + }, + tablets[2]: { + "mode": "kTableFollower", + "state": "kTableNormal", + }, + }, + } + } + show_table_info(t_info, replicas, tablet2idx) + + print("more partitions, display well") + partnum = 13 + meta_pattern = { + "partition_meta": [ + { + "endpoint": tablets[0], + "is_leader": True, + }, + ], + } + t_info = { + "name": "TABLE_A", + "table_partition": [], + "tid": 0, + "partition_num": partnum, + "replica_num": 1, + "db": "DB_A", + } + replicas = {0: {}} + + for i in range(partnum): + t_info["table_partition"].append({"pid": i, **meta_pattern}) + + for i in range(partnum): + replicas[0][i] = { + tablets[0]: { + "mode": "kTableLeader", + "state": "kTableNormal", + "tablet": tablets[0], + } + } + print(t_info, replicas) + show_table_info(t_info, replicas, tablet2idx) + + print("nocolor") + flags.FLAGS["nocolor"].parse(True) + show_table_info(t_info, replicas, tablet2idx) From 4f5d0ea4e321e4a4f7662d52a639e63d2f3bb562 Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Wed, 18 Oct 2023 16:18:35 +0800 Subject: [PATCH 2/6] docs: diag and must read --- docs/zh/maintain/diagnose.md | 12 +++++++++++- docs/zh/quickstart/beginner_must_read.md | 24 ++++++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/zh/maintain/diagnose.md b/docs/zh/maintain/diagnose.md index a5d126f09df..93395d76791 100644 --- a/docs/zh/maintain/diagnose.md +++ b/docs/zh/maintain/diagnose.md @@ -15,7 +15,8 @@ openmldb_tool # 注意下划线 ```bash usage: openmldb_tool [-h] [--helpfull] {status,inspect,rpc,test,static-check} ... ``` -只有`static-check`静态检查命令需要指定`--dist_conf`参数，该参数指定OpenMLDB节点分布的配置文件。其他命令只需要`--cluster`参数，格式为`/`，默认为镜像中的OpenMLDB集群地址`127.0.0.1:2181/openmldb`。如果是自行设置的OpenMLDB集群，请配置此参数。 + +注意`-c/--cluster`参数，格式为`/`，默认将访问`127.0.0.1:2181/openmldb`。如果是自行设置的OpenMLDB集群，请配置此参数。其他参数根据子命令不同而不同，可以使用`-h`查看，或查看各个子命令的详细文档。 ### 一键inspect @@ -70,6 +71,15 @@ Table: Check 'Table Partitions Detail' above for detail. ``` +### 其他常用命令 + +除了一键inspect，在这样几个场景中，我们推荐使用诊断工具的子命令来帮助用户判断集群状态、简化运维。 + +- 部署好集群后，可以使用`test`测试集群是否能正常工作，不需要用户手动测试。如果发现问题，再使用`inspect`诊断。 +- 离线job如果出现问题，`SHOW JOBLOG id`可以查看日志，但经验较少的用户可能会被日志中的无关信息干扰，可以使用`inspect job`来提取job日志中的关键信息。 +- 在一些棘手的问题中，可能需要用户通过RPC来获得一些信息，帮助定位问题。`openmldb_tool rpc`可以帮助用户简单快速地调用RPC，降低运维门槛。 +- 如果你的操作节点到各个组件的机器是ssh免密的，那么，可以使用`static-check`检查配置文件是否正确，版本是否统一，避免部署失败。还可以一键收集整个集群的日志，方便打包并提供给开发人员分析。 + ## 子命令详情 ### status 状态 diff --git a/docs/zh/quickstart/beginner_must_read.md b/docs/zh/quickstart/beginner_must_read.md index f5ba729613f..def0e3728d1 100644 --- a/docs/zh/quickstart/beginner_must_read.md +++ b/docs/zh/quickstart/beginner_must_read.md @@ -2,6 +2,20 @@ 由于OpenMLDB是分布式系统，多种模式，客户端丰富，初次使用可能会有很多疑问，或者遇到一些运行、使用问题，本文从新手使用的角度，讲解如何进行诊断调试，需求帮助时如何提供有效信息给技术人员等等。 +## 错误诊断 + +在使用OpenMLDB的过程中，除了SQL语法错误，其他错误信息可能不够直观，但很可能与集群状态有关。所以，错误诊断需要**先确认集群状态**。在发现错误时，请先使用诊断工具的一键诊断功能。一键诊断可以输出全面直观的诊断报告，如果不能使用此工具，可以手动执行`SHOW COMPONENTS;`和`SHOW TABLE STATUS LIKE '%';`提供部分信息。 + +报告将展示集群的组件、在线表等状态，也会提示用户如何修复，请按照报告内容进行操作，详情见[一键inspect](../maintain/diagnose.md#一键inspect)。 + +``` +openmldb_tool inspect [-c=0.0.0.0:2181/openmldb] +``` + +需要注意，由于离线存储只会在执行离线job时被读取，而离线job也不是一个持续的状态，所以，一键诊断只能展示TaskManager组件状态，不会诊断离线存储，也无法诊断离线job的执行错误，离线job诊断见[离线SQL执行](#离线)。 + +如果诊断报告认为集群健康，但仍然无法解决问题，请提供错误和诊断报告给我们。 + ## 创建OpenMLDB与连接 首先，我们建议不熟悉分布式多进程管理的新手使用docker创建OpenMLDB，方便快速上手。熟悉OpenMLDB各组件之后，再尝试分布式部署。 @@ -71,12 +85,14 @@ create table t1(c1 int; 如果是集群离线命令，默认异步模式下，发送命令会得到job id的返回。可使用`show job `来查询job执行情况。 -离线job如果是异步SELECT（并不INTO保存结果），也不会将结果打印在客户端（同步SELECT将会打印结果）。可以通过`show joblog `来获得结果，结果中包含stdout和stderr两部分，stdout为查询结果，stderr为job运行日志。如果发现job failed或者其他状态，不符合你的预期，请仔细查看job运行日志。 +离线job如果是异步SELECT（并不INTO保存结果），也不会将结果打印在客户端，而同步SELECT将会打印结果到控制台。可以通过`show joblog `来获得结果，结果中包含stdout和stderr两部分，stdout为查询结果，stderr为job运行日志。如果发现job failed或者其他状态，不符合你的预期，请仔细查看job运行日志。 -```{note} -日志地址由taskmanager.properties的`job.log.path`配置，如果你改变了此配置项，需要到配置的目的地寻找日志。stdout日志默认在`/work/openmldb/taskmanager/bin/logs/job_x.log`，job运行日志默认在`/work/openmldb/taskmanager/bin/logs/job_x_error.log`(注意有error后缀)， +离线job日志中可能有一定的干扰日志，用户可以使用`openmldb_tool inspect job --id x`进行日志的解析提取，帮助定位错误，更多信息请参考[诊断工具job检查](../maintain/diagnose.md#job-检查)。 -如果taskmanager是yarn模式，而不是local模式，`job_x_error.log`中的信息会较少，不会有job错误的详细信息。需要通过`job_x_error.log`中记录的yarn app id，去yarn系统中查询job的真正错误原因。 +如果taskmanager是yarn模式，而不是local模式，`job_x_error.log`中的信息会较少，只会打印异常。如果异常不直观，需要更早时间的执行日志，执行日志不在`job_x_error.log`中，需要通过`job_x_error.log`中记录的yarn app id，去yarn系统中查询yarn app的container的日志。yarn app container里，执行日志也保存在stderr中。 + +```{note} +如果你无法通过show joblog获得日志，或者想要直接拿到日志文件，可以直接在TaskManager机器上获取。日志地址由taskmanager.properties的`job.log.path`配置，如果你改变了此配置项，需要到配置的目录中寻找日志。stdout查询结果默认在`/work/openmldb/taskmanager/bin/logs/job_x.log`，stderr job运行日志默认在`/work/openmldb/taskmanager/bin/logs/job_x_error.log`(注意有error后缀)。 ``` #### 在线 From d428b8466de854e212bdd8c4bb2500d8cb3911ba Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Wed, 18 Oct 2023 16:39:18 +0800 Subject: [PATCH 3/6] fix --- docs/zh/maintain/diagnose.md | 2 ++ .../openmldb_tool/diagnostic_tool/diagnose.py | 31 +++++++++---------- .../openmldb_tool/diagnostic_tool/inspect.py | 5 +-- .../diagnostic_tool/table_checker.py | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/zh/maintain/diagnose.md b/docs/zh/maintain/diagnose.md index 93395d76791..0dabc24b3bc 100644 --- a/docs/zh/maintain/diagnose.md +++ b/docs/zh/maintain/diagnose.md @@ -77,7 +77,9 @@ Table: - 部署好集群后，可以使用`test`测试集群是否能正常工作，不需要用户手动测试。如果发现问题，再使用`inspect`诊断。 - 离线job如果出现问题，`SHOW JOBLOG id`可以查看日志，但经验较少的用户可能会被日志中的无关信息干扰，可以使用`inspect job`来提取job日志中的关键信息。 +- 离线job太多时，CLI中的展示会不容易读，可以使用`inspect offline`筛选所有failed的job，或者`inspect job --state `来筛选出特定状态的job。 - 在一些棘手的问题中，可能需要用户通过RPC来获得一些信息，帮助定位问题。`openmldb_tool rpc`可以帮助用户简单快速地调用RPC，降低运维门槛。 +- 没有Prometheus监控时，可以通过`inspect online --dist`获得数据分布信息。 - 如果你的操作节点到各个组件的机器是ssh免密的，那么，可以使用`static-check`检查配置文件是否正确，版本是否统一，避免部署失败。还可以一键收集整个集群的日志，方便打包并提供给开发人员分析。 ## 子命令详情 diff --git a/python/openmldb_tool/diagnostic_tool/diagnose.py b/python/openmldb_tool/diagnostic_tool/diagnose.py index 80639036981..614c904063d 100644 --- a/python/openmldb_tool/diagnostic_tool/diagnose.py +++ b/python/openmldb_tool/diagnostic_tool/diagnose.py @@ -39,7 +39,7 @@ from absl.flags import argparse_flags from absl import logging # --verbosity --log_dir -# only some sub cmd needs dist file +# only some sub cmd needs dist file TODO(hw): better to move then to other py file, to avoid -h show them flags.DEFINE_string( "conf_file", "", @@ -126,26 +126,22 @@ def inspect(args): def insepct_online(args): - """show table status""" - conn = Connector() + """inspect online""" + connect = Connector() # scan all db include system db - fails = [] - rs = conn.execfetch("show table status like '%';") - rs.sort(key=lambda x: x[0]) - print(f"inspect {len(rs)} online tables(including system tables)") - for t in rs: + fails = table_ins(connect) + for t in fails: if t[13]: print(f"unhealthy table {t[2]}.{t[1]}:\n {t[:13]}") # sqlalchemy truncated ref https://github.com/sqlalchemy/sqlalchemy/commit/591e0cf08a798fb16e0ee9b56df5c3141aa48959 # so we print warnings alone print(f"full warnings:\n{t[13]}") - fails.append(f"{t[2]}.{t[1]}") - - assert not fails, f"unhealthy tables: {fails}" - print(f"all tables are healthy") + # if has fails, summary will print in table_ins + if not fails: + print(f"all tables are healthy") if getattr(args, "dist", False): - table_checker = TableChecker(conn) + table_checker = TableChecker(connect) dbs = flags.FLAGS.db db_list = dbs.split(",") if dbs else None table_checker.check_distribution(dbs=db_list) @@ -341,18 +337,19 @@ def parse_arg(argv): "inspect", help="Get full inspect report, --nocolor for batch mode, --table_width for partition tables display", ) - # TODO inspect_parser.set_defaults(command=inspect) inspect_sub = inspect_parser.add_subparsers() - # inspect online TODO + # inspect online online = inspect_sub.add_parser("online", help="only inspect online table.") online.set_defaults(command=insepct_online) online.add_argument( "--dist", action="store_true", help="Inspect online distribution." ) - # inspect offline TODO - offline = inspect_sub.add_parser("offline", help="only inspect offline jobs.") + # inspect offline + offline = inspect_sub.add_parser( + "offline", help="only inspect offline jobs, show failed jobs." + ) offline.set_defaults(command=inspect_offline) # inspect job ins_job = inspect_sub.add_parser( diff --git a/python/openmldb_tool/diagnostic_tool/inspect.py b/python/openmldb_tool/diagnostic_tool/inspect.py index 6f9401f3b66..2a3622ad8c0 100644 --- a/python/openmldb_tool/diagnostic_tool/inspect.py +++ b/python/openmldb_tool/diagnostic_tool/inspect.py @@ -241,9 +241,10 @@ def table_ins(connect): for t in rs: # any warning means unhealthy, partition_unalive may be 0 but already unhealthy, warnings is accurate? if t[13]: - warn_tables.append(f"{t[2]}.{t[1]}") + warn_tables.append(t) if warn_tables: - s = "\n".join(warn_tables) + # only show tables name + s = "\n".join([f"{t[2]}.{t[1]}" for t in warn_tables]) cr_print(RED, f"unhealthy tables:\n{s}") else: cr_print(GREEN, "all tables are healthy") diff --git a/python/openmldb_tool/diagnostic_tool/table_checker.py b/python/openmldb_tool/diagnostic_tool/table_checker.py index 6c4a2cf934e..f9703054d5c 100644 --- a/python/openmldb_tool/diagnostic_tool/table_checker.py +++ b/python/openmldb_tool/diagnostic_tool/table_checker.py @@ -111,7 +111,7 @@ def get_max(di): print("tablet server load distribution") print("tablet2partition") self._show_dist(tablet2partition) - print("tablet2count") + print("tablet2count(row)") self._show_dist(tablet2count) print("tablet2mem(MB)") self._byte2mb(tablet2mem) From 5d9080e51ff2549c60d861e3cf9e9c0391be6b96 Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Wed, 18 Oct 2023 19:26:02 +0800 Subject: [PATCH 4/6] fix --- docs/zh/maintain/diagnose.md | 13 +++- python/openmldb_tool/README.md | 16 ++-- .../openmldb_tool/diagnostic_tool/diagnose.py | 6 +- .../openmldb_tool/diagnostic_tool/inspect.py | 9 +-- python/openmldb_tool/diagnostic_tool/pb.py | 74 +++++++++++++------ python/openmldb_tool/diagnostic_tool/rpc.py | 15 ++-- src/proto/tablet.proto | 2 +- 7 files changed, 89 insertions(+), 46 deletions(-) diff --git a/docs/zh/maintain/diagnose.md b/docs/zh/maintain/diagnose.md index 0dabc24b3bc..3d8764c3114 100644 --- a/docs/zh/maintain/diagnose.md +++ b/docs/zh/maintain/diagnose.md @@ -76,6 +76,7 @@ Table: 除了一键inspect，在这样几个场景中，我们推荐使用诊断工具的子命令来帮助用户判断集群状态、简化运维。 - 部署好集群后，可以使用`test`测试集群是否能正常工作，不需要用户手动测试。如果发现问题，再使用`inspect`诊断。 +- 组件都在线，但出现超时或错误提示某组件无法连接时，可以使用`status --conn`检查与各组件的连接，会打印出简单访问的耗时。也可以用它来测试客户端主机与集群的连接情况，及时发现网络隔离。 - 离线job如果出现问题，`SHOW JOBLOG id`可以查看日志，但经验较少的用户可能会被日志中的无关信息干扰，可以使用`inspect job`来提取job日志中的关键信息。 - 离线job太多时，CLI中的展示会不容易读，可以使用`inspect offline`筛选所有failed的job，或者`inspect job --state `来筛选出特定状态的job。 - 在一些棘手的问题中，可能需要用户通过RPC来获得一些信息，帮助定位问题。`openmldb_tool rpc`可以帮助用户简单快速地调用RPC，降低运维门槛。 @@ -94,7 +95,8 @@ usage: openmldb_tool status [-h] [--helpfull] [--diff] optional arguments: -h, --help show this help message and exit --helpfull show full help message and exit - --diff check if all endpoints in conf are in cluster. If set, need to set `--conf_file/-f` + --diff check if all endpoints in conf are in cluster. If set, need to set `-f,--conf_file` + --conn check network connection of all servers ``` - 简单查询集群状态： @@ -113,6 +115,11 @@ optional arguments: +-----------------+-------------+---------------+--------+---------+ ``` +- 检查并测试集群链接与版本： + ``` + openmldb_tool status --conn + ``` + #### 检查配置文件与集群状态是否一致 如果指定`--diff`参数，会检查配置文件中的所有节点是否都在已经启动的集群中，如果有节点不在集群中，会输出异常信息。如果集群中有节点不在配置文件中，不会输出异常信息。需要配置`-f,--conf_file`，例如，你可以在镜像里这样检查： @@ -145,7 +152,7 @@ positional arguments: #### offline离线检查 -`inspect offline`离线检查会输出最终状态为失败的任务（不检查“运行中”的任务），等价于`SHOW JOBS`并筛选出失败任务。 +`inspect offline`离线检查会输出最终状态为失败的任务（不检查“运行中”的任务），等价于`SHOW JOBS`并筛选出失败任务。更多功能待补充。 #### JOB 检查 @@ -276,7 +283,7 @@ openmldb_tool static-check --conf_file=/work/openmldb/conf/hosts -VCL --local 其中组件不使用ip，可以直接使用角色名。NameServer与TaskManager只有一个活跃，所以我们用ns和tm来代表这两个组件。而TabletServer有多个，我们用`tablet1`，`tablet2`等来指定某个TabletServer，从1开始，顺序可通过`openmldb_tool rpc`或`openmldb_tool status`来查看。 -如果对RPC服务的方法或者输入参数不熟悉，可以通过`openmldb_tool rpc [method] --hint`查看帮助信息。但它是一个额外组件，需要通过`pip install openmldb-tool[pb]`安装。hint还需要额外的pb文件，帮助解析输入参数，默认是从`/tmp/diag_cache`中读取，如果不存在则自动下载。如果你已有相应的文件，或者已经手动下载，可以通过`--pbdir`指定该目录。 +如果对RPC服务的方法或者输入参数不熟悉，可以通过`openmldb_tool rpc [method] --hint`查看帮助信息。但它是一个额外组件，需要通过`pip install openmldb-tool[pb]`安装。hint还需要额外的pb文件，帮助解析输入参数，默认是从`/tmp/diag_cache`中读取，如果不存在则自动下载。如果你已有相应的文件，或者已经手动下载，可以通过`--pbdir`指定该目录。自行编译pb文件，见[openmldb tool开发文档](https://github.com/4paradigm/OpenMLDB/blob/main/python/openmldb_tool/README.md#rpc)。 例如： ```bash diff --git a/python/openmldb_tool/README.md b/python/openmldb_tool/README.md index 3381751edf9..d5168a4bf25 100644 --- a/python/openmldb_tool/README.md +++ b/python/openmldb_tool/README.md @@ -48,21 +48,27 @@ status [-h] [--helpfull] [--diff DIFF] optional arguments: -h, --help show this help message and exit --helpfull show full help message and exit - --diff check if all endpoints in conf are in cluster. If set, need to set `--conf_file` + --diff check if all endpoints in conf are in cluster. If set, need to set `-f,--conf_file` ``` Use `show components` to show servers(no apiserver now). +--conn: +- ping all servers, brpc /health to check ok，and +- online servers version and cost time, we can get from brpc http:///version. (ns,tablet, apiserver set_version in brpc server) + TODO: -- ping all servers, brpc /health to check ok -- online servers version, we can get from brpc http:///version. (ns,tablet, apiserver set_version in brpc server) - brpc /flags to get all gflags(including openmldb), `--enable_flags_service=true` required ## Inspect -Use `show table status like '%';` in all dbs, even the hidden db(system db). +`inspect` for full report, no offline diag now. + +inspect online: Use `show table status like '%';` in all dbs, even the hidden db(system db). + +inspect offline: failed jobs, no more info. TODO: check register table? -If you found some online tables are not behaving properly, do inspect online. +inspect job: full support of offline job, select jobs, parse job log ## Test diff --git a/python/openmldb_tool/diagnostic_tool/diagnose.py b/python/openmldb_tool/diagnostic_tool/diagnose.py index 614c904063d..7a3b9761663 100644 --- a/python/openmldb_tool/diagnostic_tool/diagnose.py +++ b/python/openmldb_tool/diagnostic_tool/diagnose.py @@ -83,7 +83,7 @@ def status(args): # --diff with dist conf file, conf_file is required if args.diff: - assert flags.FLAGS.conf_file, "need --conf_file/-f" + assert flags.FLAGS.conf_file, "need -f,--conf_file" print( "only check components in conf file, if cluster has more components, ignore them" ) @@ -151,7 +151,7 @@ def inspect_offline(args): """scan jobs status, show job log if failed""" final_failed = ["failed", "killed", "lost"] total, num, jobs = _get_jobs(final_failed) - # TODO some failed jobs are known, what if we want skip them? + # TODO some failed jobs are known or too old, what if we want skip them? print(f"inspect {total} offline jobs") if num: failed_jobs_str = "\n".join(jobs) @@ -323,7 +323,7 @@ def parse_arg(argv): status_parser.add_argument( "--diff", action="store_true", - help="check if all endpoints in conf are in cluster. If set, need to set `--conf_file`", + help="check if all endpoints in conf are in cluster. If set, need to set `-f,--conf_file`", ) # TODO action support in all python 3.x? status_parser.add_argument( "--conn", diff --git a/python/openmldb_tool/diagnostic_tool/inspect.py b/python/openmldb_tool/diagnostic_tool/inspect.py index 2a3622ad8c0..c9cb626e40e 100644 --- a/python/openmldb_tool/diagnostic_tool/inspect.py +++ b/python/openmldb_tool/diagnostic_tool/inspect.py @@ -89,7 +89,7 @@ def state2light(state): else: # ref https://github.com/4paradigm/OpenMLDB/blob/0462f8a9682f8d232e8d44df7513cff66870d686/tools/tool.py#L291 # undefined is loading too: state == "kTableLoading" or state == "kTableUndefined" - # TODO what about state == "kTableLoadingSnapshot" or state == "kSnapshotPaused" + # snapshot doesn't mean unhealthy: state == "kMakingSnapshot" or state == "kSnapshotPaused" return light(YELLOW, "=", state) @@ -162,7 +162,7 @@ def check_table_info(t, replicas_on_tablet, tablet2idx): lrep["state"] = "NotLeaderOnT" # modify the state leader_row[cursor + leader + 1] = state2light(lrep["state"]) else: - # can't find leader in nameserver metadata TODO: is it ok to set in the first column? + # can't find leader in nameserver metadata, set in the first column(we can't find leader on any tablet) leader_row[cursor] = state2light("NotFound") # fulfill follower line @@ -261,7 +261,6 @@ def partition_ins(server_map, related_ops): return res = json.loads(res) all_table_info = res["table_info"] - # TODO split system tables and user tables # get table info from tablet server # >> @@ -347,7 +346,7 @@ def partition_ins(server_map, related_ops): def ops_ins(connect): - # op sorted by id TODO: detail to show all? + # op sorted by id TODO: detail to show all include succ op? print("\n\nOps Detail") print("> failed ops do not mean cluster is unhealthy, just for reference") rs = connect.execfetch("show jobs from NameServer;") @@ -377,7 +376,7 @@ def ops_ins(connect): return related_ops -def inspect_hint(server_hint, table_hints): # TODO: +def inspect_hint(server_hint, table_hints): print( """ diff --git a/python/openmldb_tool/diagnostic_tool/pb.py b/python/openmldb_tool/diagnostic_tool/pb.py index de0b6877471..06219d00b61 100644 --- a/python/openmldb_tool/diagnostic_tool/pb.py +++ b/python/openmldb_tool/diagnostic_tool/pb.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.protobuf.descriptor import FieldDescriptor +from google.protobuf.descriptor import Descriptor, FieldDescriptor from absl import flags + class DescriptorHelper: def __init__(self, service): - # TODO(hw): symbol_database is useful? # lazy import assert flags.FLAGS.pbdir, "pbdir not set" import sys @@ -28,6 +28,8 @@ def __init__(self, service): import name_server_pb2 import taskmanager_pb2 + # google.protobuf.symbol_database can get service desc by name, but we have already included all pb2 files we need + # just use one file pb_map = { "TabletServer": tablet_pb2, "NameServer": name_server_pb2, @@ -36,10 +38,20 @@ def __init__(self, service): # "DataSync": data_sync_pb2, } self.descriptor = pb_map[service].DESCRIPTOR.services_by_name[service] + # from google.protobuf import symbol_database + # self.sym_db = symbol_database.Default() def get_input_json(self, method): - inp = self.descriptor.FindMethodByName(method).input_type - return Field.to_json(inp) + m = self.descriptor.FindMethodByName(method) + if not m: + return False, f"method {method} not found" + if not m.input_type.fields: # e.g. ShowTabletRequest is emtpy + return False, f"method {method} has no input" + # GeneratedProtocolMessageType __dict__ is complex, can't use it directly + # cl = self.sym_db.GetSymbol(m.input_type.full_name) + + # fields build a map, message is Descriptor, fields in msg is FieldDescriptor + return True, Field.to_json(m.input_type) class Field: @@ -64,29 +76,43 @@ def to_str(typ): def to_json(field): # label optional, required, or repeated. label = {1: "optional", 2: "required", 3: "repeated"} + + def is_map(f): + # I'm a map(containing_type = who includes me and my fields name are key-value) + # e.g. tm RunBatchSql --hint the conf field is map + return f.containing_type and [ff.name for ff in f.fields] == [ + "key", + "value", + ] + if isinstance(field, FieldDescriptor): - key = f"({label[field.label]})" + field.name - if field.type == FieldDescriptor.TYPE_MESSAGE: - value = Field.to_json(field.message_type) + if field.message_type: + # message_type is a Descriptor, check if it's a map + if is_map(field.message_type): + m = field.message_type + # treat key-value as map type, can't figure out custom type, no nested, so just generate here + return { + f"<{m.fields[0].name}>": f"<{m.fields[1].name}>", + "...": "...", + } + else: + # normal nested message + return Field.to_json(field.message_type) elif field.type == FieldDescriptor.TYPE_ENUM: - value = "/".join([n.name for n in field.enum_type.values]) + return "/".join([n.name for n in field.enum_type.values]) else: - value = Field.to_str(field.type) - if field.label == 3: - # json list style - return {key: [value, "..."]} - else: - return {key: value} - else: - # field is a message - if field.containing_type and [f.name for f in field.fields] == [ - "key", - "value", - ]: - # treat key-value as map type, can't figure out custom type - # TODO(hw): it's ok to pass a json list to proto map? - return {"k": "v", "...": "..."} + return f"<{Field.to_str(field.type)}>" + + elif isinstance(field, Descriptor): d = {} for f in field.fields: - d.update(Field.to_json(f)) + # each one is FieldDescriptor + # map is repeated too, but it's not a list + if f.label == 3 and not is_map(f.message_type): + # json list style + d[f"({label[f.label]})" + f.name] = [Field.to_json(f), "..."] + else: + d[f"({label[f.label]})" + f.name] = Field.to_json(f) return d + else: + raise ValueError(f"unknown type {type(field)}") diff --git a/python/openmldb_tool/diagnostic_tool/rpc.py b/python/openmldb_tool/diagnostic_tool/rpc.py index b0615ca6f81..8e3f8efc660 100644 --- a/python/openmldb_tool/diagnostic_tool/rpc.py +++ b/python/openmldb_tool/diagnostic_tool/rpc.py @@ -64,8 +64,8 @@ def rpc_exec(self, operation, field): ) return r.text - def hint(self, info): - if not info: + def hint(self, method): + if not method: # show service name and all rpc methods print(self.rpc_help()) return @@ -78,12 +78,17 @@ def hint(self, info): ) from .pb import DescriptorHelper - helper = DescriptorHelper(service) + ok, input_json = DescriptorHelper(service).get_input_json(method) + if not ok: + print(input_json) # if not ok, it's message + return # input message to json style - json_str = json.dumps(helper.get_input_json(info), indent=4) + json_str = json.dumps(input_json, indent=4) print( - f"You should input json like this, ignore round brackets in the key and double quotation marks in the value: --field '{json_str}'" + f"You should input json like this:\n --field '{json_str}'" ) + print("ignore round brackets in the key, e.g. (required)") + print('"<>" shows the data type, e.g. "" means you should set string') def search_in(self, typ, info): for item in typ: diff --git a/src/proto/tablet.proto b/src/proto/tablet.proto index 2944794b0d9..0938c9d965c 100755 --- a/src/proto/tablet.proto +++ b/src/proto/tablet.proto @@ -829,7 +829,7 @@ message BulkLoadInfoResponse { required uint32 key = 1; // TODO(hw): java will use int, cpp uses uint32. Not good? required uint32 value = 2; } - repeated MapFieldEntry ts_idx_map = 2; // TODO(hw): proto3 supports map + repeated MapFieldEntry ts_idx_map = 2; // TODO(hw): proto3 can build map in proto2 syntax } repeated Segment segment = 1; } From a3a52f71283441f9dcfc0454484370627c2cd40d Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Fri, 20 Oct 2023 11:32:28 +0800 Subject: [PATCH 5/6] fix test --- python/openmldb_tool/tests/inspect_test.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/python/openmldb_tool/tests/inspect_test.py b/python/openmldb_tool/tests/inspect_test.py index a548a76f81c..6f5ece39c05 100644 --- a/python/openmldb_tool/tests/inspect_test.py +++ b/python/openmldb_tool/tests/inspect_test.py @@ -49,7 +49,7 @@ def test_show(): tablets[0]: { "tid": 0, # actually not used in show_table_info "pid": 0, # not used in show_table_info - # "offset": 5, + "offset": 5, # check offset on tablet, not ns "mode": "kTableLeader", "state": "kTableNormal", # "is_expire": True, @@ -71,6 +71,7 @@ def test_show(): tablets[1]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[1], }, } @@ -112,16 +113,19 @@ def test_show(): tablets[0]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[0], }, tablets[1]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[1], }, tablets[2]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[2], }, } @@ -162,16 +166,19 @@ def test_show(): tablets[0]: { "mode": "kTableLeader", "state": "kTableNormal", + "offset": 0, "tablet": tablets[0], }, tablets[1]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[1], }, tablets[2]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[2], }, } @@ -212,11 +219,13 @@ def test_show(): tablets[1]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[1], }, tablets[2]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, "tablet": tablets[2], }, } @@ -274,16 +283,19 @@ def test_show(): tablets[0]: { "mode": "kTableFollower", "state": "kTableLoading", + "offset": 0, "tablet": tablets[0], }, tablets[1]: { "mode": "kTableFollower", "state": "kMakingSnapshot", + "offset": 0, "tablet": tablets[1], }, tablets[2]: { "mode": "kTableFollower", "state": "kSnapshotPaused", + "offset": 0, "tablet": tablets[2], }, }, @@ -291,10 +303,12 @@ def test_show(): tablets[1]: { "mode": "kTableFollower", "state": "kTableUndefined", + "offset": 0, }, tablets[2]: { "mode": "kTableFollower", "state": "kTableNormal", + "offset": 0, }, }, } @@ -329,6 +343,7 @@ def test_show(): tablets[0]: { "mode": "kTableLeader", "state": "kTableNormal", + "offset": 0, "tablet": tablets[0], } } From 95980fb732a9da8d8a571cf017cf223088f9e3de Mon Sep 17 00:00:00 2001 From: Huang Wei Date: Thu, 26 Oct 2023 12:03:31 +0800 Subject: [PATCH 6/6] fix --- docs/zh/maintain/diagnose.md | 16 +++++------- .../openmldb_tool/diagnostic_tool/diagnose.py | 22 +++++++--------- .../openmldb_tool/diagnostic_tool/inspect.py | 25 +++++++++++-------- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/docs/zh/maintain/diagnose.md b/docs/zh/maintain/diagnose.md index 3d8764c3114..cb5d7a30f74 100644 --- a/docs/zh/maintain/diagnose.md +++ b/docs/zh/maintain/diagnose.md @@ -35,24 +35,20 @@ Server Detail {server online/offline report} -Table Healthy Detail -summary: {total table cnt} -{unhealthy tables db.name} +Table Partitions Detail +tablet server order: {tablet ip -> idx} +{partition tables of unhealthy tables} +Example: +{a detailed description of partition table} Ops Detail > failed ops do not mean cluster is unhealthy, just for reference last one op(check time): {} -last 10 unfinished ops: +last 10 ops != finished: {op list} -Table Partitions Detail -tablet server order: {tablet ip -> idx} -{partition tables of unhealthy tables} -Example: -{a detailed description of partition table} - ================== Summary & Hint diff --git a/python/openmldb_tool/diagnostic_tool/diagnose.py b/python/openmldb_tool/diagnostic_tool/diagnose.py index 7a3b9761663..21ee2961421 100644 --- a/python/openmldb_tool/diagnostic_tool/diagnose.py +++ b/python/openmldb_tool/diagnostic_tool/diagnose.py @@ -31,7 +31,7 @@ import diagnostic_tool.server_checker as checker from diagnostic_tool.table_checker import TableChecker from diagnostic_tool.parser import LogParser -from .inspect import server_ins, table_ins, partition_ins, ops_ins, inspect_hint +from .inspect import server_ins, table_ins, partition_ins, ops_ins, ops_hint, inspect_hint from .rpc import RPC from absl import app @@ -105,19 +105,15 @@ def inspect(args): server_map = status_checker._get_components() offlines = server_ins(server_map) - # 2. table level - # we use `show table status` instead of partition inspection, cuz it's simple and quick - warn_tables = table_ins(connect) + # 3. ns ops level, but show only if has unhealthy tables, so hint later + last_one, should_warn, related_ops = ops_ins(connect) - # if has unhealthy tables, do partition and ops check, otherwise skip - hints = [] - if warn_tables: - # 3. ns ops - related_ops = ops_ins(connect) - # 4. partition level and get some hint about table - hints = partition_ins(server_map, related_ops) - - # 5. hint + # 2. partition level: show unhealthy tables and get some hints about table + hints = partition_ins(server_map, related_ops) + if hints: + # show 3 here + ops_hint(last_one, should_warn) + # 4. hint # let user know what to do # 1) start offline servers # 2) let user know the warning table is fatal or not, related ops, warn if offset is too large diff --git a/python/openmldb_tool/diagnostic_tool/inspect.py b/python/openmldb_tool/diagnostic_tool/inspect.py index c9cb626e40e..288f819bb78 100644 --- a/python/openmldb_tool/diagnostic_tool/inspect.py +++ b/python/openmldb_tool/diagnostic_tool/inspect.py @@ -347,8 +347,6 @@ def partition_ins(server_map, related_ops): def ops_ins(connect): # op sorted by id TODO: detail to show all include succ op? - print("\n\nOps Detail") - print("> failed ops do not mean cluster is unhealthy, just for reference") rs = connect.execfetch("show jobs from NameServer;") should_warn = [] from datetime import datetime @@ -360,21 +358,28 @@ def ops_ins(connect): op[4] = str(datetime.fromtimestamp(int(op[4]) / 1000)) if op[4] else "..." if op[2] != "FINISHED": should_warn.append(op) - # peek last one to let user know if cluster has tried to recover, or we should wait - print("last one op(check time): ", ops[-1]) - if not should_warn: - print("all nameserver ops are finished") - else: - print("last 10 unfinished ops:") - print(*should_warn[-10:], sep="\n") + recover_type = ["kRecoverTableOP", "kChangeLeaderOP", "kReAddReplicaOP", "kOfflineReplicaOP"] related_ops = [ op for op in should_warn if op[1] in recover_type and op[2] in ["Submitted", "RUNNING"] ] - return related_ops + return ops[-1] if ops else None, should_warn, related_ops +def ops_hint(last_one, should_warn): + print("\n\nOps Detail") + print("> failed ops do not mean cluster is unhealthy, just for reference") + # peek last one to let user know if cluster has tried to recover, or we should wait + if last_one: + print("last one op(check time): ", last_one) + else: + print("no ops in nameserver") + if not should_warn: + print("all nameserver ops are finished") + else: + print("last 10 ops != finished:") + print(*should_warn[-10:], sep="\n") def inspect_hint(server_hint, table_hints): print(