From b38d153689dcac18fba6aabfac320c4893c16a79 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Sep 2024 13:27:00 +0300 Subject: [PATCH 01/13] Fix whois Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 2153 ++++++++++++++++++++++++++++-- essentials/src/api/api_client.rs | 18 +- essentials/src/api/executor.rs | 20 +- essentials/src/types.rs | 1 + whois/Cargo.toml | 9 + whois/src/main.rs | 386 +++++- 6 files changed, 2405 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49c1fcdf..79598f51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "addr2line" version = "0.19.0" @@ -27,6 +33,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.7" @@ -129,6 +160,45 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -297,7 +367,20 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", +] + +[[package]] +name = "asynchronous-codec" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", ] [[package]] @@ -312,6 +395,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" +dependencies = [ + "http 0.2.9", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -333,6 +427,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base58" version = "0.2.0" @@ -351,6 +451,18 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "beef" version = "0.5.2" @@ -360,6 +472,15 @@ dependencies = [ "serde", ] +[[package]] +name = "binary-stream" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ef03ef225ea9a0b680a5926a58cb45d8eb56abf23d8a8b5c5dbc61235e2dac" +dependencies = [ + "thiserror", +] + [[package]] name = "bincode" version = "1.3.3" @@ -381,13 +502,13 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease", + "prettyplease 0.2.4", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -459,6 +580,30 @@ dependencies = [ "constant_time_eq 0.2.5", ] +[[package]] +name = "blake2s_simd" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" +dependencies = [ + "arrayref", + "arrayvec 0.7.4", + "constant_time_eq 0.3.1", +] + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec 0.7.4", + "cc", + "cfg-if", + "constant_time_eq 0.3.1", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -522,9 +667,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "bzip2-sys" @@ -539,12 +684,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -579,6 +725,19 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "cipher" version = "0.4.4" @@ -587,6 +746,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -631,7 +791,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -692,6 +852,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -704,6 +870,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -726,6 +898,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.6" @@ -815,6 +996,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -839,7 +1029,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -887,7 +1077,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -909,14 +1099,58 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core 0.20.8", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] name = "data-encoding" -version = "2.3.3" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + +[[package]] +name = "data-encoding-macro" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] [[package]] name = "deranged" @@ -935,7 +1169,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -977,6 +1211,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "downcast" version = "0.11.0" @@ -989,15 +1234,37 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ + "pkcs8", "signature", ] +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core", + "serde", + "sha2 0.10.8", + "subtle", + "zeroize", +] + [[package]] name = "ed25519-zebra" version = "4.0.3" @@ -1028,6 +1295,30 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -1167,6 +1458,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.9.0" @@ -1266,6 +1563,16 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-bounded" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b07bbbe7d7e78809544c6f718d875627addc73a7c3582447abc052cd3dc67e0" +dependencies = [ + "futures-timer", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1291,6 +1598,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", + "num_cpus", ] [[package]] @@ -1335,7 +1643,17 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", +] + +[[package]] +name = "futures-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" +dependencies = [ + "futures-io", + "rustls 0.21.11", ] [[package]] @@ -1401,9 +1719,20 @@ version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea1015b5a70616b688dc230cfe50c8af89d972cb132d5a622814d29773b10b9" dependencies = [ + "rand", "rand_core", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.27.2" @@ -1510,6 +1839,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.8.1" @@ -1541,21 +1879,41 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.9" +name = "home" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "bytes", - "fnv", - "itoa", + "windows-sys 0.52.0", ] [[package]] -name = "http" -version = "1.1.0" +name = "hostname" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -1650,6 +2008,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.5.0" @@ -1660,6 +2039,54 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "if-addrs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "if-watch" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b0422c86d7ce0e97169cc42e04ae643caf278874a7a3c87b8150a220dc7e1e" +dependencies = [ + "async-io 2.3.0", + "core-foundation", + "fnv", + "futures", + "if-addrs", + "ipnet", + "log", + "rtnetlink", + "system-configuration", + "tokio", + "windows", +] + +[[package]] +name = "igd-next" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4" +dependencies = [ + "async-trait", + "attohttpc", + "bytes", + "futures", + "http 0.2.9", + "hyper", + "log", + "rand", + "tokio", + "url", + "xmltree", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -1750,12 +2177,39 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "ip_network" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2f047c0a98b2f299aa5d6d7088443570faae494e9ae1305e48be000c9e0eb1" + +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.5", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +[[package]] +name = "ipnetwork" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355" +dependencies = [ + "serde", +] + [[package]] name = "is-terminal" version = "0.4.5" @@ -1786,6 +2240,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1794,9 +2257,9 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -1810,19 +2273,50 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpsee" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9579d0ca9fb30da026bac2f0f7d9576ec93489aeb7cd4971dd5b4617d82c79b2" +dependencies = [ + "jsonrpsee-client-transport 0.21.0", + "jsonrpsee-core 0.21.0", +] + [[package]] name = "jsonrpsee" version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4b0e68d9af1f066c06d6e2397583795b912d78537d7d907c561e82c13d69fa1" dependencies = [ - "jsonrpsee-client-transport", - "jsonrpsee-core", + "jsonrpsee-client-transport 0.22.4", + "jsonrpsee-core 0.22.4", "jsonrpsee-http-client", - "jsonrpsee-types", + "jsonrpsee-types 0.22.4", "jsonrpsee-ws-client", ] +[[package]] +name = "jsonrpsee-client-transport" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9f9ed46590a8d5681975f126e22531698211b926129a40a2db47cbca429220" +dependencies = [ + "futures-util", + "http 0.2.9", + "jsonrpsee-core 0.21.0", + "pin-project", + "rustls-native-certs 0.7.0", + "rustls-pki-types", + "soketto 0.7.1", + "thiserror", + "tokio", + "tokio-rustls 0.25.0", + "tokio-util", + "tracing", + "url", +] + [[package]] name = "jsonrpsee-client-transport" version = "0.22.4" @@ -1831,11 +2325,11 @@ checksum = "92f254f56af1ae84815b9b1325094743dcf05b92abb5e94da2e81a35cff0cada" dependencies = [ "futures-util", "http 0.2.9", - "jsonrpsee-core", + "jsonrpsee-core 0.22.4", "pin-project", "rustls-native-certs 0.7.0", "rustls-pki-types", - "soketto", + "soketto 0.7.1", "thiserror", "tokio", "tokio-rustls 0.25.0", @@ -1844,6 +2338,29 @@ dependencies = [ "url", ] +[[package]] +name = "jsonrpsee-core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776d009e2f591b78c038e0d053a796f94575d66ca4e77dd84bfc5e81419e436c" +dependencies = [ + "anyhow", + "async-lock 3.3.0", + "async-trait", + "beef", + "futures-timer", + "futures-util", + "jsonrpsee-types 0.21.0", + "pin-project", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "jsonrpsee-core" version = "0.22.4" @@ -1856,7 +2373,7 @@ dependencies = [ "futures-timer", "futures-util", "hyper", - "jsonrpsee-types", + "jsonrpsee-types 0.22.4", "pin-project", "rustc-hash", "serde", @@ -1876,8 +2393,8 @@ dependencies = [ "async-trait", "hyper", "hyper-rustls", - "jsonrpsee-core", - "jsonrpsee-types", + "jsonrpsee-core 0.22.4", + "jsonrpsee-types 0.22.4", "serde", "serde_json", "thiserror", @@ -1887,6 +2404,19 @@ dependencies = [ "url", ] +[[package]] +name = "jsonrpsee-types" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3266dfb045c9174b24c77c2dfe0084914bb23a6b2597d70c9dc6018392e1cd1b" +dependencies = [ + "anyhow", + "beef", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "jsonrpsee-types" version = "0.22.4" @@ -1904,57 +2434,476 @@ dependencies = [ name = "jsonrpsee-ws-client" version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f00abe918bf34b785f87459b9205790e5361a3f7437adb50e928dc243f27eb" +checksum = "32f00abe918bf34b785f87459b9205790e5361a3f7437adb50e928dc243f27eb" +dependencies = [ + "http 0.2.9", + "jsonrpsee-client-transport 0.22.4", + "jsonrpsee-core 0.22.4", + "jsonrpsee-types 0.22.4", + "url", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libp2p" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94495eb319a85b70a68b85e2389a95bb3555c71c49025b78c691a854a7e6464" +dependencies = [ + "bytes", + "either", + "futures", + "futures-timer", + "getrandom", + "instant", + "libp2p-allow-block-list", + "libp2p-connection-limits", + "libp2p-core", + "libp2p-dns", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-mdns", + "libp2p-metrics", + "libp2p-noise", + "libp2p-ping", + "libp2p-quic", + "libp2p-request-response", + "libp2p-swarm", + "libp2p-tcp", + "libp2p-upnp", + "libp2p-websocket", + "libp2p-yamux", + "multiaddr", + "pin-project", + "rw-stream-sink", + "thiserror", +] + +[[package]] +name = "libp2p-allow-block-list" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b46558c5c0bf99d3e2a1a38fd54ff5476ca66dd1737b12466a1824dd219311" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "void", +] + +[[package]] +name = "libp2p-connection-limits" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5107ad45cb20b2f6c3628c7b6014b996fcb13a88053f4569c872c6e30abf58" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "void", +] + +[[package]] +name = "libp2p-core" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd44289ab25e4c9230d9246c475a22241e301b23e8f4061d3bdef304a1a99713" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libp2p-identity", + "log", + "multiaddr", + "multihash", + "multistream-select", + "once_cell", + "parking_lot", + "pin-project", + "quick-protobuf", + "rand", + "rw-stream-sink", + "smallvec", + "thiserror", + "unsigned-varint", + "void", +] + +[[package]] +name = "libp2p-dns" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a18db73084b4da2871438f6239fef35190b05023de7656e877c18a00541a3b" +dependencies = [ + "async-trait", + "futures", + "libp2p-core", + "libp2p-identity", + "log", + "parking_lot", + "smallvec", + "trust-dns-resolver", +] + +[[package]] +name = "libp2p-identify" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a96638a0a176bec0a4bcaebc1afa8cf909b114477209d7456ade52c61cd9cd" +dependencies = [ + "asynchronous-codec", + "either", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "log", + "lru", + "quick-protobuf", + "quick-protobuf-codec", + "smallvec", + "thiserror", + "void", +] + +[[package]] +name = "libp2p-identity" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "999ec70441b2fb35355076726a6bc466c932e9bdc66f6a11c6c0aa17c7ab9be0" +dependencies = [ + "bs58", + "ed25519-dalek", + "hkdf", + "multihash", + "quick-protobuf", + "rand", + "sha2 0.10.8", + "thiserror", + "tracing", + "zeroize", +] + +[[package]] +name = "libp2p-kad" +version = "0.44.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ea178dabba6dde6ffc260a8e0452ccdc8f79becf544946692fff9d412fc29d" +dependencies = [ + "arrayvec 0.7.4", + "asynchronous-codec", + "bytes", + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "log", + "quick-protobuf", + "quick-protobuf-codec", + "rand", + "sha2 0.10.8", + "smallvec", + "thiserror", + "uint", + "unsigned-varint", + "void", +] + +[[package]] +name = "libp2p-mdns" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a2567c305232f5ef54185e9604579a894fd0674819402bb0ac0246da82f52a" +dependencies = [ + "data-encoding", + "futures", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "log", + "rand", + "smallvec", + "socket2 0.5.5", + "tokio", + "trust-dns-proto 0.22.0", + "void", +] + +[[package]] +name = "libp2p-metrics" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239ba7d28f8d0b5d77760dc6619c05c7e88e74ec8fbbe97f856f20a56745e620" +dependencies = [ + "instant", + "libp2p-core", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-ping", + "libp2p-swarm", + "once_cell", + "prometheus-client", +] + +[[package]] +name = "libp2p-noise" +version = "0.43.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2eeec39ad3ad0677551907dd304b2f13f17208ccebe333bef194076cd2e8921" +dependencies = [ + "bytes", + "curve25519-dalek", + "futures", + "libp2p-core", + "libp2p-identity", + "log", + "multiaddr", + "multihash", + "once_cell", + "quick-protobuf", + "rand", + "sha2 0.10.8", + "snow", + "static_assertions", + "thiserror", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "libp2p-ping" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e702d75cd0827dfa15f8fd92d15b9932abe38d10d21f47c50438c71dd1b5dae3" +dependencies = [ + "either", + "futures", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "log", + "rand", + "void", +] + +[[package]] +name = "libp2p-quic" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "130d451d83f21b81eb7b35b360bc7972aeafb15177784adc56528db082e6b927" +dependencies = [ + "bytes", + "futures", + "futures-timer", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-tls", + "log", + "parking_lot", + "quinn", + "rand", + "ring 0.16.20", + "rustls 0.21.11", + "socket2 0.5.5", + "thiserror", + "tokio", +] + +[[package]] +name = "libp2p-request-response" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e3b4d67870478db72bac87bfc260ee6641d0734e0e3e275798f089c3fecfd4" +dependencies = [ + "async-trait", + "futures", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "log", + "rand", + "smallvec", + "void", +] + +[[package]] +name = "libp2p-swarm" +version = "0.43.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580189e0074af847df90e75ef54f3f30059aedda37ea5a1659e8b9fca05c0141" dependencies = [ - "http 0.2.9", - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-types", - "url", + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm-derive", + "log", + "multistream-select", + "once_cell", + "rand", + "smallvec", + "tokio", + "void", ] [[package]] -name = "keccak" -version = "0.1.5" +name = "libp2p-swarm-derive" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "c4d5ec2a3df00c7836d7696c136274c9c59705bac69133253696a6c932cd1d74" dependencies = [ - "cpufeatures", + "heck 0.4.1", + "proc-macro-warning", + "proc-macro2", + "quote", + "syn 2.0.77", ] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "libp2p-tcp" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "b558dd40d1bcd1aaaed9de898e9ec6a436019ecc2420dd0016e712fbb61c5508" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libc", + "libp2p-core", + "libp2p-identity", + "log", + "socket2 0.5.5", + "tokio", +] [[package]] -name = "lazycell" -version = "1.3.0" +name = "libp2p-tls" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "8218d1d5482b122ccae396bbf38abdcb283ecc96fa54760e1dfd251f0546ac61" +dependencies = [ + "futures", + "futures-rustls", + "libp2p-core", + "libp2p-identity", + "rcgen", + "ring 0.16.20", + "rustls 0.21.11", + "rustls-webpki 0.101.7", + "thiserror", + "x509-parser", + "yasna", +] [[package]] -name = "libc" -version = "0.2.149" +name = "libp2p-upnp" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "82775a47b34f10f787ad3e2a22e2c1541e6ebef4fe9f28f3ac553921554c94c1" +dependencies = [ + "futures", + "futures-timer", + "igd-next", + "libp2p-core", + "libp2p-swarm", + "log", + "tokio", + "void", +] [[package]] -name = "libloading" -version = "0.7.4" +name = "libp2p-websocket" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "004ee9c4a4631435169aee6aad2f62e3984dc031c43b6d29731e8e82a016c538" dependencies = [ - "cfg-if", - "winapi", + "either", + "futures", + "futures-rustls", + "libp2p-core", + "libp2p-identity", + "log", + "parking_lot", + "pin-project-lite", + "rw-stream-sink", + "soketto 0.8.0", + "thiserror", + "url", + "webpki-roots", ] [[package]] -name = "libm" -version = "0.2.8" +name = "libp2p-yamux" +version = "0.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8eedcb62824c4300efb9cfd4e2a6edaf3ca097b9e68b36dabe45a44469fd6a85" +dependencies = [ + "futures", + "libp2p-core", + "log", + "thiserror", + "yamux", +] [[package]] name = "librocksdb-sys" @@ -2031,6 +2980,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -2074,6 +3029,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lz4" version = "1.24.0" @@ -2094,6 +3058,39 @@ dependencies = [ "libc", ] +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + +[[package]] +name = "maxminddb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2ba61113f9f7a9f0e87c519682d39c43a6f3f79c2cc42c3ba3dda83b1fa334" +dependencies = [ + "ipnetwork", + "log", + "memchr", + "serde", +] + [[package]] name = "memchr" version = "2.5.0" @@ -2209,6 +3206,111 @@ dependencies = [ "version_check", ] +[[package]] +name = "multiaddr" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b852bc02a2da5feed68cd14fa50d0774b92790a5bdbfa932a813926c8472070" +dependencies = [ + "arrayref", + "byteorder", + "data-encoding", + "libp2p-identity", + "multibase", + "multihash", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint", + "url", +] + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +dependencies = [ + "core2", + "unsigned-varint", +] + +[[package]] +name = "multihash-codetable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d815ecb3c8238d00647f8630ede7060a642c9f704761cd6082cb4028af6935" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest 0.10.7", + "multihash-derive", + "ripemd", + "serde", + "sha1", + "sha2 0.10.8", + "sha3", + "strobe-rs", +] + +[[package]] +name = "multihash-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890e72cb7396cb99ed98c1246a97b243cc16394470d94e0bc8b0c2c11d84290e" +dependencies = [ + "core2", + "multihash", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3958713ce794e12f7c6326fac9aa274c68d74c4881dd37b3e2662b8a2046bb19" +dependencies = [ + "proc-macro-crate 2.0.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.77", + "synstructure 0.13.1", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "multistream-select" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" +dependencies = [ + "bytes", + "futures", + "log", + "pin-project", + "smallvec", + "unsigned-varint", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2227,6 +3329,83 @@ dependencies = [ "tempfile", ] +[[package]] +name = "netlink-packet-core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" +dependencies = [ + "anyhow", + "byteorder", + "libc", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "byteorder", + "libc", + "netlink-packet-core", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-utils" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror", +] + +[[package]] +name = "netlink-proto" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" +dependencies = [ + "bytes", + "futures", + "log", + "netlink-packet-core", + "netlink-sys", + "thiserror", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" +dependencies = [ + "bytes", + "futures", + "libc", + "log", + "tokio", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "no-std-net" version = "0.6.0" @@ -2239,6 +3418,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -2255,6 +3440,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -2322,6 +3517,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -2378,6 +3582,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "owo-colors" version = "3.5.0" @@ -2481,12 +3691,31 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -2504,7 +3733,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -2530,6 +3759,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.26" @@ -2688,9 +3927,17 @@ dependencies = [ "color-eyre", "env_logger", "futures", + "hex", + "itertools 0.13.0", + "libp2p", "log", "polkadot-introspector-essentials", "polkadot-introspector-priority-channel", + "serde", + "serde-binary", + "serde_json", + "subp2p-explorer", + "subp2p-explorer-cli", "thiserror", "tokio", ] @@ -2736,6 +3983,18 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -2778,6 +4037,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "prettyplease" version = "0.2.4" @@ -2785,7 +4054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -2811,6 +4080,15 @@ dependencies = [ "toml_edit 0.19.8", ] +[[package]] +name = "proc-macro-crate" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +dependencies = [ + "toml_edit 0.20.7", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -2844,6 +4122,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-warning" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -2867,6 +4156,182 @@ dependencies = [ "thiserror", ] +[[package]] +name = "prometheus-client" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c99afa9a01501019ac3a14d71d9f94050346f55ca471ce90c799a15c58f61e2" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools 0.10.5", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease 0.1.25", + "prost 0.11.9", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost 0.11.9", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + +[[package]] +name = "quick-protobuf-codec" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ededb1cd78531627244d51dd0c7139fbe736c7d57af0092a76f0ffb2f56e98" +dependencies = [ + "asynchronous-codec", + "bytes", + "quick-protobuf", + "thiserror", + "unsigned-varint", +] + +[[package]] +name = "quinn" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +dependencies = [ + "bytes", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.21.11", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" +dependencies = [ + "bytes", + "rand", + "ring 0.16.20", + "rustc-hash", + "rustls 0.21.11", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" +dependencies = [ + "bytes", + "libc", + "socket2 0.5.5", + "tracing", + "windows-sys 0.48.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -2918,6 +4383,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89a3273d76a0cdbb09ecead2f6007d6d169211f3c9c6dd30cd8a2e8175adcb74" +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring 0.16.20", + "time", + "yasna", +] + [[package]] name = "reconnecting-jsonrpsee-ws-client" version = "0.4.2" @@ -2927,7 +4404,7 @@ dependencies = [ "cfg_aliases", "finito", "futures", - "jsonrpsee", + "jsonrpsee 0.22.4", "serde_json", "thiserror", "tokio", @@ -2954,6 +4431,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -3000,6 +4486,16 @@ dependencies = [ "winreg", ] +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + [[package]] name = "ring" version = "0.16.20" @@ -3029,6 +4525,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "rocksdb" version = "0.21.0" @@ -3039,6 +4544,21 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "rtnetlink" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" +dependencies = [ + "futures", + "log", + "netlink-packet-route", + "netlink-proto", + "nix", + "thiserror", + "tokio", +] + [[package]] name = "rustc-demangle" version = "0.1.22" @@ -3066,6 +4586,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.36.16" @@ -3221,6 +4750,17 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "rw-stream-sink" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" +dependencies = [ + "futures", + "pin-project", + "static_assertions", +] + [[package]] name = "ryu" version = "1.0.13" @@ -3339,7 +4879,7 @@ dependencies = [ "proc-macro2", "quote", "scale-info", - "syn 2.0.48", + "syn 2.0.77", "thiserror", ] @@ -3452,6 +4992,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-binary" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b550db407b83ed53a4f76f888bfd7441b685abc2c086e20fb47781a286940506" +dependencies = [ + "binary-stream", + "serde", + "thiserror", +] + [[package]] name = "serde_bytes" version = "0.11.15" @@ -3469,7 +5020,7 @@ checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -3604,6 +5155,9 @@ name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] [[package]] name = "siphasher" @@ -3697,7 +5251,7 @@ dependencies = [ "siphasher 1.0.0", "slab", "smallvec", - "soketto", + "soketto 0.7.1", "twox-hash", "wasmi", "x25519-dalek", @@ -3746,6 +5300,23 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +[[package]] +name = "snow" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85" +dependencies = [ + "aes-gcm", + "blake2", + "chacha20poly1305", + "curve25519-dalek", + "rand_core", + "ring 0.17.7", + "rustc_version", + "sha2 0.10.8", + "subtle", +] + [[package]] name = "socket2" version = "0.4.9" @@ -3770,15 +5341,30 @@ dependencies = [ name = "soketto" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +dependencies = [ + "base64 0.13.1", + "bytes", + "futures", + "httparse", + "log", + "rand", + "sha-1", +] + +[[package]] +name = "soketto" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37468c595637c10857701c990f93a40ce0e357cedb0953d1c26c8d8027f9bb53" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", "bytes", "futures", "httparse", "log", "rand", - "sha-1", + "sha1", ] [[package]] @@ -3807,12 +5393,49 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "ss58-registry" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43fce22ed1df64d04b262351c8f9d5c6da4f76f79f25ad15529792f893fad25d" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "serde", + "serde_json", + "unicode-xid", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strobe-rs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabb238a1cccccfa4c4fb703670c0d157e1256c1ba695abf1b93bd2bb14bab2d" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle", + "zeroize", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3844,7 +5467,76 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.48", + "syn 2.0.77", +] + +[[package]] +name = "subp2p-explorer" +version = "0.1.0" +dependencies = [ + "async-trait", + "asynchronous-codec", + "blake2", + "bs58", + "bytes", + "either", + "env_logger", + "fnv", + "futures", + "hex", + "libp2p", + "log", + "multiaddr", + "multihash-codetable", + "parity-scale-codec", + "pin-project", + "primitive-types", + "prost 0.12.6", + "prost-build", + "rand", + "schnorrkel", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "unsigned-varint", + "void", +] + +[[package]] +name = "subp2p-explorer-cli" +version = "0.1.0" +dependencies = [ + "async-trait", + "asynchronous-codec", + "bytes", + "clap", + "either", + "env_logger", + "fnv", + "futures", + "hex", + "ip_network", + "jsonrpsee 0.21.0", + "libp2p", + "log", + "maxminddb", + "multihash-codetable", + "parity-scale-codec", + "pin-project", + "primitive-types", + "rand", + "serde", + "serde_json", + "ss58-registry", + "subp2p-explorer", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "trust-dns-resolver", + "unsigned-varint", + "void", ] [[package]] @@ -3880,7 +5572,7 @@ dependencies = [ "hex", "impl-serde", "instant", - "jsonrpsee", + "jsonrpsee 0.22.4", "parity-scale-codec", "primitive-types", "reconnecting-jsonrpsee-ws-client", @@ -3911,14 +5603,14 @@ dependencies = [ "frame-metadata 16.0.0", "heck 0.5.0", "hex", - "jsonrpsee", + "jsonrpsee 0.22.4", "parity-scale-codec", "proc-macro2", "quote", "scale-info", "scale-typegen", "subxt-metadata", - "syn 2.0.48", + "syn 2.0.77", "thiserror", "tokio", ] @@ -3979,7 +5671,7 @@ dependencies = [ "quote", "scale-typegen", "subxt-codegen", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -4008,9 +5700,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -4023,6 +5715,29 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4095,7 +5810,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -4181,7 +5896,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -4271,6 +5986,17 @@ dependencies = [ "winnow 0.4.1", ] +[[package]] +name = "toml_edit" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow 0.5.25", +] + [[package]] name = "toml_edit" version = "0.21.1" @@ -4329,7 +6055,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -4352,15 +6078,105 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", + "smallvec", "thread_local", + "tracing", "tracing-core", + "tracing-log", +] + +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner 0.5.1", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "rand", + "smallvec", + "socket2 0.4.9", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-proto" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner 0.6.1", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", + "trust-dns-proto 0.23.2", ] [[package]] @@ -4459,6 +6275,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-xid" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" + [[package]] name = "universal-hash" version = "0.5.1" @@ -4469,6 +6291,18 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +dependencies = [ + "asynchronous-codec", + "bytes", + "futures-io", + "futures-util", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -4488,7 +6322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", ] @@ -4522,6 +6356,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "waker-fn" version = "1.1.1" @@ -4691,6 +6531,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.17", +] + +[[package]] +name = "widestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" + [[package]] name = "winapi" version = "0.3.9" @@ -4722,6 +6586,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -4752,7 +6635,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] @@ -4781,17 +6664,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -4817,9 +6700,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" @@ -4835,9 +6718,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" @@ -4853,9 +6736,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" @@ -4871,9 +6754,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" @@ -4889,9 +6772,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" @@ -4907,9 +6790,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" @@ -4925,9 +6808,9 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" @@ -4984,12 +6867,68 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "xml-rs" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "yamux" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed0164ae619f2dc144909a9f082187ebb5893693d8c0196e8085283ccd4b776" +dependencies = [ + "futures", + "log", + "nohash-hasher", + "parking_lot", + "pin-project", + "rand", + "static_assertions", +] + [[package]] name = "yap" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff4524214bc4629eba08d78ceb1d6507070cc0bcbbed23af74e19e6e924a24cf" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "zerocopy" version = "0.7.32" @@ -5007,7 +6946,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] @@ -5027,7 +6966,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.77", ] [[package]] diff --git a/essentials/src/api/api_client.rs b/essentials/src/api/api_client.rs index 0bba1e49..1e6d2c9c 100644 --- a/essentials/src/api/api_client.rs +++ b/essentials/src/api/api_client.rs @@ -23,7 +23,9 @@ use crate::{ polkadot_runtime_parachains::hrmp::HrmpChannel, }, }, - types::{AccountId32, BlockNumber, Header, InherentData, SessionKeys, SubxtHrmpChannel, Timestamp, H256}, + types::{ + AccountId32, BlockNumber, Header, InherentData, QueuedKeys, SessionKeys, SubxtHrmpChannel, Timestamp, H256, + }, }; use clap::ValueEnum; use std::collections::BTreeMap; @@ -187,6 +189,11 @@ impl> ApiClient { self.storage().at(hash).fetch(&addr).await } + pub async fn get_session_index_now(&self) -> Result, subxt::Error> { + let addr = polkadot::storage().session().current_index(); + self.storage().at_latest().await?.fetch(&addr).await + } + pub async fn get_session_account_keys(&self, session_index: u32) -> Result>, subxt::Error> { let addr = polkadot::storage().para_session_info().account_keys(session_index); self.storage().at_latest().await?.fetch(&addr).await @@ -197,6 +204,15 @@ impl> ApiClient { self.storage().at_latest().await?.fetch(&addr).await } + pub async fn get_session_queued_keys(&self, hash: Option) -> Result, subxt::Error> { + let addr = polkadot::storage().session().queued_keys(); + if let Some(hash) = hash { + self.storage().at(hash).fetch(&addr).await + } else { + self.storage().at_latest().await?.fetch(&addr).await + } + } + pub async fn get_inbound_outbound_hrmp_channels( &self, block_hash: H256, diff --git a/essentials/src/api/executor.rs b/essentials/src/api/executor.rs index 33613edf..3326ccf3 100644 --- a/essentials/src/api/executor.rs +++ b/essentials/src/api/executor.rs @@ -24,7 +24,7 @@ use crate::{ }, constants::MAX_MSG_QUEUE_SIZE, init::Shutdown, - metadata::polkadot_primitives, + metadata::{polkadot::session::storage::types::queued_keys::QueuedKeys, polkadot_primitives}, types::{ AccountId32, BlockNumber, ClaimQueue, CoreOccupied, Header, InboundOutBoundHrmpChannels, InherentData, SessionKeys, SubxtHrmpChannel, Timestamp, H256, @@ -63,7 +63,9 @@ pub enum Request { GetSessionIndex(H256), GetSessionAccountKeys(u32), GetSessionNextKeys(AccountId32), + GetSessionQueuedKeys(Option), GetInboundOutBoundHrmpChannels(H256, Vec), + GetSessionIndexNow, GetHostConfiguration, GetBestBlockSubscription, GetFinalizedBlockSubscription, @@ -96,6 +98,8 @@ enum Response { SessionAccountKeys(Option>), /// Session next keys for a validator SessionNextKeys(Option), + /// Session queued keys. + SessionQueuedKeys(Option), /// HRMP channels for given parachain (e.g. who are sending messages to us) InboundOutBoundHrmpChannels(InboundOutBoundHrmpChannels), /// The current host configuration @@ -229,6 +233,8 @@ impl RequestExecutorBackend { GetSessionAccountKeys(session_index) => SessionAccountKeys(client.get_session_account_keys(session_index).await?), GetSessionNextKeys(ref account) => SessionNextKeys(client.get_session_next_keys(account).await?), + GetSessionQueuedKeys(at) => SessionQueuedKeys(client.get_session_queued_keys(at).await?), + GetSessionIndexNow => SessionIndex(client.get_session_index_now().await?.unwrap_or_default()), GetInboundOutBoundHrmpChannels(hash, para_ids) => InboundOutBoundHrmpChannels(client.get_inbound_outbound_hrmp_channels(hash, para_ids).await?), GetHostConfiguration => HostConfiguration(DynamicHostConfiguration::new( @@ -410,6 +416,10 @@ impl RequestExecutor { wrap_backend_call!(self, url, GetSessionIndex, SessionIndex, hash) } + pub async fn get_session_index_now(&mut self, url: &str) -> color_eyre::Result { + wrap_backend_call!(self, url, GetSessionIndexNow, SessionIndex) + } + pub async fn get_session_account_keys( &mut self, url: &str, @@ -426,6 +436,14 @@ impl RequestExecutor { wrap_backend_call!(self, url, GetSessionNextKeys, SessionNextKeys, account) } + pub async fn get_session_queued_keys( + &mut self, + url: &str, + at: Option, + ) -> color_eyre::Result, RequestExecutorError> { + wrap_backend_call!(self, url, GetSessionQueuedKeys, SessionQueuedKeys, at) + } + pub async fn get_inbound_outbound_hrmp_channels( &mut self, url: &str, diff --git a/essentials/src/types.rs b/essentials/src/types.rs index 8a7a63d3..98384050 100644 --- a/essentials/src/types.rs +++ b/essentials/src/types.rs @@ -30,6 +30,7 @@ pub type AccountId32 = utils::AccountId32; pub type Header = SubstrateHeader; pub type Timestamp = u64; pub type SessionKeys = runtime::SessionKeys; +pub type QueuedKeys = crate::metadata::polkadot::session::storage::types::queued_keys::QueuedKeys; pub type SubxtCall = runtime::RuntimeCall; pub type ClaimQueue = BTreeMap>>; diff --git a/whois/Cargo.toml b/whois/Cargo.toml index efa16121..e584e44f 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -17,3 +17,12 @@ polkadot-introspector-essentials = { workspace = true } polkadot-introspector-priority-channel = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +serde = {workspace = true } +serde_json = {workspace = true } +itertools = "0.13" +serde-binary = "0.5" +hex = "0.4.3" +subp2p-explorer-cli = { path = "../../subp2p-explorer-use/cli"} +subp2p-explorer = { path = "../../subp2p-explorer-use/subp2p-explorer"} + +libp2p = { version = "0.52.0", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp", "tokio", "yamux", "websocket", "request-response"] } diff --git a/whois/src/main.rs b/whois/src/main.rs index 30f31f97..d65e93e0 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -15,20 +15,27 @@ // along with polkadot-introspector. If not, see . use clap::{Args, Parser, Subcommand}; +use futures::future; +use itertools::Itertools; +use libp2p::{Multiaddr, PeerId}; use polkadot_introspector_essentials::{ api::{ api_client::ApiClientMode, executor::{RequestExecutor, RequestExecutorError}, }, - consumer::{EventConsumerInit, EventStream}, init, - telemetry_feed::{AddedNode, TelemetryFeed}, - telemetry_subscription::{TelemetryEvent, TelemetrySubscription}, - types::{AccountId32, SessionKeys}, + types::{AccountId32, H256}, utils, }; -use polkadot_introspector_priority_channel::Receiver; -use std::str::FromStr; +use serde::{Deserialize, Serialize}; +use serde_binary::binary_stream::Endian; +use std::{ + collections::{HashMap, HashSet}, + fs::{self, File}, + io::Write, +}; +use subp2p_explorer::util::{crypto::sr25519, p2p::get_peer_id}; +use subp2p_explorer_cli::commands::authorities::PeerDetails; #[derive(Clone, Debug, Parser)] #[clap(author, version, about = "Simple telemetry feed")] @@ -38,9 +45,6 @@ struct TelemetryOptions { /// Web-Socket URLs of a relay chain node. #[clap(long)] pub ws: String, - /// Web-Socket URL of a telemetry backend - #[clap(long)] - pub feed: String, /// Name of a chain to connect #[clap(long)] pub chain: Option, @@ -48,38 +52,98 @@ struct TelemetryOptions { pub verbose: init::VerbosityOptions, #[clap(flatten)] pub retry: utils::RetryOptions, + #[clap(long)] + pub session_index: u32, + #[clap(long)] + pub queued_keys_at_block: Option, + #[clap(long)] + pub update_p2p_cache: bool, + /// Hex-encoded genesis hash of the chain. + /// + /// For example, "781e4046b4e8b5e83d33dde04b32e7cb5d43344b1f19b574f6d31cbbd99fe738" + #[clap(long, short)] + genesis: String, + /// Bootnodes of the chain, must contain a multiaddress together with the peer ID. + /// For example, "/ip4/127.0.0.1/tcp/30333/ws/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp". + #[clap(long, use_value_delimiter = true, value_parser)] + bootnodes: Vec, + /// The number of seconds the discovery process should run for. + #[clap(long, short, value_parser = parse_duration)] + timeout: std::time::Duration, + /// The address format name of the chain. + /// Used to display the SS58 address of the authorities. + /// + /// For example: + /// - "polkadot" for Polkadot + /// - "substrate" for Substrate + /// - "kusama" for Kusama + #[clap(long, short)] + address_format: String, +} + +fn parse_duration(arg: &str) -> Result { + let seconds = arg.parse()?; + Ok(std::time::Duration::from_secs(seconds)) } #[derive(Clone, Debug, Subcommand)] enum WhoisCommand { - Account(AccountOptions), - Session(SessionOptions), + ByAccount(AccountOptions), + ByValidatorIndex(SessionOptions), + ByAuthorityDiscovery(AuthorithyDiscoveryOptions), + ByPeerId(PeerIdOptions), + DumpAll, } #[derive(Clone, Debug, Args)] struct AccountOptions { /// SS58-formated validator's address - pub validator: AccountId32, + pub validators: Vec, +} + +#[derive(Clone, Debug, Args)] +struct AuthorithyDiscoveryOptions { + /// Authorithy Discovery in the hex format + pub authority_discovery: Vec, +} + +#[derive(Clone, Debug, Args)] +struct PeerIdOptions { + /// PeerId of the validator + pub peer_id: Vec, } #[derive(Clone, Debug, Args)] struct SessionOptions { - pub session_index: u32, - pub validator_index: usize, + pub validator_indices: Vec, } #[derive(Debug, thiserror::Error)] pub enum WhoisError { #[error("Validator's next keys not found")] NoNextKeys, + #[error("Could not fetch current session index")] + NoSessionIndex, #[error("Keys for the session with given index not found")] NoSessionKeys, #[error("Validator with given index not found")] NoValidator, #[error("Can't connect to relay chain")] SubxtError(RequestExecutorError), - #[error("Can't connect to telemetry feed")] - TelemetryError(color_eyre::Report), + #[error("Could not fetch para session account keys")] + NoParaSessionAccountKeys, + #[error("Could not fetch session queued keys")] + NoSessionQueuedKeys, + #[error("Validator index {0} is invalid in para session account keys")] + InvalidValidatorIndex(usize), + #[error("AuthorityDiscovery {0} is invalid in para session account keys")] + InvalidAuthorityDiscovery(String), + #[error("Could not find PeerId {0} in the p2p network cache, consider updating the cache if not updated")] + InvalidPeerId(PeerId), + #[error("Could not find authority key for peer id {0}")] + InvalidPeerIdNoAuthority(PeerId), + #[error("Invalid p2p cache, consider deleting and updating the cache at {0}")] + InvalidP2PCache(String), } struct Whois { @@ -93,77 +157,256 @@ impl Whois { async fn run( self, - consumer_config: EventConsumerInit, mut executor: RequestExecutor, ) -> color_eyre::Result>, WhoisError> { - let validator = match self.opts.command { - WhoisCommand::Account(v) => v.validator, - WhoisCommand::Session(v) => match executor.get_session_account_keys(&self.opts.ws, v.session_index).await { - Ok(Some(validators)) => match validators.get(v.validator_index) { - Some(v) => v.clone(), - None => return Err(WhoisError::NoValidator), - }, - Err(e) => return Err(WhoisError::SubxtError(e)), - _ => return Err(WhoisError::NoSessionKeys), - }, + let Ok(session_index_now) = executor.get_session_index_now(&self.opts.ws).await else { + return Err(WhoisError::NoSessionIndex) }; - let next_keys = match executor.get_session_next_keys(&self.opts.ws, validator.clone()).await { - Ok(Some(v)) => v, + + let para_session_account_keys = + match executor.get_session_account_keys(&self.opts.ws, self.opts.session_index).await { + Ok(Some(validators)) => validators, + Err(e) => return Err(WhoisError::SubxtError(e)), + _ => return Err(WhoisError::NoParaSessionAccountKeys), + }; + + let session_queued_keys = match executor + .get_session_queued_keys(&self.opts.ws, self.opts.queued_keys_at_block) + .await + { + Ok(Some(queued_keys)) => queued_keys, Err(e) => return Err(WhoisError::SubxtError(e)), - _ => return Err(WhoisError::NoNextKeys), + _ => return Err(WhoisError::NoSessionQueuedKeys), }; - let authority_key = get_authority_key(next_keys); - let consumer_channels: Vec> = consumer_config.into(); - let futures = consumer_channels - .into_iter() - .map(|c: Receiver| tokio::spawn(Self::watch(c, authority_key.clone(), validator.clone()))) - .collect(); + let network_cache = NetworkCache::build_cache(session_index_now, &self.opts).await?; + let network_cache_for_session = network_cache.get_closest_to_session(self.opts.session_index)?; - Ok(futures) - } + // A vector of (validator, validator_index) pairs representing the validator account + // and its index in para_session_account_keys. + let accounts_to_discover = match self.opts.command { + WhoisCommand::ByAccount(v) => v + .validators + .into_iter() + .map(|v| { + let index = para_session_account_keys.iter().position(|x| &v == x); + (v, index) + }) + .collect_vec(), + WhoisCommand::ByValidatorIndex(v) => v + .validator_indices + .into_iter() + .map(|validator_index| { + para_session_account_keys + .get(validator_index) + .cloned() + .ok_or(WhoisError::InvalidValidatorIndex(validator_index)) + .map(|account| (account, Some(validator_index))) + }) + .collect::, _>>()?, + WhoisCommand::ByAuthorityDiscovery(authority_discovery) => authority_discovery + .authority_discovery + .into_iter() + .map(|authority_discovery| { + let account = session_queued_keys + .iter() + .find(|(_, session_keys)| { + format!("0x{}", hex::encode(session_keys.authority_discovery.0 .0)) == authority_discovery + }) + .ok_or(WhoisError::InvalidAuthorityDiscovery(authority_discovery)); + account.map(|(account, _)| { + let validator_index = para_session_account_keys.iter().position(|x| account == x); + (account.clone(), validator_index) + }) + }) + .collect::, _>>()?, + WhoisCommand::ByPeerId(opts) => opts + .peer_id + .into_iter() + .map(|peer_id| { + let authority_key = network_cache_for_session + .get_authority_key_from_peer_id(peer_id) + .ok_or(WhoisError::InvalidPeerId(peer_id)); - async fn watch(update: Receiver, authority_key: AccountId32, validator: AccountId32) { - let mut count = 0_u32; - while let Ok(TelemetryEvent::NewMessage(message)) = update.recv().await { - if count > 0 { - clear_last_two_lines(); - } - count += 1; - println!("Looking for a validator {}...\n{} telemetry messages parsed, CTRL+C to exit", validator, count); - match message { - TelemetryFeed::AddedNode(node) => - if desired_node_id(&node, authority_key.clone()) { - println!("\n========================================\nValidator Node\n{}", node); - std::process::exit(0); - }, - _ => continue, + authority_key.and_then(|authority_key| { + let account_for_key = session_queued_keys + .iter() + .find(|(_, session_keys)| session_keys.authority_discovery.0 .0 == authority_key) + .ok_or(WhoisError::InvalidPeerIdNoAuthority(peer_id)); + + account_for_key.map(|(account, _)| { + let validator_index = para_session_account_keys.iter().position(|x| account == x); + (account.clone(), validator_index) + }) + }) + }) + .collect::, _>>()?, + WhoisCommand::DumpAll => para_session_account_keys + .into_iter() + .enumerate() + .map(|(validator_index, account)| (account, Some(validator_index))) + .collect_vec(), + }; + + println!( + "======================================== List of validators ========================================" + ); + + for (validator, validator_index) in accounts_to_discover { + let session_keys_for_validator = &session_queued_keys.iter().find(|(account, _)| account == &validator); + + if let Some((_, session_keys_for_validator)) = session_keys_for_validator { + let authorithy_discovery_key = session_keys_for_validator.authority_discovery.0 .0.clone(); + let (peer_details, info, peer_id) = network_cache_for_session.get_details(authorithy_discovery_key); + + println!( + "validator_index={:?}, account={:}, peer_id={:}, authorithy_id_discover=0x{:}, addresses={:?}, version={:?}", + validator_index.unwrap_or(usize::MAX), + validator, + peer_id, + hex::encode(authorithy_discovery_key), + peer_details.map(|details| details.addresses().clone()), + info, + ); + } else { + println!( + "validator_index={:?}, account={:}, no information could be found", + validator_index.unwrap_or(usize::MAX), + validator, + ); } + println!("=============================================================================================="); } + executor.close().await; + + Ok(vec![]) } } -fn desired_node_id(node: &AddedNode, authority_key: AccountId32) -> bool { - if node.details.validator.is_none() { - return false - } +#[derive(Serialize, Deserialize)] +struct NetworkCacheInfo { + pub peer_details: HashMap, PeerDetails>, + pub peer_versions: HashMap, String>, + pub authority_to_details: HashMap>, +} + +#[derive(Serialize, Deserialize, Default)] +struct NetworkCache { + session_to_network_info: HashMap, +} + +const DEFAULT_CACHE_DIR: &str = "~/.whois/cache"; - if let Ok(node_authority_key) = AccountId32::from_str(&node.details.validator.clone().unwrap()) { - if node_authority_key == authority_key { - return true +impl NetworkCache { + async fn build_cache(session_index_now: u32, opts: &TelemetryOptions) -> color_eyre::Result { + let mut update_cache = opts.update_p2p_cache; + let cache_path = format!("{}/{}/p2p_cache", DEFAULT_CACHE_DIR, opts.chain.clone().unwrap_or("any".to_string())); + println!("Using cache path: {}", cache_path); + let mut cache: NetworkCache = if let Ok(serialized_cache) = fs::read(cache_path.as_str()) { + serde_binary::from_vec(serialized_cache, Endian::Big) + .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))? + } else { + update_cache = true; + Default::default() + }; + + if update_cache { + println!("Discovering DHT authorithies, this may take a while..."); + let (authorithy_discovery, _) = subp2p_explorer_cli::commands::authorities::discover_authorities( + opts.ws.clone(), + opts.genesis.clone(), + opts.bootnodes.clone(), + opts.timeout, + opts.address_format.clone(), + Default::default(), + ) + .await + .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; + + let peer_details = authorithy_discovery.peer_details().clone(); + let peer_info = authorithy_discovery.peer_info().clone(); + let authority_to_details = authorithy_discovery.authority_to_details().clone(); + + let network_cache = NetworkCacheInfo { + peer_details: peer_details.into_iter().map(|(key, value)| (key.to_bytes(), value)).collect(), + peer_versions: peer_info + .into_iter() + .map(|(key, value)| (key.to_bytes(), value.agent_version)) + .collect(), + authority_to_details, + }; + + cache.session_to_network_info.insert(session_index_now, network_cache); } - }; - false -} + let serialized_cache = serde_binary::to_vec(&cache, Endian::Big) + .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; + let path = std::path::Path::new(cache_path.as_str()); + path.parent().map(|prefix| { + let _ = std::fs::create_dir_all(prefix).map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into())); + }); + let mut file = + File::create(cache_path.as_str()).map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; + file.write_all(serialized_cache.as_slice()) + .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; + Ok(cache) + } + + fn get_closest_to_session(&self, session_index: u32) -> color_eyre::Result<&NetworkCacheInfo, WhoisError> { + let closest_session_lower = self + .session_to_network_info + .keys() + .filter(|value| *value <= &session_index) + .max(); + + let closest_session_larger = self + .session_to_network_info + .keys() + .filter(|value| *value >= &session_index) + .min(); -fn get_authority_key(keys: SessionKeys) -> AccountId32 { - AccountId32::from(keys.grandpa.0 .0) + if let Some(closest_session_lower) = closest_session_lower { + Ok(self + .session_to_network_info + .get(closest_session_lower) + .expect("closest_session_lower is obtained from session_to_network_info; qed")) + } else if let Some(closest_session_larger) = closest_session_larger { + Ok(self + .session_to_network_info + .get(closest_session_larger) + .expect("closest_session_larger is obtained from session_to_network_info; qed")) + } else { + Err(WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into())) + } + } } -fn clear_last_two_lines() { - print!("\x1B[2A"); - print!("\x1B[0J"); +impl NetworkCacheInfo { + fn get_details(&self, authority_discovery_key: sr25519::PublicKey) -> (Option, String, PeerId) { + let Some(details) = self.authority_to_details.get(&authority_discovery_key) else { + return (Default::default(), Default::default(), PeerId::random()) + }; + + let Some(addr) = details.iter().next() else { + return (Default::default(), Default::default(), PeerId::random()) + }; + + let peer_id = get_peer_id(addr).unwrap_or(PeerId::random()); + let serialized_key = peer_id.to_bytes(); + ( + self.peer_details.get(&serialized_key).cloned(), + self.peer_versions + .get(&serialized_key) + .cloned() + .unwrap_or("unknown".to_string()), + peer_id, + ) + } + + fn get_authority_key_from_peer_id(&self, peer_id: PeerId) -> Option { + let serialized_key = peer_id.to_bytes(); + self.peer_details.get(&serialized_key).map(|x| x.authority_id()).cloned() + } } #[tokio::main] @@ -174,14 +417,11 @@ async fn main() -> color_eyre::Result<()> { let whois = Whois::new(opts.clone())?; let shutdown_tx = init::init_shutdown(); let executor = RequestExecutor::build(opts.ws.clone(), ApiClientMode::RPC, &opts.retry, &shutdown_tx).await?; - let mut sub = TelemetrySubscription::new(opts.ws.clone(), opts.chain.clone()); - let consumer_init = sub.create_consumer(); let mut futures = vec![]; - futures.extend(whois.run(consumer_init, executor).await?); - futures.extend(sub.run(&shutdown_tx).await?); + futures.extend(whois.run(executor).await?); - init::run(futures, &shutdown_tx).await?; + future::try_join_all(futures).await?; Ok(()) } From e0ac3534b09e88f782d125bf1ab84cf19e800fea Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Sep 2024 16:44:46 +0300 Subject: [PATCH 02/13] Modify whois Signed-off-by: Alexandru Gheorghe --- whois/README.md | 119 +++++++++++++++++++++++++++++++++++++++++++--- whois/src/main.rs | 47 +++++++++++++----- 2 files changed, 147 insertions(+), 19 deletions(-) diff --git a/whois/README.md b/whois/README.md index 8b35c2fd..0b686aa5 100644 --- a/whois/README.md +++ b/whois/README.md @@ -1,13 +1,120 @@ # polkadot-whois -Tracking of validators using on-chain and substrate telemetry data. The data can be queried by validator's `AccountId` or `(session_index, validator_index)`. +A tool for converting and querying information about validators using interchangeable unique identifiers, information about a validator can be queried using the following indentifiers: +- Validator index in `paraSessionInfo` pallet. +- The account key. +- The authority discovery id, provided in the hex format, it is the same format as what is displayed in `session.next_keys`. +- The peer id, the network peer id used for connecting to other nodes. -Usage: +It prints a list of information for each queried validator, G.g: +``` +validator_index=>, account=, peer_id=, authorithy_id_discover=0x, addresses=, version="POLKADOT_NODE_VERSION" +``` + +It uses the subp2p-explorer for querying the DHT information about each validator in the network. + +## Usage: +### Query by validator index +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + by-validator-index 295 293 +``` +### Query by peer id + +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + by-peer-id 12D3KooWEtD4vrMGsaAmETPx9VXuAu3UyFc7hL92x7ky3TJZwnT7 \ + 12D3KooWJJC4ACkC6fvsDuAvcgiebPMVNjMsfL3LtgmitAKhC39N + +``` + +### Query by authority discovery key + +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + by-authority-discovery 0x1cbbd313b592c053da0dc85fe0ae3c010d7bfc3c858a303418a1707846b6507d \ + 0xf2fe41ba85a8b16db8642126fd7d4bd3f9cf46c45e9852d528867f3d19474972 +``` + +### Query by account id +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + by-account 5D8DuA8a3obyN6ADUTJPUvx5yj8nnKmE1PqsKBQYCRoiqEak +``` +### Dump all +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + dump-all +``` +### Usage examples by network. + +## Polkadot +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rpc.polkadot.io:443 \ + --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ + --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ + --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + by-validator-index 295 293 +``` + +## Kusama +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://kusama-rpc.polkadot.io:443 \ + --genesis 0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe \ + --bootnodes /dns/kusama-bootnode-0.polkadot.io/tcp/30333/p2p/12D3Koo +WSueCPH3puP2PcvqPJdNaDNF3jMZjtJtDiSy35pWrbt5h \ + --address-format kusama --timeout 900 --chain kusama --session-index 42088 \ + by-validator-index 295 293 +``` + +## Westend +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://westend-rpc.polkadot.io:443 \ + --genesis 0xe143f23803ac50e8f6f8e62695d1ce9e4e1d68aa36c1cd2cfd15340213f3423e \ + --bootnodes /dns/westend-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWKer94o1REDPtAhjtYR4SdLehnSrN8PEhBnZm5NBoCrMC \ + --address-format substrate --timeout 120 --chain westend --session-index 42088 \ + by-validator-index 5 2 +``` +## Rococo +``` +cargo run --bin polkadot-whois -- \ + --ws=wss://rococo-rpc.polkadot.io:443 \ + --genesis 0x6408de7737c59c238890533af25896a2c20608d8b380bb01029acb392781063e \ + --bootnodes /dns/rococo-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWGikJMBmRiG5ofCqn8aijCijgfmZR5H9f53yUF3srm6Nm \ + --address-format substrate --timeout 120 --chain rococo --session-index 42088 \ + by-validator-index 295 293 +``` +## Paseo ``` -# by AccountId -cargo run --features=polkadot --bin polkadot-whois -- --ws=wss://rpc.polkadot.io:443 --feed=wss://feed.telemetry.polkadot.io/feed account 1497QNdycmxqMi3VJDxZDhaJh4s9tytr5RFWyrLcNse2xqPD +cargo run --bin polkadot-whois -- \ + --ws=wss://paseo-rpc.dwellir.com:443 \ + --genesis 0x77afd6190f1554ad45fd0d31aee62aacc33c6db0ea801129acb813f913e0764f \ + --bootnodes /dns/paseo.bootnode.amforc.com/tcp/29999/wss/p2p/12D3KooWSdf63rZjtGdeWXpQwQwPh8K8c22upcB3B1VmqW8rxrjw \ + --address-format substrate --timeout 120 --chain paseo --session-index 42088 \ + by-validator-index 11 12 -# by (session_index, validator_index) -cargo run --features=polkadot --bin polkadot-whois -- --ws=wss://rpc.polkadot.io:443 --feed=wss://feed.telemetry.polkadot.io/feed session 1046 12 ``` diff --git a/whois/src/main.rs b/whois/src/main.rs index d65e93e0..4468f97d 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -39,7 +39,7 @@ use subp2p_explorer_cli::commands::authorities::PeerDetails; #[derive(Clone, Debug, Parser)] #[clap(author, version, about = "Simple telemetry feed")] -struct TelemetryOptions { +struct WhoIsOptions { #[clap(subcommand)] command: WhoisCommand, /// Web-Socket URLs of a relay chain node. @@ -52,10 +52,15 @@ struct TelemetryOptions { pub verbose: init::VerbosityOptions, #[clap(flatten)] pub retry: utils::RetryOptions, + /// The session index used for computing the validator indicies. #[clap(long)] pub session_index: u32, + /// An optional block hash to fetch the queued keys at, otherwise we use the queued keys at the current block. #[clap(long)] pub queued_keys_at_block: Option, + /// Tells if we should update the p2p cache. + /// + /// Note building the p2p cache takes around 10 to 15 minutes #[clap(long)] pub update_p2p_cache: bool, /// Hex-encoded genesis hash of the chain. @@ -77,6 +82,8 @@ struct TelemetryOptions { /// - "polkadot" for Polkadot /// - "substrate" for Substrate /// - "kusama" for Kusama + /// + /// Used to display the SS58 address of the authorities. #[clap(long, short)] address_format: String, } @@ -88,10 +95,15 @@ fn parse_duration(arg: &str) -> Result, } @@ -147,11 +160,11 @@ pub enum WhoisError { } struct Whois { - opts: TelemetryOptions, + opts: WhoIsOptions, } impl Whois { - fn new(opts: TelemetryOptions) -> color_eyre::Result { + fn new(opts: WhoIsOptions) -> color_eyre::Result { Ok(Self { opts }) } @@ -283,24 +296,31 @@ impl Whois { } } +const DEFAULT_CACHE_DIR: &str = "whois_p2pcache"; + +// Information about the p2p network at a given session index. #[derive(Serialize, Deserialize)] -struct NetworkCacheInfo { +struct PerSessionNetworkCache { pub peer_details: HashMap, PeerDetails>, pub peer_versions: HashMap, String>, pub authority_to_details: HashMap>, } +// A cache of the p2p network information for different sessions. #[derive(Serialize, Deserialize, Default)] struct NetworkCache { - session_to_network_info: HashMap, + session_to_network_info: HashMap, } -const DEFAULT_CACHE_DIR: &str = "~/.whois/cache"; - impl NetworkCache { - async fn build_cache(session_index_now: u32, opts: &TelemetryOptions) -> color_eyre::Result { + // Build the p2p network cache. + // + // Because build the p2p cache takes around 10 to 15 minutes, + // we only build the cache if the update_p2p_cache flag is set + // or if the cache does not exist. + async fn build_cache(session_index_now: u32, opts: &WhoIsOptions) -> color_eyre::Result { let mut update_cache = opts.update_p2p_cache; - let cache_path = format!("{}/{}/p2p_cache", DEFAULT_CACHE_DIR, opts.chain.clone().unwrap_or("any".to_string())); + let cache_path = format!("{}/{}", DEFAULT_CACHE_DIR, opts.chain.clone().unwrap_or("any".to_string())); println!("Using cache path: {}", cache_path); let mut cache: NetworkCache = if let Ok(serialized_cache) = fs::read(cache_path.as_str()) { serde_binary::from_vec(serialized_cache, Endian::Big) @@ -327,7 +347,7 @@ impl NetworkCache { let peer_info = authorithy_discovery.peer_info().clone(); let authority_to_details = authorithy_discovery.authority_to_details().clone(); - let network_cache = NetworkCacheInfo { + let network_cache = PerSessionNetworkCache { peer_details: peer_details.into_iter().map(|(key, value)| (key.to_bytes(), value)).collect(), peer_versions: peer_info .into_iter() @@ -352,7 +372,8 @@ impl NetworkCache { Ok(cache) } - fn get_closest_to_session(&self, session_index: u32) -> color_eyre::Result<&NetworkCacheInfo, WhoisError> { + /// Get the p2p network cache closest to the given session index. + fn get_closest_to_session(&self, session_index: u32) -> color_eyre::Result<&PerSessionNetworkCache, WhoisError> { let closest_session_lower = self .session_to_network_info .keys() @@ -381,7 +402,7 @@ impl NetworkCache { } } -impl NetworkCacheInfo { +impl PerSessionNetworkCache { fn get_details(&self, authority_discovery_key: sr25519::PublicKey) -> (Option, String, PeerId) { let Some(details) = self.authority_to_details.get(&authority_discovery_key) else { return (Default::default(), Default::default(), PeerId::random()) @@ -411,7 +432,7 @@ impl NetworkCacheInfo { #[tokio::main] async fn main() -> color_eyre::Result<()> { - let opts = TelemetryOptions::parse(); + let opts = WhoIsOptions::parse(); init::init_cli(&opts.verbose)?; let whois = Whois::new(opts.clone())?; From f64227a8e13dcf294c7eda6353a099096622eb5d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Sep 2024 16:53:45 +0300 Subject: [PATCH 03/13] Update Cargo.* Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 2 ++ whois/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 205cf819..e619c749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5473,6 +5473,7 @@ dependencies = [ [[package]] name = "subp2p-explorer" version = "0.1.0" +source = "git+https://github.com/alexggh/subp2p-explorer?branch=alexggh/make-subp2p-usable-as-library#f7fa5d0474ca348f76f3f86e452e342bcaeb2b39" dependencies = [ "async-trait", "asynchronous-codec", @@ -5506,6 +5507,7 @@ dependencies = [ [[package]] name = "subp2p-explorer-cli" version = "0.1.0" +source = "git+https://github.com/alexggh/subp2p-explorer?branch=alexggh/make-subp2p-usable-as-library#f7fa5d0474ca348f76f3f86e452e342bcaeb2b39" dependencies = [ "async-trait", "asynchronous-codec", diff --git a/whois/Cargo.toml b/whois/Cargo.toml index e584e44f..d6a6b3eb 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -22,7 +22,7 @@ serde_json = {workspace = true } itertools = "0.13" serde-binary = "0.5" hex = "0.4.3" -subp2p-explorer-cli = { path = "../../subp2p-explorer-use/cli"} -subp2p-explorer = { path = "../../subp2p-explorer-use/subp2p-explorer"} +subp2p-explorer-cli = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} +subp2p-explorer = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} libp2p = { version = "0.52.0", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp", "tokio", "yamux", "websocket", "request-response"] } From 09303ceb414ab65029f38deecc7637169d87a591 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Sep 2024 19:23:54 +0300 Subject: [PATCH 04/13] Correctly handle unknown validators Signed-off-by: Alexandru Gheorghe --- whois/src/main.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/whois/src/main.rs b/whois/src/main.rs index 4468f97d..8e29f31b 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -276,7 +276,7 @@ impl Whois { "validator_index={:?}, account={:}, peer_id={:}, authorithy_id_discover=0x{:}, addresses={:?}, version={:?}", validator_index.unwrap_or(usize::MAX), validator, - peer_id, + peer_id.map(|peer_id| peer_id.to_string()).unwrap_or("unknown".to_string()), hex::encode(authorithy_discovery_key), peer_details.map(|details| details.addresses().clone()), info, @@ -403,17 +403,18 @@ impl NetworkCache { } impl PerSessionNetworkCache { - fn get_details(&self, authority_discovery_key: sr25519::PublicKey) -> (Option, String, PeerId) { + fn get_details( + &self, + authority_discovery_key: sr25519::PublicKey, + ) -> (Option, String, Option) { let Some(details) = self.authority_to_details.get(&authority_discovery_key) else { - return (Default::default(), Default::default(), PeerId::random()) + return (Default::default(), Default::default(), None) }; - let Some(addr) = details.iter().next() else { - return (Default::default(), Default::default(), PeerId::random()) - }; + let Some(addr) = details.iter().next() else { return (Default::default(), Default::default(), None) }; - let peer_id = get_peer_id(addr).unwrap_or(PeerId::random()); - let serialized_key = peer_id.to_bytes(); + let peer_id = get_peer_id(addr); + let serialized_key = peer_id.map(|peer_d| peer_d.to_bytes()).unwrap_or_default(); ( self.peer_details.get(&serialized_key).cloned(), self.peer_versions From 007f0fe14bda1d278108dfbcc36b905e9bf93ff2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 11:57:46 +0300 Subject: [PATCH 05/13] Add error on incompatible data Signed-off-by: Alexandru Gheorghe --- whois/src/main.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/whois/src/main.rs b/whois/src/main.rs index 8e29f31b..ef9eceb2 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -137,6 +137,8 @@ pub enum WhoisError { NoNextKeys, #[error("Could not fetch current session index")] NoSessionIndex, + #[error("Invalid session index, session can not be older than {0}")] + InvalidSessionIndex(u32), #[error("Keys for the session with given index not found")] NoSessionKeys, #[error("Validator with given index not found")] @@ -163,6 +165,9 @@ struct Whois { opts: WhoIsOptions, } +// The maximum number of para sessions a node stores before starting to proun +const NUMBER_OF_STORED_SESSIONS: u32 = 6; + impl Whois { fn new(opts: WhoIsOptions) -> color_eyre::Result { Ok(Self { opts }) @@ -176,6 +181,12 @@ impl Whois { return Err(WhoisError::NoSessionIndex) }; + if session_index_now < self.opts.session_index || + session_index_now - self.opts.session_index > NUMBER_OF_STORED_SESSIONS + { + return Err(WhoisError::InvalidSessionIndex(session_index_now - NUMBER_OF_STORED_SESSIONS)); + } + let para_session_account_keys = match executor.get_session_account_keys(&self.opts.ws, self.opts.session_index).await { Ok(Some(validators)) => validators, From 5dbfea4368f7d919eb5b8308aeef0e3c5ba86889 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 12:58:41 +0300 Subject: [PATCH 06/13] Address review feedback Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 11 +---------- Cargo.toml | 1 + whois/Cargo.toml | 8 ++++---- whois/README.md | 10 ---------- whois/src/main.rs | 29 +++++++++++++++++------------ 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2459d0b..1fdea0a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2240,15 +2240,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.6" @@ -3928,7 +3919,7 @@ dependencies = [ "env_logger", "futures", "hex", - "itertools 0.13.0", + "itertools 0.12.1", "libp2p", "log", "polkadot-introspector-essentials", diff --git a/Cargo.toml b/Cargo.toml index 99c15e3a..bf653b61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ rasciigraph = "0.2.0" reqwest = { version = "0.11.27" } rocksdb = "0.21.0" serde = "1.0.210" +serde-binary = "0.5" serde_bytes = "0.11.15" serde_derive = "1.0.138" serde_json = "1.0.128" diff --git a/whois/Cargo.toml b/whois/Cargo.toml index d6a6b3eb..ab0c2461 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -19,10 +19,10 @@ thiserror = { workspace = true } tokio = { workspace = true } serde = {workspace = true } serde_json = {workspace = true } -itertools = "0.13" -serde-binary = "0.5" -hex = "0.4.3" +itertools = {workspace = true } +serde-binary = {workspace = true } +hex = {workspace = true } subp2p-explorer-cli = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} subp2p-explorer = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} -libp2p = { version = "0.52.0", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp", "tokio", "yamux", "websocket", "request-response"] } +libp2p = { version = "0.52.4", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp", "tokio", "yamux", "websocket", "request-response"] } diff --git a/whois/README.md b/whois/README.md index 0b686aa5..a6cd0935 100644 --- a/whois/README.md +++ b/whois/README.md @@ -19,7 +19,6 @@ It uses the subp2p-explorer for querying the DHT information about each validato ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ by-validator-index 295 293 @@ -29,7 +28,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ by-peer-id 12D3KooWEtD4vrMGsaAmETPx9VXuAu3UyFc7hL92x7ky3TJZwnT7 \ @@ -42,7 +40,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ by-authority-discovery 0x1cbbd313b592c053da0dc85fe0ae3c010d7bfc3c858a303418a1707846b6507d \ @@ -53,7 +50,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ by-account 5D8DuA8a3obyN6ADUTJPUvx5yj8nnKmE1PqsKBQYCRoiqEak @@ -62,7 +58,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ dump-all @@ -73,7 +68,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ - --genesis 0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ by-validator-index 295 293 @@ -83,7 +77,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://kusama-rpc.polkadot.io:443 \ - --genesis 0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe \ --bootnodes /dns/kusama-bootnode-0.polkadot.io/tcp/30333/p2p/12D3Koo WSueCPH3puP2PcvqPJdNaDNF3jMZjtJtDiSy35pWrbt5h \ --address-format kusama --timeout 900 --chain kusama --session-index 42088 \ @@ -94,7 +87,6 @@ WSueCPH3puP2PcvqPJdNaDNF3jMZjtJtDiSy35pWrbt5h \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://westend-rpc.polkadot.io:443 \ - --genesis 0xe143f23803ac50e8f6f8e62695d1ce9e4e1d68aa36c1cd2cfd15340213f3423e \ --bootnodes /dns/westend-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWKer94o1REDPtAhjtYR4SdLehnSrN8PEhBnZm5NBoCrMC \ --address-format substrate --timeout 120 --chain westend --session-index 42088 \ by-validator-index 5 2 @@ -103,7 +95,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://rococo-rpc.polkadot.io:443 \ - --genesis 0x6408de7737c59c238890533af25896a2c20608d8b380bb01029acb392781063e \ --bootnodes /dns/rococo-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWGikJMBmRiG5ofCqn8aijCijgfmZR5H9f53yUF3srm6Nm \ --address-format substrate --timeout 120 --chain rococo --session-index 42088 \ by-validator-index 295 293 @@ -112,7 +103,6 @@ cargo run --bin polkadot-whois -- \ ``` cargo run --bin polkadot-whois -- \ --ws=wss://paseo-rpc.dwellir.com:443 \ - --genesis 0x77afd6190f1554ad45fd0d31aee62aacc33c6db0ea801129acb813f913e0764f \ --bootnodes /dns/paseo.bootnode.amforc.com/tcp/29999/wss/p2p/12D3KooWSdf63rZjtGdeWXpQwQwPh8K8c22upcB3B1VmqW8rxrjw \ --address-format substrate --timeout 120 --chain paseo --session-index 42088 \ by-validator-index 11 12 diff --git a/whois/src/main.rs b/whois/src/main.rs index ef9eceb2..e16df75b 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -38,7 +38,7 @@ use subp2p_explorer::util::{crypto::sr25519, p2p::get_peer_id}; use subp2p_explorer_cli::commands::authorities::PeerDetails; #[derive(Clone, Debug, Parser)] -#[clap(author, version, about = "Simple telemetry feed")] +#[clap(author, version, about = "Simple command to query polkadot validator identities")] struct WhoIsOptions { #[clap(subcommand)] command: WhoisCommand, @@ -63,11 +63,6 @@ struct WhoIsOptions { /// Note building the p2p cache takes around 10 to 15 minutes #[clap(long)] pub update_p2p_cache: bool, - /// Hex-encoded genesis hash of the chain. - /// - /// For example, "781e4046b4e8b5e83d33dde04b32e7cb5d43344b1f19b574f6d31cbbd99fe738" - #[clap(long, short)] - genesis: String, /// Bootnodes of the chain, must contain a multiaddress together with the peer ID. /// For example, "/ip4/127.0.0.1/tcp/30333/ws/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp". #[clap(long, use_value_delimiter = true, value_parser)] @@ -137,6 +132,8 @@ pub enum WhoisError { NoNextKeys, #[error("Could not fetch current session index")] NoSessionIndex, + #[error("Could not determine the genesis hash")] + NoGenesisHash, #[error("Invalid session index, session can not be older than {0}")] InvalidSessionIndex(u32), #[error("Keys for the session with given index not found")] @@ -177,6 +174,10 @@ impl Whois { self, mut executor: RequestExecutor, ) -> color_eyre::Result>, WhoisError> { + let Ok(Some(genesis_hash)) = executor.get_block_hash(&self.opts.ws, Some(0)).await else { + return Err(WhoisError::NoGenesisHash) + }; + let Ok(session_index_now) = executor.get_session_index_now(&self.opts.ws).await else { return Err(WhoisError::NoSessionIndex) }; @@ -203,7 +204,7 @@ impl Whois { _ => return Err(WhoisError::NoSessionQueuedKeys), }; - let network_cache = NetworkCache::build_cache(session_index_now, &self.opts).await?; + let network_cache = NetworkCache::build_cache(session_index_now, genesis_hash, &self.opts).await?; let network_cache_for_session = network_cache.get_closest_to_session(self.opts.session_index)?; // A vector of (validator, validator_index) pairs representing the validator account @@ -235,7 +236,7 @@ impl Whois { let account = session_queued_keys .iter() .find(|(_, session_keys)| { - format!("0x{}", hex::encode(session_keys.authority_discovery.0 .0)) == authority_discovery + format!("0x{}", hex::encode(session_keys.authority_discovery.0)) == authority_discovery }) .ok_or(WhoisError::InvalidAuthorityDiscovery(authority_discovery)); account.map(|(account, _)| { @@ -255,7 +256,7 @@ impl Whois { authority_key.and_then(|authority_key| { let account_for_key = session_queued_keys .iter() - .find(|(_, session_keys)| session_keys.authority_discovery.0 .0 == authority_key) + .find(|(_, session_keys)| session_keys.authority_discovery.0 == authority_key) .ok_or(WhoisError::InvalidPeerIdNoAuthority(peer_id)); account_for_key.map(|(account, _)| { @@ -280,7 +281,7 @@ impl Whois { let session_keys_for_validator = &session_queued_keys.iter().find(|(account, _)| account == &validator); if let Some((_, session_keys_for_validator)) = session_keys_for_validator { - let authorithy_discovery_key = session_keys_for_validator.authority_discovery.0 .0.clone(); + let authorithy_discovery_key = session_keys_for_validator.authority_discovery.0.clone(); let (peer_details, info, peer_id) = network_cache_for_session.get_details(authorithy_discovery_key); println!( @@ -329,7 +330,11 @@ impl NetworkCache { // Because build the p2p cache takes around 10 to 15 minutes, // we only build the cache if the update_p2p_cache flag is set // or if the cache does not exist. - async fn build_cache(session_index_now: u32, opts: &WhoIsOptions) -> color_eyre::Result { + async fn build_cache( + session_index_now: u32, + genesis_hash: H256, + opts: &WhoIsOptions, + ) -> color_eyre::Result { let mut update_cache = opts.update_p2p_cache; let cache_path = format!("{}/{}", DEFAULT_CACHE_DIR, opts.chain.clone().unwrap_or("any".to_string())); println!("Using cache path: {}", cache_path); @@ -345,7 +350,7 @@ impl NetworkCache { println!("Discovering DHT authorithies, this may take a while..."); let (authorithy_discovery, _) = subp2p_explorer_cli::commands::authorities::discover_authorities( opts.ws.clone(), - opts.genesis.clone(), + format!("{:?}", genesis_hash), opts.bootnodes.clone(), opts.timeout, opts.address_format.clone(), From f431dd8d400dbc1bcabdda123c151c7e38ed2577 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 11:20:50 +0300 Subject: [PATCH 07/13] Make cli a bit more ergnomic Signed-off-by: Alexandru Gheorghe --- .gitignore | 1 + Cargo.lock | 1 + Cargo.toml | 1 + essentials/src/api/api_client.rs | 4 +++ essentials/src/api/executor.rs | 8 ++++++ whois/Cargo.toml | 1 + whois/README.md | 18 ++++++------ whois/src/main.rs | 48 +++++++++++++++++--------------- 8 files changed, 51 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 06f5fec0..9c260320 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target /.idea /.vscode +/whois_p2pcache \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 1fdea0a8..1828b6c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3927,6 +3927,7 @@ dependencies = [ "serde", "serde-binary", "serde_json", + "ss58-registry", "subp2p-explorer", "subp2p-explorer-cli", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index bf653b61..0f45217a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ serde_derive = "1.0.138" serde_json = "1.0.128" serde_urlencoded = "0.7.1" snap = "1.1.1" +ss58-registry = { version = "1.34.0", default-features = false } strum = { version = "0.25.0", features = ["derive"] } subxt = { default-features = false, features = [ "jsonrpsee", diff --git a/essentials/src/api/api_client.rs b/essentials/src/api/api_client.rs index 1e6d2c9c..85d7837e 100644 --- a/essentials/src/api/api_client.rs +++ b/essentials/src/api/api_client.rs @@ -298,6 +298,10 @@ impl> ApiClient { self.legacy_rpc_methods.chain_get_block_hash(maybe_block_number).await } + pub async fn legacy_get_chain_name(&self) -> Result { + self.legacy_rpc_methods.system_chain().await + } + pub async fn stream_best_block_headers(&self) -> Result { self.client.backend().stream_best_block_headers().await } diff --git a/essentials/src/api/executor.rs b/essentials/src/api/executor.rs index 3326ccf3..e9c9a734 100644 --- a/essentials/src/api/executor.rs +++ b/essentials/src/api/executor.rs @@ -69,6 +69,7 @@ pub enum Request { GetHostConfiguration, GetBestBlockSubscription, GetFinalizedBlockSubscription, + GetChainName, } /// Response types for APIs. @@ -106,6 +107,8 @@ enum Response { HostConfiguration(DynamicHostConfiguration), /// Chain subscription ChainSubscription(HeaderStream), + /// Chain name + ChainName(String), } #[derive(Debug, Error)] @@ -209,6 +212,7 @@ impl RequestExecutorBackend { }, GetBlockNumber(maybe_hash) => BlockNumber(client.get_block_number(maybe_hash).await?), GetBlockHash(maybe_block_number) => MaybeBlockHash(client.legacy_get_block_hash(maybe_block_number).await?), + GetChainName => ChainName(client.legacy_get_chain_name().await?), GetEvents(hash) => MaybeEvents(Some(client.get_events(hash).await?)), ExtractParaInherent(maybe_hash) => ParaInherentData(client.extract_parainherent(maybe_hash).await?), GetClaimQueue(hash) => { @@ -372,6 +376,10 @@ impl RequestExecutor { wrap_backend_call!(self, url, GetBlockHash, MaybeBlockHash, maybe_block_number) } + pub async fn get_chain_name(&mut self, url: &str) -> color_eyre::Result { + wrap_backend_call!(self, url, GetChainName, ChainName) + } + pub async fn get_events( &mut self, url: &str, diff --git a/whois/Cargo.toml b/whois/Cargo.toml index ab0c2461..9467ab73 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -19,6 +19,7 @@ thiserror = { workspace = true } tokio = { workspace = true } serde = {workspace = true } serde_json = {workspace = true } +ss58-registry = {workspace = true } itertools = {workspace = true } serde-binary = {workspace = true } hex = {workspace = true } diff --git a/whois/README.md b/whois/README.md index a6cd0935..253703a8 100644 --- a/whois/README.md +++ b/whois/README.md @@ -20,7 +20,7 @@ It uses the subp2p-explorer for querying the DHT information about each validato cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ by-validator-index 295 293 ``` ### Query by peer id @@ -29,7 +29,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ by-peer-id 12D3KooWEtD4vrMGsaAmETPx9VXuAu3UyFc7hL92x7ky3TJZwnT7 \ 12D3KooWJJC4ACkC6fvsDuAvcgiebPMVNjMsfL3LtgmitAKhC39N @@ -41,7 +41,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ by-authority-discovery 0x1cbbd313b592c053da0dc85fe0ae3c010d7bfc3c858a303418a1707846b6507d \ 0xf2fe41ba85a8b16db8642126fd7d4bd3f9cf46c45e9852d528867f3d19474972 ``` @@ -51,7 +51,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ by-account 5D8DuA8a3obyN6ADUTJPUvx5yj8nnKmE1PqsKBQYCRoiqEak ``` ### Dump all @@ -59,7 +59,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ dump-all ``` ### Usage examples by network. @@ -69,7 +69,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rpc.polkadot.io:443 \ --bootnodes /dns/polkadot-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWSz8r2WyCdsfWHgPyvD8GKQdJ1UAiRmrcrs8sQB3fe2KU \ - --address-format polkadot --timeout 900 --chain polkadot --session-index 9442 \ + --session-index 9442 \ by-validator-index 295 293 ``` @@ -88,7 +88,7 @@ WSueCPH3puP2PcvqPJdNaDNF3jMZjtJtDiSy35pWrbt5h \ cargo run --bin polkadot-whois -- \ --ws=wss://westend-rpc.polkadot.io:443 \ --bootnodes /dns/westend-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWKer94o1REDPtAhjtYR4SdLehnSrN8PEhBnZm5NBoCrMC \ - --address-format substrate --timeout 120 --chain westend --session-index 42088 \ + --session-index 42088 \ by-validator-index 5 2 ``` ## Rococo @@ -96,7 +96,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://rococo-rpc.polkadot.io:443 \ --bootnodes /dns/rococo-bootnode-0.polkadot.io/tcp/30333/p2p/12D3KooWGikJMBmRiG5ofCqn8aijCijgfmZR5H9f53yUF3srm6Nm \ - --address-format substrate --timeout 120 --chain rococo --session-index 42088 \ + --session-index 42088 \ by-validator-index 295 293 ``` ## Paseo @@ -104,7 +104,7 @@ cargo run --bin polkadot-whois -- \ cargo run --bin polkadot-whois -- \ --ws=wss://paseo-rpc.dwellir.com:443 \ --bootnodes /dns/paseo.bootnode.amforc.com/tcp/29999/wss/p2p/12D3KooWSdf63rZjtGdeWXpQwQwPh8K8c22upcB3B1VmqW8rxrjw \ - --address-format substrate --timeout 120 --chain paseo --session-index 42088 \ + --session-index 42088 \ by-validator-index 11 12 ``` diff --git a/whois/src/main.rs b/whois/src/main.rs index e16df75b..62f5fc38 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -29,6 +29,7 @@ use polkadot_introspector_essentials::{ }; use serde::{Deserialize, Serialize}; use serde_binary::binary_stream::Endian; +use ss58_registry::Ss58AddressFormat; use std::{ collections::{HashMap, HashSet}, fs::{self, File}, @@ -45,14 +46,11 @@ struct WhoIsOptions { /// Web-Socket URLs of a relay chain node. #[clap(long)] pub ws: String, - /// Name of a chain to connect - #[clap(long)] - pub chain: Option, #[clap(flatten)] pub verbose: init::VerbosityOptions, #[clap(flatten)] pub retry: utils::RetryOptions, - /// The session index used for computing the validator indicies. + /// The session index used for computing the validator indices. #[clap(long)] pub session_index: u32, /// An optional block hash to fetch the queued keys at, otherwise we use the queued keys at the current block. @@ -67,20 +65,11 @@ struct WhoIsOptions { /// For example, "/ip4/127.0.0.1/tcp/30333/ws/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp". #[clap(long, use_value_delimiter = true, value_parser)] bootnodes: Vec, - /// The number of seconds the discovery process should run for. - #[clap(long, short, value_parser = parse_duration)] + /// The number of seconds the discovery process should run for, default value + /// is empirically values found for the polkadot and kusama network, smaller + /// networks may require a smaller values. + #[clap(long, short, value_parser = parse_duration, default_value = "900")] timeout: std::time::Duration, - /// The address format name of the chain. - /// Used to display the SS58 address of the authorities. - /// - /// For example: - /// - "polkadot" for Polkadot - /// - "substrate" for Substrate - /// - "kusama" for Kusama - /// - /// Used to display the SS58 address of the authorities. - #[clap(long, short)] - address_format: String, } fn parse_duration(arg: &str) -> Result { @@ -134,6 +123,8 @@ pub enum WhoisError { NoSessionIndex, #[error("Could not determine the genesis hash")] NoGenesisHash, + #[error("Could not determine the chain name")] + NoChainName, #[error("Invalid session index, session can not be older than {0}")] InvalidSessionIndex(u32), #[error("Keys for the session with given index not found")] @@ -178,6 +169,10 @@ impl Whois { return Err(WhoisError::NoGenesisHash) }; + let Ok(chain_name) = executor.get_chain_name(&self.opts.ws).await else { return Err(WhoisError::NoChainName) }; + + println!("Using chain name: {} genesis_hash: {:?}", chain_name, genesis_hash); + let Ok(session_index_now) = executor.get_session_index_now(&self.opts.ws).await else { return Err(WhoisError::NoSessionIndex) }; @@ -204,7 +199,8 @@ impl Whois { _ => return Err(WhoisError::NoSessionQueuedKeys), }; - let network_cache = NetworkCache::build_cache(session_index_now, genesis_hash, &self.opts).await?; + let network_cache = + NetworkCache::build_cache(session_index_now, genesis_hash, chain_name.as_str(), &self.opts).await?; let network_cache_for_session = network_cache.get_closest_to_session(self.opts.session_index)?; // A vector of (validator, validator_index) pairs representing the validator account @@ -333,10 +329,11 @@ impl NetworkCache { async fn build_cache( session_index_now: u32, genesis_hash: H256, + chain_name: &str, opts: &WhoIsOptions, ) -> color_eyre::Result { let mut update_cache = opts.update_p2p_cache; - let cache_path = format!("{}/{}", DEFAULT_CACHE_DIR, opts.chain.clone().unwrap_or("any".to_string())); + let cache_path = format!("{}/{}", DEFAULT_CACHE_DIR, chain_name.to_ascii_lowercase()); println!("Using cache path: {}", cache_path); let mut cache: NetworkCache = if let Ok(serialized_cache) = fs::read(cache_path.as_str()) { serde_binary::from_vec(serialized_cache, Endian::Big) @@ -345,7 +342,11 @@ impl NetworkCache { update_cache = true; Default::default() }; - + let address_format = Ss58AddressFormat::all_names() + .into_iter() + .map(|x| *x) + .find(|x| chain_name.eq_ignore_ascii_case(x)) + .unwrap_or("substrate"); if update_cache { println!("Discovering DHT authorithies, this may take a while..."); let (authorithy_discovery, _) = subp2p_explorer_cli::commands::authorities::discover_authorities( @@ -353,11 +354,14 @@ impl NetworkCache { format!("{:?}", genesis_hash), opts.bootnodes.clone(), opts.timeout, - opts.address_format.clone(), + address_format.into(), Default::default(), ) .await - .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; + .map_err(|err| { + println!("Error: {:?}", err); + WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()) + })?; let peer_details = authorithy_discovery.peer_details().clone(); let peer_info = authorithy_discovery.peer_info().clone(); From 2f32f53c3c8a1f12fa6ea7817331768aec97d080 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 11:28:39 +0300 Subject: [PATCH 08/13] Make clippy happy Signed-off-by: Alexandru Gheorghe --- whois/src/main.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/whois/src/main.rs b/whois/src/main.rs index 62f5fc38..723ecefc 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -277,15 +277,15 @@ impl Whois { let session_keys_for_validator = &session_queued_keys.iter().find(|(account, _)| account == &validator); if let Some((_, session_keys_for_validator)) = session_keys_for_validator { - let authorithy_discovery_key = session_keys_for_validator.authority_discovery.0.clone(); - let (peer_details, info, peer_id) = network_cache_for_session.get_details(authorithy_discovery_key); + let authority_discovery_key = session_keys_for_validator.authority_discovery.0; + let (peer_details, info, peer_id) = network_cache_for_session.get_details(authority_discovery_key); println!( - "validator_index={:?}, account={:}, peer_id={:}, authorithy_id_discover=0x{:}, addresses={:?}, version={:?}", + "validator_index={:?}, account={:}, peer_id={:}, authority_id_discover=0x{:}, addresses={:?}, version={:?}", validator_index.unwrap_or(usize::MAX), validator, peer_id.map(|peer_id| peer_id.to_string()).unwrap_or("unknown".to_string()), - hex::encode(authorithy_discovery_key), + hex::encode(authority_discovery_key), peer_details.map(|details| details.addresses().clone()), info, ); @@ -343,8 +343,8 @@ impl NetworkCache { Default::default() }; let address_format = Ss58AddressFormat::all_names() - .into_iter() - .map(|x| *x) + .iter() + .copied() .find(|x| chain_name.eq_ignore_ascii_case(x)) .unwrap_or("substrate"); if update_cache { @@ -382,9 +382,10 @@ impl NetworkCache { let serialized_cache = serde_binary::to_vec(&cache, Endian::Big) .map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; let path = std::path::Path::new(cache_path.as_str()); - path.parent().map(|prefix| { + if let Some(prefix) = path.parent() { let _ = std::fs::create_dir_all(prefix).map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into())); - }); + }; + let mut file = File::create(cache_path.as_str()).map_err(|_| WhoisError::InvalidP2PCache(DEFAULT_CACHE_DIR.into()))?; file.write_all(serialized_cache.as_slice()) From 018de74a983f104cc60c3f4cd407b56f53b7206e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 14:08:04 +0300 Subject: [PATCH 09/13] Add some sanity checks on dump-all Signed-off-by: Alexandru Gheorghe --- whois/README.md | 9 +++++- whois/src/main.rs | 82 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/whois/README.md b/whois/README.md index 253703a8..85329220 100644 --- a/whois/README.md +++ b/whois/README.md @@ -79,7 +79,7 @@ cargo run --bin polkadot-whois -- \ --ws=wss://kusama-rpc.polkadot.io:443 \ --bootnodes /dns/kusama-bootnode-0.polkadot.io/tcp/30333/p2p/12D3Koo WSueCPH3puP2PcvqPJdNaDNF3jMZjtJtDiSy35pWrbt5h \ - --address-format kusama --timeout 900 --chain kusama --session-index 42088 \ + --session-index 42088 \ by-validator-index 295 293 ``` @@ -108,3 +108,10 @@ cargo run --bin polkadot-whois -- \ by-validator-index 11 12 ``` + +## Limitations + +To map the information between authority-discovery keys and the accounts, the tool uses the result +of `session.queued_keys` at the session the tool is used. Sometimes, the accounts of interest are not +in the current `queued_keys` and tool will mark them as `unknown` in that case you can work around this +problem by providing an older block with `--queued-keys-at-block` argument. \ No newline at end of file diff --git a/whois/src/main.rs b/whois/src/main.rs index 723ecefc..7b4cb924 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -125,8 +125,8 @@ pub enum WhoisError { NoGenesisHash, #[error("Could not determine the chain name")] NoChainName, - #[error("Invalid session index, session can not be older than {0}")] - InvalidSessionIndex(u32), + #[error("Invalid session index, session needs to be between {0} and {1}")] + InvalidSessionIndex(u32, u32), #[error("Keys for the session with given index not found")] NoSessionKeys, #[error("Validator with given index not found")] @@ -180,7 +180,10 @@ impl Whois { if session_index_now < self.opts.session_index || session_index_now - self.opts.session_index > NUMBER_OF_STORED_SESSIONS { - return Err(WhoisError::InvalidSessionIndex(session_index_now - NUMBER_OF_STORED_SESSIONS)); + return Err(WhoisError::InvalidSessionIndex( + session_index_now - NUMBER_OF_STORED_SESSIONS, + session_index_now, + )); } let para_session_account_keys = @@ -203,6 +206,8 @@ impl Whois { NetworkCache::build_cache(session_index_now, genesis_hash, chain_name.as_str(), &self.opts).await?; let network_cache_for_session = network_cache.get_closest_to_session(self.opts.session_index)?; + let run_sanity_check = matches!(self.opts.command, WhoisCommand::DumpAll); + // A vector of (validator, validator_index) pairs representing the validator account // and its index in para_session_account_keys. let accounts_to_discover = match self.opts.command { @@ -269,17 +274,14 @@ impl Whois { .collect_vec(), }; - println!( - "======================================== List of validators ========================================" - ); - + let mut current_authority_discovery_keys = HashSet::new(); for (validator, validator_index) in accounts_to_discover { let session_keys_for_validator = &session_queued_keys.iter().find(|(account, _)| account == &validator); if let Some((_, session_keys_for_validator)) = session_keys_for_validator { let authority_discovery_key = session_keys_for_validator.authority_discovery.0; let (peer_details, info, peer_id) = network_cache_for_session.get_details(authority_discovery_key); - + current_authority_discovery_keys.insert(authority_discovery_key); println!( "validator_index={:?}, account={:}, peer_id={:}, authority_id_discover=0x{:}, addresses={:?}, version={:?}", validator_index.unwrap_or(usize::MAX), @@ -296,8 +298,13 @@ impl Whois { validator, ); } - println!("=============================================================================================="); + println!(""); } + + if run_sanity_check { + network_cache_for_session.sanity_check(current_authority_discovery_keys); + } + executor.close().await; Ok(vec![]) @@ -309,8 +316,11 @@ const DEFAULT_CACHE_DIR: &str = "whois_p2pcache"; // Information about the p2p network at a given session index. #[derive(Serialize, Deserialize)] struct PerSessionNetworkCache { + /// PeerId to PeerDetails mapping. pub peer_details: HashMap, PeerDetails>, + /// PeerId to version mapping. pub peer_versions: HashMap, String>, + /// Authority Id to discovered addresses. pub authority_to_details: HashMap>, } @@ -408,11 +418,13 @@ impl NetworkCache { .min(); if let Some(closest_session_lower) = closest_session_lower { + println!("Using p2p cache took at session: {}", closest_session_lower); Ok(self .session_to_network_info .get(closest_session_lower) .expect("closest_session_lower is obtained from session_to_network_info; qed")) } else if let Some(closest_session_larger) = closest_session_larger { + println!("Using p2p cache took at session: {}", closest_session_larger); Ok(self .session_to_network_info .get(closest_session_larger) @@ -450,6 +462,58 @@ impl PerSessionNetworkCache { let serialized_key = peer_id.to_bytes(); self.peer_details.get(&serialized_key).map(|x| x.authority_id()).cloned() } + + fn sanity_check(&self, current_discovery_keys: HashSet<[u8; 32]>) { + let mut served_authorithies_by_peer = HashMap::new(); + println!("Running sanity checks on p2p cache for past present and future authorities"); + for (authority_id, addresses) in self.authority_to_details.iter() { + let mut peer_ids = HashSet::new(); + if !current_discovery_keys.contains(authority_id) { + continue; + } + // https://github.com/paritytech/polkadot-sdk/blob/b9eb68bcb5ab93e58bcba4425975ad00374da2bc/substrate/client/authority-discovery/src/worker.rs#L74 + const MAX_ADDRESSES_PER_AUTHORITY: usize = 10; + if addresses.len() > MAX_ADDRESSES_PER_AUTHORITY { + println!( + "WARN: Authority 0x{:} has more than the maximum recommended addresses recommended {:} found {:} ", + hex::encode(authority_id), + MAX_ADDRESSES_PER_AUTHORITY, + addresses.len() + ); + } + for address in addresses { + let peer_id = get_peer_id(address); + if let Some(peer_id) = peer_id { + peer_ids.insert(peer_id); + served_authorithies_by_peer + .entry(peer_id) + .or_insert(HashSet::new()) + .insert(authority_id); + } + } + + if peer_ids.len() > 1 { + println!("WARN: Authority 0x{:} has multiple peer ids: {:?}", hex::encode(authority_id), peer_ids); + } + + if peer_ids.is_empty() { + println!("WARN: Authority 0x{:} has no peer ids", hex::encode(authority_id)); + } + } + + for (peer_id, authorities) in served_authorithies_by_peer.iter() { + if authorities.len() > 1 { + println!( + "WARN: Peer {:} serves multiple authorities: {:}", + peer_id, + authorities.iter().fold(String::new(), |acc, authority_id| { + format!("{:} 0x{:}", acc, hex::encode(authority_id)) + }) + ); + } + } + println!("Sanity checks on p2p cache completed"); + } } #[tokio::main] From d3919391cc98cdb72ce09b292b302a22453b2c12 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 14:14:52 +0300 Subject: [PATCH 10/13] Add some comments Signed-off-by: Alexandru Gheorghe --- whois/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/whois/src/main.rs b/whois/src/main.rs index 7b4cb924..213c047c 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -463,6 +463,8 @@ impl PerSessionNetworkCache { self.peer_details.get(&serialized_key).map(|x| x.authority_id()).cloned() } + // Run sanity checks on the p2p cache and print the peers with known problems. + // This are things that in the past have caused issues for validators. fn sanity_check(&self, current_discovery_keys: HashSet<[u8; 32]>) { let mut served_authorithies_by_peer = HashMap::new(); println!("Running sanity checks on p2p cache for past present and future authorities"); From ddf8972f3aea326d03fb6b67aa7abd743d9837d2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 14:31:09 +0300 Subject: [PATCH 11/13] Update Cargo.toml/Cargo.lock Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 4 ++-- whois/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1828b6c0..682810db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5465,7 +5465,7 @@ dependencies = [ [[package]] name = "subp2p-explorer" version = "0.1.0" -source = "git+https://github.com/alexggh/subp2p-explorer?branch=alexggh/make-subp2p-usable-as-library#f7fa5d0474ca348f76f3f86e452e342bcaeb2b39" +source = "git+https://github.com/lexnv/subp2p-explorer?branch=main#b318ea5fe7d5db04d4c6c48c5e8bde5b0ebb3f30" dependencies = [ "async-trait", "asynchronous-codec", @@ -5499,7 +5499,7 @@ dependencies = [ [[package]] name = "subp2p-explorer-cli" version = "0.1.0" -source = "git+https://github.com/alexggh/subp2p-explorer?branch=alexggh/make-subp2p-usable-as-library#f7fa5d0474ca348f76f3f86e452e342bcaeb2b39" +source = "git+https://github.com/lexnv/subp2p-explorer?branch=main#b318ea5fe7d5db04d4c6c48c5e8bde5b0ebb3f30" dependencies = [ "async-trait", "asynchronous-codec", diff --git a/whois/Cargo.toml b/whois/Cargo.toml index 9467ab73..7180a324 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -23,7 +23,7 @@ ss58-registry = {workspace = true } itertools = {workspace = true } serde-binary = {workspace = true } hex = {workspace = true } -subp2p-explorer-cli = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} -subp2p-explorer = { git = "https://github.com/alexggh/subp2p-explorer", branch="alexggh/make-subp2p-usable-as-library"} +subp2p-explorer-cli = { git = "https://github.com/lexnv/subp2p-explorer", branch="main"} +subp2p-explorer = { git = "https://github.com/lexnv/subp2p-explorer", branch="main"} libp2p = { version = "0.52.4", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp", "tokio", "yamux", "websocket", "request-response"] } From faf2061570ad9411d1a3b7c7f81d564bb860f608 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 18:38:42 +0300 Subject: [PATCH 12/13] Add bitfields performance Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 53 ++++++ essentials/src/api/api_client.rs | 12 +- essentials/src/api/executor.rs | 9 +- essentials/src/collector/mod.rs | 2 +- whois/Cargo.toml | 1 + whois/src/main.rs | 273 ++++++++++++++++++++++++++++++- 6 files changed, 335 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 682810db..425dab84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,21 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.8" @@ -738,6 +753,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.0", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2002,6 +2031,29 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -3914,6 +3966,7 @@ dependencies = [ name = "polkadot-whois" version = "0.2.33" dependencies = [ + "chrono", "clap", "color-eyre", "env_logger", diff --git a/essentials/src/api/api_client.rs b/essentials/src/api/api_client.rs index 85d7837e..4cd582a9 100644 --- a/essentials/src/api/api_client.rs +++ b/essentials/src/api/api_client.rs @@ -194,9 +194,17 @@ impl> ApiClient { self.storage().at_latest().await?.fetch(&addr).await } - pub async fn get_session_account_keys(&self, session_index: u32) -> Result>, subxt::Error> { + pub async fn get_session_account_keys( + &self, + session_index: u32, + hash: Option, + ) -> Result>, subxt::Error> { let addr = polkadot::storage().para_session_info().account_keys(session_index); - self.storage().at_latest().await?.fetch(&addr).await + if let Some(hash) = hash { + self.storage().at(hash).fetch(&addr).await + } else { + self.storage().at_latest().await?.fetch(&addr).await + } } pub async fn get_session_next_keys(&self, account: &AccountId32) -> Result, subxt::Error> { diff --git a/essentials/src/api/executor.rs b/essentials/src/api/executor.rs index e9c9a734..59205e26 100644 --- a/essentials/src/api/executor.rs +++ b/essentials/src/api/executor.rs @@ -61,7 +61,7 @@ pub enum Request { GetOccupiedCores(H256), GetBackingGroups(H256), GetSessionIndex(H256), - GetSessionAccountKeys(u32), + GetSessionAccountKeys(u32, Option), GetSessionNextKeys(AccountId32), GetSessionQueuedKeys(Option), GetInboundOutBoundHrmpChannels(H256, Vec), @@ -234,8 +234,8 @@ impl RequestExecutorBackend { BackingGroups(decode_validator_groups(&value)?) }, GetSessionIndex(hash) => SessionIndex(client.get_session_index(hash).await?.unwrap_or_default()), - GetSessionAccountKeys(session_index) => - SessionAccountKeys(client.get_session_account_keys(session_index).await?), + GetSessionAccountKeys(session_index, at) => + SessionAccountKeys(client.get_session_account_keys(session_index, at).await?), GetSessionNextKeys(ref account) => SessionNextKeys(client.get_session_next_keys(account).await?), GetSessionQueuedKeys(at) => SessionQueuedKeys(client.get_session_queued_keys(at).await?), GetSessionIndexNow => SessionIndex(client.get_session_index_now().await?.unwrap_or_default()), @@ -432,8 +432,9 @@ impl RequestExecutor { &mut self, url: &str, session_index: u32, + at: Option, ) -> color_eyre::Result>, RequestExecutorError> { - wrap_backend_call!(self, url, GetSessionAccountKeys, SessionAccountKeys, session_index) + wrap_backend_call!(self, url, GetSessionAccountKeys, SessionAccountKeys, session_index, at) } pub async fn get_session_next_keys( diff --git a/essentials/src/collector/mod.rs b/essentials/src/collector/mod.rs index 3a6146e7..1c4277e3 100644 --- a/essentials/src/collector/mod.rs +++ b/essentials/src/collector/mod.rs @@ -608,7 +608,7 @@ impl Collector { debug!("new session: {}, hash: {}", cur_session, cur_session_hash); let accounts_keys = self .executor - .get_session_account_keys(self.endpoint.as_str(), cur_session) + .get_session_account_keys(self.endpoint.as_str(), cur_session, None) .await? .ok_or_else(|| eyre!("Missing account keys for session {}", cur_session))?; self.storage_write_prefixed( diff --git a/whois/Cargo.toml b/whois/Cargo.toml index 7180a324..2a554de4 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -15,6 +15,7 @@ futures = { workspace = true } log = { workspace = true } polkadot-introspector-essentials = { workspace = true } polkadot-introspector-priority-channel = { workspace = true } +chrono = "0.4" thiserror = { workspace = true } tokio = { workspace = true } serde = {workspace = true } diff --git a/whois/src/main.rs b/whois/src/main.rs index 213c047c..ca8c1f68 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with polkadot-introspector. If not, see . +use chrono::{DateTime, Utc}; use clap::{Args, Parser, Subcommand}; use futures::future; use itertools::Itertools; @@ -24,16 +25,21 @@ use polkadot_introspector_essentials::{ executor::{RequestExecutor, RequestExecutorError}, }, init, - types::{AccountId32, H256}, + metadata::{ + polkadot::session::events::new_session::SessionIndex, + polkadot_primitives::{AvailabilityBitfield, ValidatorIndex}, + }, + types::{AccountId32, SessionKeys, H256}, utils, }; use serde::{Deserialize, Serialize}; use serde_binary::binary_stream::Endian; use ss58_registry::Ss58AddressFormat; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, fs::{self, File}, io::Write, + time::{Duration, UNIX_EPOCH}, }; use subp2p_explorer::util::{crypto::sr25519, p2p::get_peer_id}; use subp2p_explorer_cli::commands::authorities::PeerDetails; @@ -89,6 +95,35 @@ enum WhoisCommand { ByPeerId(PeerIdOptions), /// Display information about all validators. DumpAll, + /// Display performance statistics for bitfields for each validator. + BitFieldsPerformance(BitFieldsPerformance), +} +#[derive(Copy, Clone, Debug, Args)] +struct BitFieldsPerformance { + /// The block number to start from. + start_block: u32, + /// The step to go back in blocks. + num_to_skip: u32, + /// The number of blocks to check. + num_blocks: u32, + /// Print only validators that have at least this number of sessions with poor performance. + num_sessions_bellow_threshold: u32, + /// The threshold for poor performance. + poor_performance_threshold: f64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Default)] +struct BitfieldsStats { + // Number of blocks with poor performance. + num_poor_performance: usize, + // Number of blocks with the bitfield of the validator being present. + num_present: usize, +} + +impl BitfieldsStats { + fn percentage_poor(&self) -> f64 { + (self.num_poor_performance as f64 * 100.0) / self.num_present as f64 + } } #[derive(Clone, Debug, Args)] @@ -186,12 +221,14 @@ impl Whois { )); } - let para_session_account_keys = - match executor.get_session_account_keys(&self.opts.ws, self.opts.session_index).await { - Ok(Some(validators)) => validators, - Err(e) => return Err(WhoisError::SubxtError(e)), - _ => return Err(WhoisError::NoParaSessionAccountKeys), - }; + let para_session_account_keys = match executor + .get_session_account_keys(&self.opts.ws, self.opts.session_index, None) + .await + { + Ok(Some(validators)) => validators, + Err(e) => return Err(WhoisError::SubxtError(e)), + _ => return Err(WhoisError::NoParaSessionAccountKeys), + }; let session_queued_keys = match executor .get_session_queued_keys(&self.opts.ws, self.opts.queued_keys_at_block) @@ -272,6 +309,17 @@ impl Whois { .enumerate() .map(|(validator_index, account)| (account, Some(validator_index))) .collect_vec(), + WhoisCommand::BitFieldsPerformance(opts) => { + self.run_bitfields_performance_analysis( + opts, + &mut executor, + session_queued_keys, + network_cache_for_session, + ) + .await; + executor.close().await; + return Ok(vec![]); + }, }; let mut current_authority_discovery_keys = HashSet::new(); @@ -309,6 +357,215 @@ impl Whois { Ok(vec![]) } + + async fn run_bitfields_performance_analysis( + &self, + opts: BitFieldsPerformance, + executor: &mut RequestExecutor, + session_queued_keys: Vec<(AccountId32, SessionKeys)>, + network_cache_for_session: &PerSessionNetworkCache, + ) { + println!("Running bitfields performance analysis"); + let mut per_session_by_validatory_stats: BTreeMap> = + BTreeMap::new(); + let mut accounts_by_session = BTreeMap::new(); + let mut per_account_by_session_stats: BTreeMap> = + BTreeMap::new(); + let mut session_timestamps = BTreeMap::new(); + let mut start = opts.start_block; + let mut num_blocks_poor_perf_more_than_a_third = BTreeMap::new(); + + for _ in 0..opts.num_blocks { + let Ok(Some(block_hash)) = executor.get_block_hash(&self.opts.ws, Some(start)).await else { + break; + }; + + let Ok(session_index_now) = executor.get_session_index(&self.opts.ws, block_hash).await else { + break; + }; + + if !accounts_by_session.contains_key(&session_index_now) { + let Ok(timestamp) = executor.get_block_timestamp(&self.opts.ws, block_hash).await else { + break; + }; + + let timestamp_date = UNIX_EPOCH + Duration::from_millis(timestamp); + let timestamp_date: DateTime = DateTime::from(timestamp_date); + let timestamp_str = timestamp_date.format("%Y-%m-%d %H:%M").to_string(); + session_timestamps.insert(session_index_now, timestamp_str.clone()); + println!("session: {} timestamp: {}", session_index_now, timestamp_str); + let para_session_account_keys = match executor + .get_session_account_keys(&self.opts.ws, session_index_now, Some(block_hash)) + .await + { + Ok(Some(validators)) => validators, + Err(_e) => break, + _ => break, + }; + + let para_session_account_keys = para_session_account_keys + .into_iter() + .enumerate() + .map(|(validator_index, account)| (account, validator_index)) + .collect_vec(); + accounts_by_session.insert(session_index_now, para_session_account_keys); + } + + let Ok(para_inherent) = executor.extract_parainherent_data(&self.opts.ws, Some(block_hash)).await else { + break; + }; + + let bitfields = para_inherent + .bitfields + .into_iter() + .map(|b| (b.payload, b.validator_index)) + .collect::>(); + + let mut num_poor_performance_per_block = 0; + + let Some(max) = bitfields + .iter() + .map(|(bitfield, _)| bitfield.0.as_bits().iter().filter(|bit| *bit).count()) + .max() + else { + break; + }; + + let Some(session_accounts) = accounts_by_session.get(&session_index_now) else { + break; + }; + let session_stats = per_session_by_validatory_stats.entry(session_index_now).or_default(); + + for bitfield in bitfields { + let num_bits_set = bitfield.0 .0.as_bits().iter().filter(|bit| *bit).count(); + + let Some(validator) = session_accounts + .iter() + .find(|(_, validator)| *validator as u32 == bitfield.1 .0) + .map(|val| val.0.clone()) + else { + break; + }; + + let per_account_stats = per_account_by_session_stats + .entry(validator) + .or_default() + .entry(session_index_now) + .or_default(); + + let per_session_stats = session_stats.entry(bitfield.1 .0).or_insert(Default::default()); + + // The validator has poor performance if it has less than 2 bits set. + if num_bits_set < std::cmp::min(max, 2) { + num_poor_performance_per_block += 1; + (*per_session_stats).num_poor_performance += 1; + per_account_stats.num_poor_performance += 1; + } + (*per_session_stats).num_present += 1; + per_account_stats.num_present += 1; + } + + if num_poor_performance_per_block > session_accounts.len() / 3 { + *(num_blocks_poor_perf_more_than_a_third.entry(session_index_now).or_insert(0)) += 1; + } + start -= opts.num_to_skip; + } + + print_per_session_performance(opts, per_session_by_validatory_stats, &session_timestamps); + + print_per_account_performance( + opts, + session_queued_keys, + network_cache_for_session, + per_account_by_session_stats, + &session_timestamps, + ); + + for (session, more_than_a_third) in num_blocks_poor_perf_more_than_a_third.iter() { + println!( + "block_start: {}, session: {}: {}, Count blocks with unincluded {:}", + opts.start_block, + session, + session_timestamps.get(session).cloned().unwrap_or_default(), + more_than_a_third, + ); + } + } +} + +fn print_per_account_performance( + opts: BitFieldsPerformance, + session_queued_keys: Vec<(AccountId32, SessionKeys)>, + network_cache_for_session: &PerSessionNetworkCache, + per_account_by_session_stats: BTreeMap>, + session_timestamps: &BTreeMap, +) { + for (account, account_stats) in per_account_by_session_stats.iter() { + let count_past_30 = account_stats + .iter() + .map(|(_, stats)| (stats.num_poor_performance as f64 * 100.0) / stats.num_present as f64) + .filter(|x| *x > opts.poor_performance_threshold) + .count(); + + if count_past_30 < opts.num_sessions_bellow_threshold as usize { + continue; + } + + let session_keys_for_validator = &session_queued_keys.iter().find(|(this_account, _)| this_account == account); + + let name = if let Some((_, session_keys_for_validator)) = session_keys_for_validator { + let authority_discovery_key = session_keys_for_validator.authority_discovery.0; + let (_, info, _) = network_cache_for_session.get_details(authority_discovery_key); + info + } else { + "unknown".to_string() + }; + + println!("block_start: {}, account: {} name: {}", opts.start_block, account, name); + for (session, stats) in account_stats.iter() { + let percentage = stats.percentage_poor(); + + println!( + " at {} session: {}, has {:?} with zero bits, {:.2}% percent", + session_timestamps.get(session).cloned().unwrap_or_default(), + session, + stats.num_poor_performance, + percentage + ); + } + } +} + +fn print_per_session_performance( + opts: BitFieldsPerformance, + per_session_by_validatory_stats: BTreeMap>, + session_timestamps: &BTreeMap, +) { + let mut count_per_session_poor_performance = BTreeMap::new(); + for (session, stats_by_validator) in per_session_by_validatory_stats.iter() { + for (validator, stats) in stats_by_validator.iter() { + let percentage = stats.percentage_poor(); + if percentage > opts.poor_performance_threshold { + *(count_per_session_poor_performance.entry(session).or_insert(0)) += 1; + } + + println!( + "block_start: {}, session: {}: validator: {:?} has {:?} zero bits {:.3}%", + opts.start_block, session, validator, stats.num_poor_performance, percentage + ); + } + } + + for (session, count) in count_per_session_poor_performance.iter() { + println!( + "block_start: {}, session: {}: {} Number of validators with 10% missing {:} count_all_with_0 {:}", + opts.start_block, + session, + session_timestamps.get(session).cloned().unwrap_or_default(), + count, + per_session_by_validatory_stats.len() + ); + } } const DEFAULT_CACHE_DIR: &str = "whois_p2pcache"; From a9b13abf88f8dea6233b317eb385deceacd61497 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Jan 2025 16:07:41 +0200 Subject: [PATCH 13/13] Format Signed-off-by: Alexandru Gheorghe --- whois/src/main.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/whois/src/main.rs b/whois/src/main.rs index 36635a59..4f2a8864 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -25,7 +25,10 @@ use polkadot_introspector_essentials::{ executor::{RequestExecutor, RequestExecutorError}, }, init, - metadata::{polkadot::session::events::new_session::SessionIndex, polkadot_primitives::{AvailabilityBitfield, ValidatorIndex}}, + metadata::{ + polkadot::session::events::new_session::SessionIndex, + polkadot_primitives::{AvailabilityBitfield, ValidatorIndex}, + }, types::{AccountId32, SessionKeys, H256}, utils, }; @@ -343,7 +346,7 @@ impl Whois { validator, ); } - println!(""); + println!(); } if run_sanity_check {