From 65baa145319b0506f1a43a70f488dd92313e98a9 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Tue, 3 Dec 2024 10:57:19 -0800 Subject: [PATCH] [TWAP] Update Hermes binary response format, add TwapMessage parsing to price-service-sdk (#2158) * feat: change hermes twap binary response, add twap data model and parsing to price-service-sdk * feat: reexport parseTwapMessage * test: add parse twap msg test * test: update parse twap test * refactor: pack both start & end updatedatas into a single binaryupdate * refactor: snake -> camel case --- apps/hermes/server/Cargo.lock | 2 +- apps/hermes/server/Cargo.toml | 2 +- .../server/src/api/rest/v2/latest_twaps.rs | 21 +++----- apps/hermes/server/src/api/types.rs | 4 +- apps/hermes/server/src/state/aggregate.rs | 36 +++++--------- price_service/sdk/js/package.json | 2 +- .../sdk/js/src/AccumulatorUpdateData.ts | 49 ++++++++++++++++++- .../__tests__/AccumulatorUpdateData.test.ts | 35 +++++++++++++ price_service/sdk/js/src/index.ts | 1 + 9 files changed, 110 insertions(+), 42 deletions(-) diff --git a/apps/hermes/server/Cargo.lock b/apps/hermes/server/Cargo.lock index 6ea05d5f3..e5f1866b0 100644 --- a/apps/hermes/server/Cargo.lock +++ b/apps/hermes/server/Cargo.lock @@ -1868,7 +1868,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermes" -version = "0.8.0" +version = "0.8.1" dependencies = [ "anyhow", "async-trait", diff --git a/apps/hermes/server/Cargo.toml b/apps/hermes/server/Cargo.toml index d302a1490..8bc506af4 100644 --- a/apps/hermes/server/Cargo.toml +++ b/apps/hermes/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hermes" -version = "0.8.0" +version = "0.8.1" description = "Hermes is an agent that provides Verified Prices from the Pythnet Pyth Oracle." edition = "2021" diff --git a/apps/hermes/server/src/api/rest/v2/latest_twaps.rs b/apps/hermes/server/src/api/rest/v2/latest_twaps.rs index 34c10c81c..7428ace90 100644 --- a/apps/hermes/server/src/api/rest/v2/latest_twaps.rs +++ b/apps/hermes/server/src/api/rest/v2/latest_twaps.rs @@ -131,22 +131,17 @@ where })?; let twap_update_data = twaps_with_update_data.update_data; - let binary: Vec = twap_update_data + let encoded_data = twap_update_data .into_iter() - .map(|data_vec| { - let encoded_data = data_vec - .into_iter() - .map(|data| match params.encoding { - EncodingType::Base64 => base64_standard_engine.encode(data), - EncodingType::Hex => hex::encode(data), - }) - .collect(); - BinaryUpdate { - encoding: params.encoding, - data: encoded_data, - } + .map(|data| match params.encoding { + EncodingType::Base64 => base64_standard_engine.encode(data), + EncodingType::Hex => hex::encode(data), }) .collect(); + let binary = BinaryUpdate { + encoding: params.encoding, + data: encoded_data, + }; let parsed: Option> = if params.parsed { Some( diff --git a/apps/hermes/server/src/api/types.rs b/apps/hermes/server/src/api/types.rs index cdb612435..aa430773b 100644 --- a/apps/hermes/server/src/api/types.rs +++ b/apps/hermes/server/src/api/types.rs @@ -281,9 +281,9 @@ impl From for ParsedPriceFeedTwap { #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TwapsResponse { - /// Each BinaryUpdate contains the start & end cumulative price updates used to + /// Contains the start & end cumulative price updates used to /// calculate a given price feed's TWAP. - pub binary: Vec, + pub binary: BinaryUpdate, /// The calculated TWAPs for each price ID #[serde(skip_serializing_if = "Option::is_none")] diff --git a/apps/hermes/server/src/state/aggregate.rs b/apps/hermes/server/src/state/aggregate.rs index 884533175..43928b9db 100644 --- a/apps/hermes/server/src/state/aggregate.rs +++ b/apps/hermes/server/src/state/aggregate.rs @@ -206,7 +206,7 @@ pub struct PublisherStakeCapsWithUpdateData { #[derive(Debug)] pub struct TwapsWithUpdateData { pub twaps: Vec, - pub update_data: Vec>>, + pub update_data: Vec>, } #[derive(Debug, Serialize)] @@ -652,7 +652,6 @@ where } let mut twaps = Vec::new(); - let mut update_data = Vec::new(); // Iterate through start and end messages together for (start_message, end_message) in start_messages.iter().zip(end_messages.iter()) { @@ -676,34 +675,27 @@ where end_timestamp: end_twap.publish_time, down_slots_ratio, }); - - // Combine messages for update data - let mut messages = Vec::new(); - messages.push(start_message.clone().into()); - messages.push(end_message.clone().into()); - - if let Ok(update) = construct_update_data(messages) { - update_data.push(update); - } else { - tracing::warn!( - "Failed to construct update data for price feed {:?}", - start_twap.feed_id - ); - continue; - } } Err(e) => { - tracing::warn!( + return Err(anyhow!( "Failed to calculate TWAP for price feed {:?}: {}", start_twap.feed_id, e - ); - continue; + )); } } } } + // Construct update data. + // update_data[0] contains the start VAA and merkle proofs + // update_data[1] contains the end VAA and merkle proofs + let mut update_data = + construct_update_data(start_messages.into_iter().map(Into::into).collect())?; + update_data.extend(construct_update_data( + end_messages.into_iter().map(Into::into).collect(), + )?); + Ok(TwapsWithUpdateData { twaps, update_data }) } @@ -1316,10 +1308,8 @@ mod test { assert_eq!(twap_2.start_timestamp, 100); assert_eq!(twap_2.end_timestamp, 200); - // Verify update data contains both start and end messages for both feeds + // update_data should have 2 elements, one for the start block and one for the end block. assert_eq!(result.update_data.len(), 2); - assert_eq!(result.update_data[0].len(), 2); // Should contain 2 messages - assert_eq!(result.update_data[1].len(), 2); // Should contain 2 messages } #[tokio::test] diff --git a/price_service/sdk/js/package.json b/price_service/sdk/js/package.json index d3a68a964..a6e0dad27 100644 --- a/price_service/sdk/js/package.json +++ b/price_service/sdk/js/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/price-service-sdk", - "version": "1.7.1", + "version": "1.8.0", "description": "Pyth price service SDK", "homepage": "https://pyth.network", "main": "lib/index.js", diff --git a/price_service/sdk/js/src/AccumulatorUpdateData.ts b/price_service/sdk/js/src/AccumulatorUpdateData.ts index b531a4960..b57132bed 100644 --- a/price_service/sdk/js/src/AccumulatorUpdateData.ts +++ b/price_service/sdk/js/src/AccumulatorUpdateData.ts @@ -5,12 +5,12 @@ const MAJOR_VERSION = 1; const MINOR_VERSION = 0; const KECCAK160_HASH_SIZE = 20; const PRICE_FEED_MESSAGE_VARIANT = 0; +const TWAP_MESSAGE_VARIANT = 1; export type AccumulatorUpdateData = { vaa: Buffer; updates: { message: Buffer; proof: number[][] }[]; }; - export type PriceFeedMessage = { feedId: Buffer; price: BN; @@ -22,6 +22,17 @@ export type PriceFeedMessage = { emaConf: BN; }; +export type TwapMessage = { + feedId: Buffer; + cumulativePrice: BN; + cumulativeConf: BN; + numDownSlots: BN; + exponent: number; + publishTime: BN; + prevPublishTime: BN; + publishSlot: BN; +}; + export function isAccumulatorUpdateData(updateBytes: Buffer): boolean { return ( updateBytes.toString("hex").slice(0, 8) === ACCUMULATOR_MAGIC && @@ -29,6 +40,7 @@ export function isAccumulatorUpdateData(updateBytes: Buffer): boolean { updateBytes[5] === MINOR_VERSION ); } + export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage { let cursor = 0; const variant = message.readUInt8(cursor); @@ -64,6 +76,41 @@ export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage { }; } +export function parseTwapMessage(message: Buffer): TwapMessage { + let cursor = 0; + const variant = message.readUInt8(cursor); + if (variant !== TWAP_MESSAGE_VARIANT) { + throw new Error("Not a twap message"); + } + cursor += 1; + const feedId = message.subarray(cursor, cursor + 32); + cursor += 32; + const cumulativePrice = new BN(message.subarray(cursor, cursor + 16), "be"); + cursor += 16; + const cumulativeConf = new BN(message.subarray(cursor, cursor + 16), "be"); + cursor += 16; + const numDownSlots = new BN(message.subarray(cursor, cursor + 8), "be"); + cursor += 8; + const exponent = message.readInt32BE(cursor); + cursor += 4; + const publishTime = new BN(message.subarray(cursor, cursor + 8), "be"); + cursor += 8; + const prevPublishTime = new BN(message.subarray(cursor, cursor + 8), "be"); + cursor += 8; + const publishSlot = new BN(message.subarray(cursor, cursor + 8), "be"); + cursor += 8; + return { + feedId, + cumulativePrice, + cumulativeConf, + numDownSlots, + exponent, + publishTime, + prevPublishTime, + publishSlot, + }; +} + /** * An AccumulatorUpdateData contains a VAA and a list of updates. This function returns a new serialized AccumulatorUpdateData with only the updates in the range [start, end). */ diff --git a/price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts b/price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts index 1c78ddecf..53707b053 100644 --- a/price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts +++ b/price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts @@ -1,6 +1,7 @@ import { parseAccumulatorUpdateData, parsePriceFeedMessage, + parseTwapMessage, sliceAccumulatorUpdateData, } from "../AccumulatorUpdateData"; @@ -113,4 +114,38 @@ describe("Test parse accumulator update", () => { "Invalid accumulator message" ); }); + + test("Parse TWAP message", () => { + // Sample data from the Hermes latest TWAP endpoint. + const testAccumulatorDataTwap = + "UE5BVQEAAAADuAEAAAAEDQB0NFyANOScwaiDg0Z/8auG9F+gU98tL7TkAP7Oh5T6phJ1ztvkN/C+2vyPwzuYsY2qtW81C/TsmDISW4jprp7/AAOrwFH1EEaS7yDJ36Leva1xYh+iMITR6iQitFceC0+oPgIa24JOBZkhVn+2QU92LG5fQ7Qaigm1+SeeB5X1A8XJAQRrrQ5UwkYGFtE2XNU+pdYuSxUUaF7AbLAYu0tQ0UZEmFFRxYEhOM5dI+CmER4iXcXnbJY6vds6B4lCBGMu7dq1AAa0mOMBi3R2jUReD5fn0doFzGm7B8BD51CJYa7JL1th1g3KsgJUafvGVxRW8pVvMKGxJVnTEAty4073n0Yso72qAAgSZI1VGEhfft2ZRSbFNigZtqULTAHUs1Z/jEY1H9/VhgCOrkcX4537ypQag0782/8NOWMzyx/MIcC2TO1paC0FAApLUa4AH2mRbh9UBeMZrHhq8pqp8NiZkU91J4c97x2HpXOBuqbD+Um/zEhpBMWT2ew+5i5c2znOynCBRKmfVfX9AQvfJRz5/U2/ym9YVL2Cliq5eg7CyItz54tAoRaYr0N0RUP/S0w4o+3Vedcik1r7kE0rtulxy8GkCTmQMIhQ3zDTAA3Rug0WuQLb+ozeXprjwx/IrTY2pCo0hqOTTtYY/RqRDAnlxMWXnfFAADa2AkrPIdkrc9rcY7Vk7Q3OA2A2UDk7AQ6oE+H8iwtc6vuGgqSlPezdQwV+utfqsAtBEu4peTGYwGzgRQT6HAu3KA73IF9bS+JdDnffRIyaaSmAtgqKDc1yAQ8h92AsTgpNY+fKFwbFJKuyp92M9zVzoe8I+CNx1Mp59El/ScLRYYWfaYh3bOiJ7FLk5sWp8vKKuTv0CTNxtND5ABAKJqOrb7LSJZDP89VR7WszEW3y2ldxbWgzPcooMxczsXqFGdgKoj5puH6gNnU7tF3WDBaT2znkkQgZIE1fVGdtABEYOz3yXevBkKcPRY7Frn9RgLujva9qCJA75QTdor7w2XIhNFs8dTraTGdDE53s2syYIhh47MPYRfbrDJvJIZJ3ABJSt1XkGdeGsEA4S/78vJbmmcRndrJM5MDl1S3ChJ2iRVQgZxe0dxOHxWbwX4z5yDExkY0lfTTK3fQF2H0KQs6/AWdN2T8AAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAFykghAUFVV1YAAAAAAArXIu8AACcQCNiVurGRlVTMB0BmraQJiubDgKEDAGUBSfa2XLHeaxDq9158A8oCnDBtA1fpG1MRsXUISlrVVogAAAAAAAAAAAAGQO17DQ6NAAAAAAAAAAAAAASmkl6YWgAAAAAESzQb////+wAAAABnTdk/AAAAAGdN2T4AAAAACtci7wsj6vNMqJrG2JNfJY5yygVRvYFPfqEccSfDTemrudDuCgdhZucSwdNcVF/3QkxaBwCdfedAX7wyPoSu6LJJa57CwK41xm+wQUxF+sQXHePp4CsWWFrlzQNVzU4XsKhrTEdfjsRJslSTLbZpdRfIlxmaUtbr8xBKcpEQzfZjnCntTVTIQYeFvSqAdbz2Re5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcTAGUBK5qx6XKigVhQhBSLoTiYAHmb1L5juVdQfbE0kxTkdEUAAAAAAAAAAA0ueWD9HZgqAAAAAAAAAAAAA3UA2y4cRwAAAAAAAGoE////+AAAAABnTdk/AAAAAGdN2T4AAAAACtci7wvdelw0MqOTe1cEWlMuAQOb+g+aOjj25mEaG17nGLUt6R+fbQmWnpeAMBY2iyR21sQh/HkkPVZ7WUvi8LIDs0l6CxKFlqBJ/GpO27lLI1ua4pgCTInm3pR6PSha3omIpRyBLlDCi+TdAW4pHS03DJ5HfzKsxxTLTsQLf+ToMwDmEQ7oOuukWrswx6YE5+5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcTAGUBKgHersnlGleSd7NLEiOZmE0Lv1fiRYp+Qv7NKCmGeg0AAAAAAAAAAAAN5aKJ8+yVAAAAAAAAAAAAAAOCrlpWWgAAAAAAAGoI////+AAAAABnTdk/AAAAAGdN2T4AAAAACtci7wuKT84vWz8EFU5vAJ7UMs01HF1LnfUK2NS0SoHjdzdaIE3KToeRn1qn+JgVyownBm5NO6eveTckccp2xHbt9YeiASNxDuEx6AM7TbDcQBtoTj2s3Pk3icB5ivrH9sSOohCUJPoyi+TdAW4pHS03DJ5HfzKsxxTLTsQLf+ToMwDmEQ7oOuukWrswx6YE5+5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcT"; + const { updates } = parseAccumulatorUpdateData( + Buffer.from(testAccumulatorDataTwap, "base64") + ); + + // Test that both messages are parsed successfully + const twapMessage1 = parseTwapMessage(updates[0].message); + expect(twapMessage1.feedId.toString("hex")).toBe( + "49f6b65cb1de6b10eaf75e7c03ca029c306d0357e91b5311b175084a5ad55688" + ); + expect(twapMessage1.cumulativePrice.toString()).toBe("1760238576144013"); + expect(twapMessage1.cumulativeConf.toString()).toBe("5113466755162"); + expect(twapMessage1.numDownSlots.toString()).toBe("72037403"); + expect(twapMessage1.exponent).toBe(-5); + expect(twapMessage1.publishTime.toString()).toBe("1733155135"); + expect(twapMessage1.prevPublishTime.toString()).toBe("1733155134"); + expect(twapMessage1.publishSlot.toString()).toBe("181871343"); + + const twapMessage2 = parseTwapMessage(updates[1].message); + expect(twapMessage2.feedId.toString("hex")).toBe( + "2b9ab1e972a281585084148ba1389800799bd4be63b957507db1349314e47445" + ); + expect(twapMessage2.cumulativePrice.toString()).toBe("949830028892149802"); + expect(twapMessage2.cumulativeConf.toString()).toBe("973071467813959"); + expect(twapMessage2.numDownSlots.toString()).toBe("27140"); + expect(twapMessage2.exponent).toBe(-8); + expect(twapMessage2.publishTime.toString()).toBe("1733155135"); + expect(twapMessage2.prevPublishTime.toString()).toBe("1733155134"); + expect(twapMessage2.publishSlot.toString()).toBe("181871343"); + }); }); diff --git a/price_service/sdk/js/src/index.ts b/price_service/sdk/js/src/index.ts index 6739a2dd6..27640f111 100644 --- a/price_service/sdk/js/src/index.ts +++ b/price_service/sdk/js/src/index.ts @@ -15,6 +15,7 @@ export { parseAccumulatorUpdateData, AccumulatorUpdateData, parsePriceFeedMessage, + parseTwapMessage, } from "./AccumulatorUpdateData"; /**