From facb415f3c3ef7f5cc3d423963349a237bb4a5e5 Mon Sep 17 00:00:00 2001 From: Patrick Beza Date: Fri, 22 Nov 2024 12:18:10 +0100 Subject: [PATCH] feat(tee): add error handling for unstable_getTeeProofs API endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds more information to the response of the `unstable_getTeeProofs` API endpoint, enabling the [client][1] that sent the [request][2] to determine whether it makes sense to retry fetching the TEE proof for a particular batch number. Currently, the [TEE verifier][1] – the tool for continuous SGX attestation and batch signature verification – is [stuck][3] on batches that failed to be proven and are marked as `permanently_ignored`. The tool should be able to distinguish between batches that are permanently ignored (and should be skipped) and batches that have failed but will be retried. This PR enables that distinction. Example use cases: - requesting TEE proof for a batch with the `permanently_ignored` status ``` $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [14, "sgx"] }' 'http://localhost:3152' HTTP/1.1 200 OK content-type: application/json; charset=utf-8 vary: origin, access-control-request-method, access-control-request-headers access-control-allow-origin: * content-length: 207 date: Tue, 26 Nov 2024 12:42:48 GMT {"jsonrpc":"2.0","result":[{"l1BatchNumber":14,"teeType":"sgx","pubkey":null,"signature":null,"proof":null,"provedAt":"2024-11-20T15:43:46.112146Z","status":"permanently_ignored","attestation":null}],"id":1} ``` - requesting TEE proof for a batch with the `failed` status ``` $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [15, "sgx"] }' 'http://localhost:3152' HTTP/1.1 200 OK content-type: application/json; charset=utf-8 vary: origin, access-control-request-method, access-control-request-headers access-control-allow-origin: * content-length: 194 date: Tue, 26 Nov 2024 12:44:19 GMT {"jsonrpc":"2.0","result":[{"l1BatchNumber":15,"teeType":"sgx","pubkey":null,"signature":null,"proof":null,"provedAt":"2024-11-20T15:43:46.121432Z","status":"failed","attestation":null}],"id":1} ``` - requesting TEE proof for a batch with the `generated` status ``` $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [28, "sgx"] }' 'http://localhost:3152' HTTP/1.1 200 OK content-type: application/json; charset=utf-8 vary: origin, access-control-request-method, access-control-request-headers access-control-allow-origin: * content-length: 229 date: Tue, 26 Nov 2024 12:45:27 GMT {"jsonrpc":"2.0","result":[{"l1BatchNumber":28,"teeType":"sgx","pubkey":"0506070809","signature":"0001020304","proof":"1011121314","provedAt":"2024-11-20T15:21:16.129128Z","status":"generated","attestation":"0403020100"}],"id":1} ``` - requesting TEE proof for a non-existent batch ``` $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [1337, "sgx"] }' 'http://localhost:3152' HTTP/1.1 200 OK content-type: application/json; charset=utf-8 vary: origin, access-control-request-method, access-control-request-headers access-control-allow-origin: * content-length: 36 date: Tue, 26 Nov 2024 12:46:08 GMT {"jsonrpc":"2.0","result":[],"id":1} ``` Relevant database entries for the use cases mentioned above: ``` zksync_server_localhost_legacy=# SELECT * FROM tee_proof_generation_details WHERE l1_batch_number IN (14, 15, 28, 1337); l1_batch_number | status | signature | pubkey | proof | tee_type | created_at | updated_at | prover_taken_at -----------------+---------------------+--------------+--------------+--------------+----------+----------------------------+----------------------------+---------------------------- 14 | permanently_ignored | | | | sgx | 2023-11-20 15:27:47.281293 | 2024-11-20 15:43:46.112146 | 2024-11-20 15:43:46.106042 15 | failed | | | | sgx | 2024-11-20 15:27:47.287777 | 2024-11-20 15:43:46.121432 | 2024-11-20 15:43:46.115853 28 | generated | \x0001020304 | \x0506070809 | \x1011121314 | sgx | 2024-11-20 12:56:33.055642 | 2024-11-20 15:21:16.129128 | 2024-11-20 14:53:14.25949 (3 rows) ``` [1]: https://github.com/matter-labs/teepot/blob/main/bin/verify-era-proof-attestation/src/main.rs [2]: https://github.com/matter-labs/teepot/blob/1a8a9f17fa7284f83c41a63d37fe380aef6d550d/bin/verify-era-proof-attestation/src/proof.rs#L15-L21 [3]: https://grafana.matterlabs.dev/goto/unFqf57Hg?orgId=1 --- ...5e419ffae1fece87427c6571713047d44f015.json | 52 ++++++++++++++++++ ...3a0aedbd2455b2ff728fe88cab94728925c0e.json | 53 +++++++++++++++++++ ...5393ec02458104c483a2023b24881ae0c6716.json | 23 -------- core/lib/dal/src/models/storage_tee_proof.rs | 1 + core/lib/dal/src/tee_proof_generation_dal.rs | 32 ++++++----- core/lib/types/src/api/mod.rs | 1 + .../src/web3/namespaces/unstable.rs | 7 ++- core/node/proof_data_handler/src/tests.rs | 2 +- 8 files changed, 131 insertions(+), 40 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-8065f779631edb7ba30229485ff5e419ffae1fece87427c6571713047d44f015.json create mode 100644 core/lib/dal/.sqlx/query-b7fa581d662640236d11143953e3a0aedbd2455b2ff728fe88cab94728925c0e.json delete mode 100644 core/lib/dal/.sqlx/query-cf3c7b918a3f82476543841d4dc5393ec02458104c483a2023b24881ae0c6716.json diff --git a/core/lib/dal/.sqlx/query-8065f779631edb7ba30229485ff5e419ffae1fece87427c6571713047d44f015.json b/core/lib/dal/.sqlx/query-8065f779631edb7ba30229485ff5e419ffae1fece87427c6571713047d44f015.json new file mode 100644 index 000000000000..7559630217be --- /dev/null +++ b/core/lib/dal/.sqlx/query-8065f779631edb7ba30229485ff5e419ffae1fece87427c6571713047d44f015.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n tp.pubkey,\n tp.signature,\n tp.proof,\n tp.updated_at,\n tp.status,\n ta.attestation\n FROM\n tee_proof_generation_details tp\n LEFT JOIN\n tee_attestations ta ON tp.pubkey = ta.pubkey\n WHERE\n tp.l1_batch_number = $1\n ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pubkey", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "signature", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "proof", + "type_info": "Bytea" + }, + { + "ordinal": 3, + "name": "updated_at", + "type_info": "Timestamp" + }, + { + "ordinal": 4, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attestation", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + true, + true, + false, + false, + true + ] + }, + "hash": "8065f779631edb7ba30229485ff5e419ffae1fece87427c6571713047d44f015" +} diff --git a/core/lib/dal/.sqlx/query-b7fa581d662640236d11143953e3a0aedbd2455b2ff728fe88cab94728925c0e.json b/core/lib/dal/.sqlx/query-b7fa581d662640236d11143953e3a0aedbd2455b2ff728fe88cab94728925c0e.json new file mode 100644 index 000000000000..89f0706843bd --- /dev/null +++ b/core/lib/dal/.sqlx/query-b7fa581d662640236d11143953e3a0aedbd2455b2ff728fe88cab94728925c0e.json @@ -0,0 +1,53 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n tp.pubkey,\n tp.signature,\n tp.proof,\n tp.updated_at,\n tp.status,\n ta.attestation\n FROM\n tee_proof_generation_details tp\n LEFT JOIN\n tee_attestations ta ON tp.pubkey = ta.pubkey\n WHERE\n tp.l1_batch_number = $1\n AND tp.tee_type = $2ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pubkey", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "signature", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "proof", + "type_info": "Bytea" + }, + { + "ordinal": 3, + "name": "updated_at", + "type_info": "Timestamp" + }, + { + "ordinal": 4, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attestation", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + false, + false, + true + ] + }, + "hash": "b7fa581d662640236d11143953e3a0aedbd2455b2ff728fe88cab94728925c0e" +} diff --git a/core/lib/dal/.sqlx/query-cf3c7b918a3f82476543841d4dc5393ec02458104c483a2023b24881ae0c6716.json b/core/lib/dal/.sqlx/query-cf3c7b918a3f82476543841d4dc5393ec02458104c483a2023b24881ae0c6716.json deleted file mode 100644 index 59bfa4858c02..000000000000 --- a/core/lib/dal/.sqlx/query-cf3c7b918a3f82476543841d4dc5393ec02458104c483a2023b24881ae0c6716.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n COUNT(*) AS \"count!\"\n FROM\n (\n SELECT\n *\n FROM\n storage_logs\n WHERE\n hashed_key = $1\n AND miniblock_number <= COALESCE(\n (\n SELECT\n MAX(number)\n FROM\n miniblocks\n ),\n (\n SELECT\n miniblock_number\n FROM\n snapshot_recovery\n )\n )\n ORDER BY\n miniblock_number DESC,\n operation_number DESC\n LIMIT\n 1\n ) sl\n WHERE\n sl.value != $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Bytea", - "Bytea" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cf3c7b918a3f82476543841d4dc5393ec02458104c483a2023b24881ae0c6716" -} diff --git a/core/lib/dal/src/models/storage_tee_proof.rs b/core/lib/dal/src/models/storage_tee_proof.rs index 6f80c59511f9..6e031674b585 100644 --- a/core/lib/dal/src/models/storage_tee_proof.rs +++ b/core/lib/dal/src/models/storage_tee_proof.rs @@ -9,6 +9,7 @@ pub struct StorageTeeProof { pub signature: Option>, pub proof: Option>, pub updated_at: NaiveDateTime, + pub status: String, pub attestation: Option>, } diff --git a/core/lib/dal/src/tee_proof_generation_dal.rs b/core/lib/dal/src/tee_proof_generation_dal.rs index 4d19c3ff0c8b..e6b2df974b26 100644 --- a/core/lib/dal/src/tee_proof_generation_dal.rs +++ b/core/lib/dal/src/tee_proof_generation_dal.rs @@ -7,6 +7,7 @@ use zksync_db_connection::{ connection::Connection, error::DalResult, instrument::{InstrumentExt, Instrumented}, + interpolate_query, match_query_as, utils::pg_interval_from_duration, }; use zksync_types::{tee_types::TeeType, L1BatchNumber}; @@ -242,13 +243,16 @@ impl TeeProofGenerationDal<'_, '_> { batch_number: L1BatchNumber, tee_type: Option, ) -> DalResult> { - let query = format!( + let query = match_query_as!( + StorageTeeProof, + [ r#" SELECT tp.pubkey, tp.signature, tp.proof, tp.updated_at, + tp.status, ta.attestation FROM tee_proof_generation_details tp @@ -256,22 +260,22 @@ impl TeeProofGenerationDal<'_, '_> { tee_attestations ta ON tp.pubkey = ta.pubkey WHERE tp.l1_batch_number = $1 - AND tp.status = $2 - {} - ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC "#, - tee_type.map_or_else(String::new, |_| "AND tp.tee_type = $3".to_string()) + _, + "ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC" + ], + match(&tee_type) { + Some(tee_type) => + ("AND tp.tee_type = $2"; i64::from(batch_number.0), tee_type.to_string()), + None => (""; i64::from(batch_number.0)), + } ); - let mut query = sqlx::query_as(&query) - .bind(i64::from(batch_number.0)) - .bind(TeeProofGenerationJobStatus::Generated.to_string()); - - if let Some(tee_type) = tee_type { - query = query.bind(tee_type.to_string()); - } - - let proofs: Vec = query.fetch_all(self.storage.conn()).await.unwrap(); + let proofs = query + .instrument("get_tee_proofs") + .with_arg("l1_batch_number", &batch_number) + .fetch_all(self.storage) + .await?; Ok(proofs) } diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index b5d2b3276527..509dd8065281 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -878,6 +878,7 @@ pub struct TeeProof { #[serde_as(as = "Option")] pub proof: Option>, pub proved_at: DateTime, + pub status: String, #[serde_as(as = "Option")] pub attestation: Option>, } diff --git a/core/node/api_server/src/web3/namespaces/unstable.rs b/core/node/api_server/src/web3/namespaces/unstable.rs index 783088cdc36a..d323902b1dd4 100644 --- a/core/node/api_server/src/web3/namespaces/unstable.rs +++ b/core/node/api_server/src/web3/namespaces/unstable.rs @@ -42,7 +42,7 @@ impl UnstableNamespace { tee_type: Option, ) -> Result, Web3Error> { let mut storage = self.state.acquire_connection().await?; - Ok(storage + let proofs = storage .tee_proof_generation_dal() .get_tee_proofs(l1_batch_number, tee_type) .await @@ -55,8 +55,11 @@ impl UnstableNamespace { signature: proof.signature, proof: proof.proof, proved_at: DateTime::::from_naive_utc_and_offset(proof.updated_at, Utc), + status: proof.status, attestation: proof.attestation, }) - .collect::>()) + .collect::>(); + + Ok(proofs) } } diff --git a/core/node/proof_data_handler/src/tests.rs b/core/node/proof_data_handler/src/tests.rs index dae2ef8cd0c0..f02baeda0b16 100644 --- a/core/node/proof_data_handler/src/tests.rs +++ b/core/node/proof_data_handler/src/tests.rs @@ -112,7 +112,7 @@ async fn submit_tee_proof() { .await .expect("Failed to save attestation"); - // resend the same request; this time, it should be successful. + // resend the same request; this time, it should be successful let response = send_submit_tee_proof_request(&app, &uri, &tee_proof_request).await; assert_eq!(response.status(), StatusCode::OK);