feat(tee): add error handling for unstable_getTeeProofs API endpoint

This PR adds more information to the response of the
`unstable_getTeeProofs` API endpoint, enabling the [client][1] that sent
the [request][2] to determine whether it makes sense to retry fetching
the TEE proof for a particular batch number.

Currently, the [TEE verifier][1] – the tool for continuous SGX
attestation and batch signature verification – is [stuck][3] on batches
that failed to be proven and are marked as `permanently_ignored`. The
tool should be able to distinguish between batches that are permanently
ignored (and should be skipped) and batches that have failed but will be
retried. This PR enables that distinction.

Example use cases:
- requesting TEE proof for a batch with the `permanently_ignored` status
  ```
  $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [14, "sgx"] }' 'http://localhost:3152'
  HTTP/1.1 200 OK
  content-type: application/json; charset=utf-8
  vary: origin, access-control-request-method, access-control-request-headers
  access-control-allow-origin: *
  content-length: 207
  date: Tue, 26 Nov 2024 12:42:48 GMT

  {"jsonrpc":"2.0","result":[{"l1BatchNumber":14,"teeType":"sgx","pubkey":null,"signature":null,"proof":null,"provedAt":"2024-11-20T15:43:46.112146Z","status":"permanently_ignored","attestation":null}],"id":1}
  ```
- requesting TEE proof for a batch with the `failed` status
  ```
  $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [15, "sgx"] }' 'http://localhost:3152'
  HTTP/1.1 200 OK
  content-type: application/json; charset=utf-8
  vary: origin, access-control-request-method, access-control-request-headers
  access-control-allow-origin: *
  content-length: 194
  date: Tue, 26 Nov 2024 12:44:19 GMT

  {"jsonrpc":"2.0","result":[{"l1BatchNumber":15,"teeType":"sgx","pubkey":null,"signature":null,"proof":null,"provedAt":"2024-11-20T15:43:46.121432Z","status":"failed","attestation":null}],"id":1}
  ```
- requesting TEE proof for a batch with the `generated` status
  ```
  $ curl -i  -X POST -H "Content-Type: application/json"     --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [28, "sgx"] }' 'http://localhost:3152'
  HTTP/1.1 200 OK
  content-type: application/json; charset=utf-8
  vary: origin, access-control-request-method, access-control-request-headers
  access-control-allow-origin: *
  content-length: 229
  date: Tue, 26 Nov 2024 12:45:27 GMT

  {"jsonrpc":"2.0","result":[{"l1BatchNumber":28,"teeType":"sgx","pubkey":"0506070809","signature":"0001020304","proof":"1011121314","provedAt":"2024-11-20T15:21:16.129128Z","status":"generated","attestation":"0403020100"}],"id":1}
  ```
- requesting TEE proof for a non-existent batch
  ```
  $ curl -i -X POST -H "Content-Type: application/json" --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [1337, "sgx"] }' 'http://localhost:3152'
  HTTP/1.1 200 OK
  content-type: application/json; charset=utf-8
  vary: origin, access-control-request-method, access-control-request-headers
  access-control-allow-origin: *
  content-length: 36
  date: Tue, 26 Nov 2024 12:46:08 GMT

  {"jsonrpc":"2.0","result":[],"id":1}
  ```

Relevant database entries for the use cases mentioned above:
```
zksync_server_localhost_legacy=# SELECT * FROM tee_proof_generation_details WHERE l1_batch_number IN (14, 15, 28, 1337);
 l1_batch_number |       status        |  signature   |    pubkey    |    proof     | tee_type |         created_at         |         updated_at         |      prover_taken_at
-----------------+---------------------+--------------+--------------+--------------+----------+----------------------------+----------------------------+----------------------------
              14 | permanently_ignored |              |              |              | sgx      | 2023-11-20 15:27:47.281293 | 2024-11-20 15:43:46.112146 | 2024-11-20 15:43:46.106042
              15 | failed              |              |              |              | sgx      | 2024-11-20 15:27:47.287777 | 2024-11-20 15:43:46.121432 | 2024-11-20 15:43:46.115853
              28 | generated           | \x0001020304 | \x0506070809 | \x1011121314 | sgx      | 2024-11-20 12:56:33.055642 | 2024-11-20 15:21:16.129128 | 2024-11-20 14:53:14.25949
(3 rows)
```

[1]: https://github.com/matter-labs/teepot/blob/main/bin/verify-era-proof-attestation/src/main.rs
[2]: https://github.com/matter-labs/teepot/blob/1a8a9f17fa7284f83c41a63d37fe380aef6d550d/bin/verify-era-proof-attestation/src/proof.rs#L15-L21
[3]: https://grafana.matterlabs.dev/goto/unFqf57Hg?orgId=1

core/lib/dal/src/models/storage_tee_proof.rs

@@ -9,6 +9,7 @@ pub struct StorageTeeProof {
     pub signature: Option<Vec<u8>>,
     pub proof: Option<Vec<u8>>,
     pub updated_at: NaiveDateTime,
+    pub status: String,
     pub attestation: Option<Vec<u8>>,
 }
 

core/lib/dal/src/tee_proof_generation_dal.rs

@@ -7,6 +7,7 @@ use zksync_db_connection::{
     connection::Connection,
     error::DalResult,
     instrument::{InstrumentExt, Instrumented},
+    interpolate_query, match_query_as,
     utils::pg_interval_from_duration,
 };
 use zksync_types::{tee_types::TeeType, L1BatchNumber};
@@ -242,36 +243,39 @@ impl TeeProofGenerationDal<'_, '_> {
         batch_number: L1BatchNumber,
         tee_type: Option<TeeType>,
     ) -> DalResult<Vec<StorageTeeProof>> {
-        let query = format!(
+        let query = match_query_as!(
+            StorageTeeProof,
+            [
             r#"
             SELECT
                 tp.pubkey,
                 tp.signature,
                 tp.proof,
                 tp.updated_at,
+                tp.status,
                 ta.attestation
             FROM
                 tee_proof_generation_details tp
             LEFT JOIN
                 tee_attestations ta ON tp.pubkey = ta.pubkey
             WHERE
                 tp.l1_batch_number = $1
-                AND tp.status = $2
-                {}
-            ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC
             "#,
-            tee_type.map_or_else(String::new, |_| "AND tp.tee_type = $3".to_string())
+            _,
+            "ORDER BY tp.l1_batch_number ASC, tp.tee_type ASC"
+            ],
+            match(&tee_type) {
+                Some(tee_type) =>
+                    ("AND tp.tee_type = $2"; i64::from(batch_number.0), tee_type.to_string()),
+                None => (""; i64::from(batch_number.0)),
+            }
         );
 
-        let mut query = sqlx::query_as(&query)
-            .bind(i64::from(batch_number.0))
-            .bind(TeeProofGenerationJobStatus::Generated.to_string());
-
-        if let Some(tee_type) = tee_type {
-            query = query.bind(tee_type.to_string());
-        }
-
-        let proofs: Vec<StorageTeeProof> = query.fetch_all(self.storage.conn()).await.unwrap();
+        let proofs = query
+            .instrument("get_tee_proofs")
+            .with_arg("l1_batch_number", &batch_number)
+            .fetch_all(self.storage)
+            .await?;
 
         Ok(proofs)
     }

core/lib/types/src/api/mod.rs

@@ -878,6 +878,7 @@ pub struct TeeProof {
     #[serde_as(as = "Option<Hex>")]
     pub proof: Option<Vec<u8>>,
     pub proved_at: DateTime<Utc>,
+    pub status: String,
     #[serde_as(as = "Option<Hex>")]
     pub attestation: Option<Vec<u8>>,
 }

core/node/api_server/src/web3/namespaces/unstable.rs

@@ -42,7 +42,7 @@ impl UnstableNamespace {
         tee_type: Option<TeeType>,
     ) -> Result<Vec<TeeProof>, Web3Error> {
         let mut storage = self.state.acquire_connection().await?;
-        Ok(storage
+        let proofs = storage
             .tee_proof_generation_dal()
             .get_tee_proofs(l1_batch_number, tee_type)
             .await
@@ -55,8 +55,11 @@ impl UnstableNamespace {
                 signature: proof.signature,
                 proof: proof.proof,
                 proved_at: DateTime::<Utc>::from_naive_utc_and_offset(proof.updated_at, Utc),
+                status: proof.status,
                 attestation: proof.attestation,
             })
-            .collect::<Vec<_>>())
+            .collect::<Vec<_>>();
+
+        Ok(proofs)
     }
 }

core/node/proof_data_handler/src/tests.rs

@@ -112,7 +112,7 @@ async fn submit_tee_proof() {
         .await
         .expect("Failed to save attestation");
 
-    // resend the same request; this time, it should be successful.
+    // resend the same request; this time, it should be successful
 
     let response = send_submit_tee_proof_request(&app, &uri, &tee_proof_request).await;
     assert_eq!(response.status(), StatusCode::OK);