feat(proxy-wasm) invoke 'on_http_call_response' on dispatch failures

Invoke `on_http_call_response` on dispatch connection failures. This
allows catching failures such as:

- timeout
- broken connection
- resolver failures
- TLS handshake failures
- Any other failures *after the connection has attempted to be
  established*

Dispatch failures occurring *before* a connection has attempted to be
established will not trigger `on_http_call_response` as they are already
supposed to trigger a Wasm exception (i.e. trap).

When a dispatch connection failure occurs, `on_http_call_response` is
invoked with: `on_http_call_response(call_id, 0, 0, 0)` (i.e. no header,
no body, no trailers). Calls to retrieve the response `:status` will
return `None`. A user may triage the connection failure by querying the
`:dispatch_status` pseudo-header:

```rust
fn on_http_call_response(
    &mut self,
    token_id: u32,
    nheaders: usize,
    body_size: usize,
    ntrailers: usize,
) {
    let dispatch_status = self.get_http_call_response_header(":dispatch_status");

    match dispatch_status.as_deref() {
        Some("timeout") => {},
        Some("broken connection") => {},
        Some("tls handshake failure") => {},
        Some("resolver failure") => {},
        Some("reader failure") => {},
        Some(s) => info!("dispatch_status: {}", s),
        None => {}
    }

    self.resume_http_request()
}
```

The socket status `"bad argument"` also exists but since the connection
has not been attempted yet, it won't show up during
`on_http_call_response` and is for internal use only (i.e. could be
added to the socket error log for example).

Fix #622.

src/common/ngx_wasm_socket_tcp.c

@@ -14,7 +14,7 @@
 
 
 static void ngx_wasm_socket_tcp_err(ngx_wasm_socket_tcp_t *sock,
-    const char *fmt, ...);
+    ngx_wasm_socket_status_e status, const char *fmt, ...);
 static void ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx);
 static ngx_int_t ngx_wasm_socket_tcp_connect_peer(ngx_wasm_socket_tcp_t *sock);
 static ngx_int_t ngx_wasm_socket_tcp_get_peer(ngx_peer_connection_t *pc,
@@ -36,14 +36,41 @@ static ngx_int_t ngx_wasm_socket_tcp_ssl_set_server_name(ngx_connection_t *c,
 #endif
 
 
+static ngx_str_t  ngx_wasm_socket_errlist[] = {
+    ngx_string("ok"),
+    ngx_string("bad argument"),
+    ngx_string("timeout"),
+    ngx_string("broken connection"),
+    ngx_string("tls handshake failure"),
+    ngx_string("resolver failure"),
+    ngx_string("reader failure"),
+    ngx_string("internal failure")
+};
+
+
+ngx_str_t *
+ngx_wasm_socket_tcp_status_strerror(ngx_wasm_socket_status_e status)
+{
+    ngx_str_t  *msg;
+
+    msg = ((ngx_uint_t) status < NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE)
+          ? &ngx_wasm_socket_errlist[status]
+          : &ngx_wasm_socket_errlist[NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE];
+
+    return msg;
+}
+
+
 static void
 ngx_wasm_socket_tcp_err(ngx_wasm_socket_tcp_t *sock,
+    ngx_wasm_socket_status_e status,
     const char *fmt, ...)
 {
     va_list   args;
     u_char   *p, *last;
 
     if (sock->err == NULL) {
+        sock->status = status;
         sock->err = ngx_pnalloc(sock->pool, NGX_MAX_ERROR_STR);
         if (sock->err == NULL) {
             return;
@@ -193,7 +220,8 @@ ngx_wasm_socket_tcp_init(ngx_wasm_socket_tcp_t *sock,
     sock->url.no_resolve = 1;
 
     if (ngx_parse_url(sock->pool, &sock->url) != NGX_OK) {
-        ngx_wasm_socket_tcp_err(sock, "%s", sock->url.err);
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BAD_ARGUMENT,
+                                "%s", sock->url.err);
         return NGX_ERROR;
     }
 
@@ -385,7 +413,8 @@ ngx_wasm_socket_tcp_connect(ngx_wasm_socket_tcp_t *sock)
     }
 
     if (rslv_ctx == NULL) {
-        ngx_wasm_socket_tcp_err(sock, "failed starting resolver");
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
+                                "failed starting resolver");
         return NGX_ERROR;
     }
 
@@ -429,7 +458,9 @@ ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx)
     if (ctx->state || !ctx->naddrs) {
 #if (NGX_WASM_LUA)
         if (ctx->state == NGX_WASM_LUA_RESOLVE_ERR) {
-            ngx_wasm_socket_tcp_err(sock, "lua resolver failed");
+            ngx_wasm_socket_tcp_err(sock,
+                                    NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
+                                    "lua resolver failed");
             goto error;
         }
 #endif
@@ -440,7 +471,8 @@ ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx)
         }
 #endif
 
-        ngx_wasm_socket_tcp_err(sock, "resolver error: %s",
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
+                                "resolver error: %s",
                                 ngx_resolver_strerror(ctx->state));
         goto error;
     }
@@ -571,12 +603,14 @@ ngx_wasm_socket_tcp_connect_peer(ngx_wasm_socket_tcp_t *sock)
         return NGX_ERROR;
 
     } else if (rc == NGX_BUSY) {
-        ngx_wasm_socket_tcp_err(sock, "no live connection");
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
+                                "no live connection");
         return NGX_BUSY;
 
     } else if (rc == NGX_DECLINED) {
         sock->socket_errno = ngx_socket_errno;
-        ngx_wasm_socket_tcp_err(sock, NULL);
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
+                                NULL);
         return NGX_ERROR;
     }
 
@@ -686,11 +720,13 @@ ngx_wasm_socket_tcp_ssl_handshake_handler(ngx_connection_t *c)
     }
 
     if (c->write->timedout) {
-        ngx_wasm_socket_tcp_err(sock, "tls handshake timed out");
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
+                                "tls handshake timed out");
         goto resume;
     }
 
-    ngx_wasm_socket_tcp_err(sock, "tls handshake failed");
+    ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
+                            "tls handshake failed");
 
 resume:
 
@@ -722,6 +758,7 @@ ngx_wasm_socket_tcp_ssl_handshake_done(ngx_connection_t *c)
         rc = SSL_get_verify_result(c->ssl->connection);
         if (rc != X509_V_OK) {
             ngx_wasm_socket_tcp_err(sock,
+                                    NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
                                     "tls certificate verify error: (%l:%s)",
                                     rc, X509_verify_cert_error_string(rc));
             return NGX_ERROR;
@@ -740,6 +777,7 @@ ngx_wasm_socket_tcp_ssl_handshake_done(ngx_connection_t *c)
 
         if (ngx_ssl_check_host(c, &sock->ssl_server_name) != NGX_OK) {
             ngx_wasm_socket_tcp_err(sock,
+                                    NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
                                     "tls certificate CN "
                                     "does not match \"%V\" sni",
                                     &sock->ssl_server_name);
@@ -809,6 +847,7 @@ ngx_wasm_socket_tcp_ssl_set_server_name(ngx_connection_t *c,
         || ngx_inet_addr(name->data, len) != INADDR_NONE)
     {
         ngx_wasm_socket_tcp_err(sock,
+                                NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
                                 "could not derive tls sni from host (\"%V\")",
                                 &sock->host);
         goto error;
@@ -877,7 +916,8 @@ ngx_wasm_socket_tcp_send(ngx_wasm_socket_tcp_t *sock, ngx_chain_t *cl)
     ngx_connection_t  *c;
 
     if (!sock->connected) {
-        ngx_wasm_socket_tcp_err(sock, "not connected");
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
+                                "not connected");
         return NGX_ERROR;
     }
 
@@ -930,7 +970,8 @@ ngx_wasm_socket_tcp_send(ngx_wasm_socket_tcp_t *sock, ngx_chain_t *cl)
     if (n == NGX_ERROR) {
         c->error = 1;
         sock->socket_errno = ngx_socket_errno;
-        ngx_wasm_socket_tcp_err(sock, NULL);
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
+                                NULL);
         return NGX_ERROR;
     }
 
@@ -1054,7 +1095,8 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,
     ngx_connection_t  *c;
 
     if (!sock->connected) {
-        ngx_wasm_socket_tcp_err(sock, "not connected");
+        ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
+                                "not connected");
         return NGX_ERROR;
     }
 
@@ -1091,7 +1133,9 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,
 
             rc = reader(sock, size, reader_ctx);
             if (rc == NGX_ERROR) {
-                ngx_wasm_socket_tcp_err(sock, "parser error");
+                ngx_wasm_socket_tcp_err(sock,
+                                        NGX_WASM_SOCKET_STATUS_READER_FAILURE,
+                                        "parser error");
                 return NGX_ERROR;
             }
 
@@ -1159,7 +1203,9 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,
 
         if (n == NGX_ERROR) {
             sock->socket_errno = ngx_socket_errno;
-            ngx_wasm_socket_tcp_err(sock, NULL);
+            ngx_wasm_socket_tcp_err(sock,
+                                    NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
+                                    NULL);
             return NGX_ERROR;
         }
 
@@ -1505,11 +1551,11 @@ ngx_wasm_socket_tcp_connect_handler(ngx_wasm_socket_tcp_t *sock)
                        "wasm tcp socket timed out connecting to \"%V:%ud\"",
                        &c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));
 
-        ngx_wasm_socket_tcp_err(sock, "timed out connecting to \"%V:%ud\"",
+        ngx_wasm_socket_tcp_err(sock,
+                                NGX_WASM_SOCKET_STATUS_TIMEOUT,
+                                "timed out connecting to \"%V:%ud\"",
                                 &c->addr_text,
                                 ngx_inet_get_port(sock->peer.sockaddr));
-
-        sock->timedout = 1;
         return;
     }
 
@@ -1520,7 +1566,9 @@ ngx_wasm_socket_tcp_connect_handler(ngx_wasm_socket_tcp_t *sock)
     if (rc != NGX_OK) {
         if (rc > 0) {
             sock->socket_errno = (ngx_err_t) rc;
-            ngx_wasm_socket_tcp_err(sock, NULL);
+            ngx_wasm_socket_tcp_err(sock,
+                                    NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
+                                    NULL);
         }
 
         return;
@@ -1560,11 +1608,11 @@ ngx_wasm_socket_tcp_send_handler(ngx_wasm_socket_tcp_t *sock)
                        "wasm tcp socket timed out writing to \"%V:%ud\"",
                        &c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));
 
-        ngx_wasm_socket_tcp_err(sock, "timed out writing to \"%V:%ud\"",
+        ngx_wasm_socket_tcp_err(sock,
+                                NGX_WASM_SOCKET_STATUS_TIMEOUT,
+                                "timed out writing to \"%V:%ud\"",
                                 &c->addr_text,
                                 ngx_inet_get_port(sock->peer.sockaddr));
-
-        sock->timedout = 1;
         return;
     }
 }
@@ -1584,11 +1632,11 @@ ngx_wasm_socket_tcp_receive_handler(ngx_wasm_socket_tcp_t *sock)
                        "wasm tcp socket timed out reading from \"%V:%ud\"",
                        &c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));
 
-        ngx_wasm_socket_tcp_err(sock, "timed out reading from \"%V:%ud\"",
+        ngx_wasm_socket_tcp_err(sock,
+                                NGX_WASM_SOCKET_STATUS_TIMEOUT,
+                                "timed out reading from \"%V:%ud\"",
                                 &c->addr_text,
                                 ngx_inet_get_port(sock->peer.sockaddr));
-
-        sock->timedout = 1;
         return;
     }
 

src/common/ngx_wasm_socket_tcp.h

@@ -18,6 +18,20 @@ typedef ngx_int_t (*ngx_wasm_socket_tcp_dns_resolver_pt)(
     ngx_resolver_ctx_t *rslv_ctx);
 
 
+typedef enum {
+    NGX_WASM_SOCKET_STATUS_OK = 0,
+    NGX_WASM_SOCKET_STATUS_BAD_ARGUMENT,
+    NGX_WASM_SOCKET_STATUS_TIMEOUT,
+    NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
+#if (NGX_SSL)
+    NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
+#endif
+    NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
+    NGX_WASM_SOCKET_STATUS_READER_FAILURE,
+    NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
+} ngx_wasm_socket_status_e;
+
+
 typedef struct {
     ngx_str_t                                host;
     in_port_t                                port;
@@ -39,6 +53,7 @@ struct ngx_wasm_socket_tcp_s {
     ngx_wasm_subsys_env_t                   *env;
 
     ngx_wasm_socket_tcp_resume_handler_pt    resume_handler;
+    ngx_wasm_socket_status_e                 status;
     void                                    *data;
 #if (NGX_WASM_LUA)
     ngx_wasm_lua_ctx_t                      *lctx;
@@ -85,19 +100,19 @@ struct ngx_wasm_socket_tcp_s {
 
     /* flags */
 
-    unsigned                                 timedout:1;
     unsigned                                 connected:1;
     unsigned                                 eof:1;
     unsigned                                 closed:1;
     unsigned                                 read_closed:1;
     unsigned                                 write_closed:1;
-
 #if (NGX_SSL)
     unsigned                                 ssl_ready:1;
 #endif
 };
 
 
+ngx_str_t *ngx_wasm_socket_tcp_status_strerror(ngx_wasm_socket_status_e status);
+
 ngx_int_t ngx_wasm_socket_tcp_init(ngx_wasm_socket_tcp_t *sock,
     ngx_str_t *host, unsigned tls, ngx_str_t *sni, ngx_wasm_subsys_env_t *env);
 ngx_int_t ngx_wasm_socket_tcp_connect(ngx_wasm_socket_tcp_t *sock);

src/common/proxy_wasm/ngx_proxy_wasm_maps.c

@@ -35,6 +35,8 @@ static ngx_str_t *ngx_proxy_wasm_maps_get_response_status(
 static ngx_int_t ngx_proxy_wasm_maps_set_response_status(
     ngx_wavm_instance_t *instance, ngx_str_t *value,
     ngx_proxy_wasm_map_type_e map_type);
+static ngx_str_t *ngx_proxy_wasm_maps_get_dispatch_response_status(
+    ngx_wavm_instance_t *instance, ngx_proxy_wasm_map_type_e map_type);
 static ngx_str_t *ngx_proxy_wasm_maps_get_dispatch_status(
     ngx_wavm_instance_t *instance, ngx_proxy_wasm_map_type_e map_type);
 #endif
@@ -75,6 +77,11 @@ static ngx_proxy_wasm_maps_key_t  ngx_proxy_wasm_maps_special_keys[] = {
     /* dispatch response */
 
     { ngx_string(":status"),
+      NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS,
+      ngx_proxy_wasm_maps_get_dispatch_response_status,
+      NULL },
+
+    { ngx_string(":dispatch_status"),
       NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS,
       ngx_proxy_wasm_maps_get_dispatch_status,
       NULL },
@@ -118,8 +125,14 @@ ngx_proxy_wasm_maps_get_map(ngx_wavm_instance_t *instance,
         }
 
         reader = &call->http_reader;
+        r = &reader->fake_r;
+
+        if (!r->signature) {
+            /* reader not initialized */
+            return NULL;
+        }
 
-        return &reader->fake_r.upstream->headers_in.headers;
+        return &r->upstream->headers_in.headers;
 #endif
 
     default:
@@ -753,7 +766,7 @@ ngx_proxy_wasm_maps_set_response_status(ngx_wavm_instance_t *instance,
 
 
 static ngx_str_t *
-ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
+ngx_proxy_wasm_maps_get_dispatch_response_status(ngx_wavm_instance_t *instance,
     ngx_proxy_wasm_map_type_e map_type)
 {
     ngx_uint_t                       status;
@@ -771,9 +784,8 @@ ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
 
     /* status */
 
-    ngx_wa_assert(reader->fake_r.upstream);
-
     if (reader->fake_r.upstream == NULL) {
+        /* response not received */
         return NULL;
     }
 
@@ -810,4 +822,22 @@ ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
 
     return &pwctx->call_status;
 }
+
+
+static ngx_str_t *
+ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
+    ngx_proxy_wasm_map_type_e map_type)
+{
+    ngx_proxy_wasm_exec_t           *pwexec;
+    ngx_http_proxy_wasm_dispatch_t  *call;
+    ngx_wasm_socket_tcp_t           *sock;
+
+    ngx_wa_assert(map_type == NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS);
+
+    pwexec = ngx_proxy_wasm_instance2pwexec(instance);
+    call = pwexec->call;
+    sock = &call->sock;
+
+    return ngx_wasm_socket_tcp_status_strerror(sock->status);
+}
 #endif