filter_kubernetes: add kubernetes_namespace metadata (#8279)

- filter_kubernetes: add kubernetes_namespace metadata

Closes #6544
Addresses #3865

- filter_kubernetes: Add Tests for namespace labels & annotations
- also updates error text for kubelet vs kube api upstream errors
- get_namespace_api_server_info passes in meta->namespace
  instead of ctx->namespace which may not be set at time of call

filter_kubernetes: make pod meta fetching optional

now that you can fetch namespace labels/annotations OR
pod labels/annotations, do not attempt to fetch pod
labels/annotations if neither of them are requested via config

- fix cache for namespace meta

---------

Signed-off-by: ryanohnemus <[email protected]>

plugins/filter_kubernetes/kube_conf.c

@@ -89,17 +89,6 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
     if (ctx->use_tag_for_meta) {
         ctx->api_https = FLB_FALSE;
     }
-    else if (ctx->use_kubelet) {
-        ctx->api_host = flb_strdup(ctx->kubelet_host);
-        ctx->api_port = ctx->kubelet_port;
-        ctx->api_https = FLB_TRUE;
-
-        /* This is for unit test diagnostic purposes */
-        if (ctx->meta_preload_cache_dir) {
-            ctx->api_https = FLB_FALSE;
-        }
-
-    }
     else if (!url) {
         ctx->api_host = flb_strdup(FLB_API_HOST);
         ctx->api_port = FLB_API_PORT;
@@ -136,11 +125,6 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
         }
     }
 
-    snprintf(ctx->kube_url, sizeof(ctx->kube_url) - 1,
-             "%s://%s:%i",
-             ctx->api_https ? "https" : "http",
-             ctx->api_host, ctx->api_port);
-
     if (ctx->kube_meta_cache_ttl > 0) {
         ctx->hash_table = flb_hash_table_create_with_ttl(ctx->kube_meta_cache_ttl,
                                                          FLB_HASH_TABLE_EVICT_OLDER,
@@ -153,7 +137,22 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
                                                 FLB_HASH_TABLE_SIZE);
     }
 
-    if (!ctx->hash_table) {
+    if (ctx->kube_meta_namespace_cache_ttl > 0) {
+        ctx->namespace_hash_table = flb_hash_table_create_with_ttl(
+                                            ctx->kube_meta_namespace_cache_ttl,
+                                            FLB_HASH_TABLE_EVICT_OLDER,
+                                            FLB_HASH_TABLE_SIZE,
+                                            FLB_HASH_TABLE_SIZE);
+    }
+    else {
+        ctx->namespace_hash_table = flb_hash_table_create(
+                                            FLB_HASH_TABLE_EVICT_RANDOM,
+                                            FLB_HASH_TABLE_SIZE,
+                                            FLB_HASH_TABLE_SIZE);
+    }
+
+
+    if (!ctx->hash_table || !ctx->namespace_hash_table) {
         flb_kube_conf_destroy(ctx);
         return NULL;
     }
@@ -203,6 +202,10 @@ void flb_kube_conf_destroy(struct flb_kube *ctx)
         flb_hash_table_destroy(ctx->hash_table);
     }
 
+    if (ctx->namespace_hash_table) {
+        flb_hash_table_destroy(ctx->namespace_hash_table);
+    }
+
     if (ctx->merge_log == FLB_TRUE) {
         flb_free(ctx->unesc_buf);
     }
@@ -218,14 +221,20 @@ void flb_kube_conf_destroy(struct flb_kube *ctx)
     flb_free(ctx->podname);
     flb_free(ctx->auth);
 
-    if (ctx->upstream) {
-        flb_upstream_destroy(ctx->upstream);
+    if (ctx->kubelet_upstream) {
+        flb_upstream_destroy(ctx->kubelet_upstream);
+    }
+    if (ctx->kube_api_upstream) {
+        flb_upstream_destroy(ctx->kube_api_upstream);
     }
 
 #ifdef FLB_HAVE_TLS
     if (ctx->tls) {
         flb_tls_destroy(ctx->tls);
     }
+    if (ctx->kubelet_tls) {
+        flb_tls_destroy(ctx->kubelet_tls);
+    }
 #endif
 
     flb_free(ctx);

plugins/filter_kubernetes/kube_conf.h

@@ -70,13 +70,12 @@ struct kube_meta;
 /* Filter context */
 struct flb_kube {
     /* Configuration parameters */
-    char *api_host;
-    int api_port;
-    int api_https;
     int use_journal;
     int cache_use_docker_id;
     int labels;
     int annotations;
+    int namespace_labels;
+    int namespace_annotations;
     int dummy_meta;
     int tls_debug;
     int tls_verify;
@@ -113,7 +112,9 @@ struct flb_kube {
     int keep_log;
 
     /* API Server end point */
-    char kube_url[1024];
+    char *api_host;
+    int api_port;
+    int api_https;
 
     /* Kubernetes tag prefix */
     flb_sds_t kube_tag_prefix;
@@ -158,12 +159,16 @@ struct flb_kube {
     int kubelet_port;
 
     int kube_meta_cache_ttl;
+    int kube_meta_namespace_cache_ttl;
 
     struct flb_tls *tls;
+    struct flb_tls *kubelet_tls;
 
     struct flb_config *config;
     struct flb_hash_table *hash_table;
-    struct flb_upstream *upstream;
+    struct flb_hash_table *namespace_hash_table;
+    struct flb_upstream *kubelet_upstream;
+    struct flb_upstream *kube_api_upstream;
     struct flb_filter_instance *ins;
 };