From 4acd34439961b155b77b0ba86103a9f1ef5e02a5 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 24 Oct 2023 17:53:39 +0900 Subject: [PATCH] in_docker: Implement a capability for cgroups V2 Signed-off-by: Hiroshi Hatake --- plugins/in_docker/CMakeLists.txt | 1 + plugins/in_docker/cgroup_v2.c | 446 +++++++++++++++++++++++++++++++ plugins/in_docker/docker.c | 35 ++- plugins/in_docker/docker.h | 29 ++ 4 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 plugins/in_docker/cgroup_v2.c diff --git a/plugins/in_docker/CMakeLists.txt b/plugins/in_docker/CMakeLists.txt index 01cf3a848aa..4069fb08fb2 100644 --- a/plugins/in_docker/CMakeLists.txt +++ b/plugins/in_docker/CMakeLists.txt @@ -1,6 +1,7 @@ set(src docker.c cgroup_v1.c + cgroup_v2.c ) FLB_PLUGIN(in_docker "${src}" "") diff --git a/plugins/in_docker/cgroup_v2.c b/plugins/in_docker/cgroup_v2.c new file mode 100644 index 00000000000..bc0c6732ce4 --- /dev/null +++ b/plugins/in_docker/cgroup_v2.c @@ -0,0 +1,446 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2023 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include "docker.h" + +/* This method returns list of currently running docker ids. */ +static struct mk_list *get_active_dockers() +{ + DIR *dp; + struct dirent *ep; + struct mk_list *list; + docker_info *docker; + char *p = NULL; + char *container_id = NULL; + + list = flb_malloc(sizeof(struct mk_list)); + if (!list) { + flb_errno(); + return NULL; + } + mk_list_init(list); + + dp = opendir(DOCKER_CGROUP_V2_DOCKER_SERVICE_DIR); + if (dp != NULL) { + ep = readdir(dp); + + while(ep != NULL) { + if (ep->d_type == OS_DIR_TYPE) { + if (strcmp(ep->d_name, CURRENT_DIR) != 0 + && strcmp(ep->d_name, PREV_DIR) != 0 + && strlen(ep->d_name) == DOCKER_CGROUP_V2_LONG_ID_LEN) { /* precautionary check */ + + p = strstr(ep->d_name, "-"); + if (p == NULL) { + continue; + } + /* get rid of the prefix "-" and the suffix ".scope" */ + container_id = strtok(p+1, "."); + if (container_id != NULL) { + docker = in_docker_init_docker_info(container_id); + mk_list_add(&docker->_head, list); + } + + } + } + ep = readdir(dp); + } + closedir(dp); + } + + return list; +} + +static char *read_line(FILE *fin) +{ + char *buffer; + char *tmp; + int read_chars = 0; + int bufsize = 1215; + char *line; + + line = (char *) flb_calloc(bufsize, sizeof(char)); + if (!line) { + flb_errno(); + return NULL; + } + + buffer = line; + + while (fgets(buffer, bufsize - read_chars, fin)) { + read_chars = strlen(line); + + if (line[read_chars - 1] == '\n') { + line[read_chars - 1] = '\0'; + return line; + } + else { + bufsize = 2 * bufsize; + tmp = flb_realloc(line, bufsize); + if (!tmp) { + flb_errno(); + return NULL; + } + else { + line = tmp; + buffer = line + read_chars; + } + } + } + + flb_free(line); + return NULL; +} + +/* This routine returns path to docker's cgroup CPU usage file. */ +static char *get_cpu_used_file(char *id) +{ + char *path; + + if (!id) { + return NULL; + } + + path = (char *) flb_calloc(115, sizeof(char)); + if (!path) { + flb_errno(); + return NULL; + } + + strcat(path, DOCKER_CGROUP_V2_DOCKER_SERVICE_DIR); + strcat(path, "/"); + strcat(path, "docker-"); + strcat(path, id); + strcat(path, ".scope"); + strcat(path, "/"); + strcat(path, DOCKER_CGROUP_V2_CPU_USAGE_FILE); + + return path; +} + +/* This routine returns path to docker's cgroup memory limit file. */ +static char *get_mem_limit_file(char *id) +{ + char *path; + + if (!id) { + return NULL; + } + + path = (char *) flb_calloc(121, sizeof(char)); + if (!path) { + flb_errno(); + return NULL; + } + strcat(path, DOCKER_CGROUP_V2_DOCKER_SERVICE_DIR); + strcat(path, "/"); + strcat(path, "docker-"); + strcat(path, id); + strcat(path, ".scope"); + strcat(path, "/"); + strcat(path, DOCKER_CGROUP_V2_MEM_MAX_FILE); + + return path; +} + +/* This routine returns path to docker's cgroup memory used file. */ +static char *get_mem_used_file(char *id) +{ + char *path; + + if (!id) { + return NULL; + } + + path = (char *) flb_calloc(121, sizeof(char)); + if (!path) { + flb_errno(); + return NULL; + } + strcat(path, DOCKER_CGROUP_V2_DOCKER_SERVICE_DIR); + strcat(path, "/"); + strcat(path, "docker-"); + strcat(path, id); + strcat(path, ".scope"); + strcat(path, "/"); + strcat(path, DOCKER_CGROUP_V2_MEM_USAGE_FILE); + + return path; +} + +static char *get_config_file(char *id) +{ + char *path; + + if (!id) { + return NULL; + } + + path = (char *) flb_calloc(107, sizeof(char)); + if (!path) { + flb_errno(); + return NULL; + } + strcat(path, DOCKER_LIB_ROOT); + strcat(path, "/"); + strcat(path, id); + strcat(path, "/"); + strcat(path, DOCKER_CONFIG_JSON); + + return path; +} + +static char *extract_name(char *line, char *start) +{ + int skip = 9; + int len = 0; + char *name; + char buff[256]; + char *curr; + + if (start != NULL) { + curr = start + skip; + while (*curr != '"') { + buff[len++] = *curr; + curr++; + } + + if (len > 0) { + name = (char *) flb_calloc(len + 1, sizeof(char)); + if (!name) { + flb_errno(); + return NULL; + } + memcpy(name, buff, len); + + return name; + } + } + + return NULL; +} + +static char *get_container_name(struct flb_docker *ctx, char *id) +{ + char *container_name = NULL; + char *config_file; + FILE *f = NULL; + char *line; + + config_file = get_config_file(id); + if (!config_file) { + return NULL; + } + + f = fopen(config_file, "r"); + if (!f) { + flb_errno(); + flb_plg_error(ctx->ins, "cannot open %s", config_file); + flb_free(config_file); + return NULL; + } + + while ((line = read_line(f))) { + char *index = strstr(line, DOCKER_NAME_ARG); + if (index != NULL) { + container_name = extract_name(line, index); + flb_free(line); + break; + } + flb_free(line); + } + + flb_free(config_file); + fclose(f); + + return container_name; +} + +/* Returns CPU metrics for docker id. */ +static cpu_snapshot *get_docker_cpu_snapshot(struct flb_docker *ctx, char *id) +{ + int c; + unsigned long cpu_used = 0; + char *usage_file; + cpu_snapshot *snapshot = NULL; + FILE *f; + char *line = NULL; + + usage_file = get_cpu_used_file(id); + if (!usage_file) { + return NULL; + } + + f = fopen(usage_file, "r"); + if (!f) { + flb_errno(); + flb_plg_error(ctx->ins, "error gathering CPU data from %s", + usage_file); + flb_free(usage_file); + return NULL; + } + + /* Read the content */ + while ((line = read_line(f))) { + if (strncmp(line, DOCKER_CGROUP_V2_CPU_USAGE_KEY, 10) == 0) { + c = sscanf(line, DOCKER_CGROUP_V2_CPU_USAGE_TEMPLATE, &cpu_used); + if (c != 1) { + flb_plg_error(ctx->ins, "error scanning used CPU value from %s with key = %s", + usage_file, DOCKER_CGROUP_V2_CPU_USAGE_KEY); + flb_free(usage_file); + fclose(f); + return NULL; + } + flb_free(line); + + break; + } + flb_free(line); + } + + snapshot = (cpu_snapshot *) flb_calloc(1, sizeof(cpu_snapshot)); + if (!snapshot) { + flb_errno(); + fclose(f); + flb_free(usage_file); + return NULL; + } + + snapshot->used = cpu_used; + + flb_free(usage_file); + fclose(f); + return snapshot; +} + +static uint64_t read_file_uint64(struct flb_docker *ctx, flb_sds_t path) +{ + int c; + uint64_t value = UINT64_MAX; + FILE *fp; + + fp = fopen(path, "r"); + if (!fp) { + flb_errno(); + flb_plg_warn(ctx->ins, "Failed to read %s", path); + return value; + } + + c = fscanf(fp, "%lu", &value); + fclose(fp); + if (c != 1) { + flb_plg_warn(ctx->ins, "Failed to read a number from %s", path); + return value; + } + + return value; +} + +/* Returns memory used by a docker in bytes. */ +static uint64_t get_docker_mem_used(struct flb_docker *ctx, char *id) +{ + char *usage_file = NULL; + uint64_t mem_used = 0; + + usage_file = get_mem_used_file(id); + if (!usage_file) { + return 0; + } + + mem_used = read_file_uint64(ctx, usage_file); + flb_free(usage_file); + + return mem_used; +} + +/* Returns memory limit for a docker in bytes. */ +static uint64_t get_docker_mem_limit(struct flb_docker *ctx, char *id) +{ + int c; + char *limit_file = get_mem_limit_file(id); + uint64_t mem_limit; + char *line = NULL; + FILE *f; + + if (!limit_file) { + return 0; + } + + f = fopen(limit_file, "r"); + if (!f) { + flb_errno(); + flb_free(limit_file); + return 0; + } + + while ((line = read_line(f))) { + if (strncmp(line, "max", 3) == 0) { + mem_limit = UINT64_MAX; + } + else { + c = sscanf(line, "%lu", &mem_limit); + if (c != 1) { + flb_plg_error(ctx->ins, "error scanning used mem_limit from %s", + limit_file); + flb_free(limit_file); + fclose(f); + return 0; + } + } + flb_free(line); + } + + flb_free(limit_file); + fclose(f); + + return mem_limit; +} + +/* Get memory snapshot for a docker id. */ +static mem_snapshot *get_docker_mem_snapshot(struct flb_docker *ctx, char *id) +{ + mem_snapshot *snapshot = NULL; + + snapshot = (mem_snapshot *) flb_calloc(1, sizeof(mem_snapshot)); + if (!snapshot) { + flb_errno(); + return NULL; + } + + snapshot->used = get_docker_mem_used(ctx, id); + snapshot->limit = get_docker_mem_limit(ctx, id); + + return snapshot; +} + +int in_docker_set_cgroup_api_v2(struct cgroup_api *api) +{ + api->cgroup_version = 2; + api->get_active_docker_ids = get_active_dockers; + api->get_container_name = get_container_name; + api->get_cpu_snapshot = get_docker_cpu_snapshot; + api->get_mem_snapshot = get_docker_mem_snapshot; + + return 0; +} diff --git a/plugins/in_docker/docker.c b/plugins/in_docker/docker.c index 135c9f6b4dd..8b86392fe94 100644 --- a/plugins/in_docker/docker.c +++ b/plugins/in_docker/docker.c @@ -278,6 +278,17 @@ static struct mk_list *apply_filters(struct flb_docker *ctx, return filtered; } +/* + * Calculate which cgroup version is used on host by checing existence of + * cgroup.controllers file (if it exists, it is V2). + */ +static int get_cgroup_version(struct flb_docker *ctx) +{ + char path[SYSFS_FILE_PATH_SIZE]; + snprintf(path, sizeof(path), "%s/%s", ctx->sysfs_path, CGROUP_V2_PATH); + return (access(path, F_OK) == 0) ? CGROUP_V2 : CGROUP_V1; +} + /* Init Docker input */ static int cb_docker_init(struct flb_input_instance *in, struct flb_config *config, void *data) @@ -292,26 +303,37 @@ static int cb_docker_init(struct flb_input_instance *in, return -1; } ctx->ins = in; - in_docker_set_cgroup_api_v1(&ctx->cgroup_api); /* TODO: support cgroup v2*/ init_filter_lists(in, ctx); /* Set the context */ flb_input_set_context(in, ctx); - + /* Load the config map */ ret = flb_input_config_map_set(in, (void *)ctx); if (ret == -1) { flb_free(ctx); flb_plg_error(in, "unable to load configuration."); return -1; - } - + } + if (ctx->interval_sec <= 0 && ctx->interval_nsec <= 0) { ctx->interval_sec = atoi(DEFAULT_INTERVAL_SEC); ctx->interval_nsec = atoi(DEFAULT_INTERVAL_NSEC); } + /* Detect cgroups version v2 or v1 */ + if (get_cgroup_version(ctx) == CGROUP_V2) { + flb_plg_info(ctx->ins, "Detected cgroups v2"); + in_docker_set_cgroup_api_v2(&ctx->cgroup_api); + ctx->cgroup_version = CGROUP_V2; + } + else { + flb_plg_info(ctx->ins, "Detected cgroups v1"); + in_docker_set_cgroup_api_v1(&ctx->cgroup_api); + ctx->cgroup_version = CGROUP_V1; + } + /* Set our collector based on time, CPU usage every 1 second */ ret = flb_input_set_collector_time(in, cb_docker_collect, ctx->interval_sec, @@ -541,6 +563,11 @@ static struct flb_config_map config_map[] = { 0, FLB_FALSE, 0, "A space-separated list of containers to exclude" }, + { + FLB_CONFIG_MAP_STR, "path.sysfs", SYSFS_PATH, + 0, FLB_TRUE, offsetof(struct flb_docker, sysfs_path), + "sysfs mount point" + }, /* EOF */ {0} }; diff --git a/plugins/in_docker/docker.h b/plugins/in_docker/docker.h index d3814c39050..79695d9c6d1 100644 --- a/plugins/in_docker/docker.h +++ b/plugins/in_docker/docker.h @@ -26,17 +26,39 @@ #include #include +/* Distinguish cgroup v2 or v1 */ +#define SYSFS_FILE_PATH_SIZE 512 + +#define SYSFS_PATH "/sys/fs/cgroup" +#define CGROUP_V2_PATH "cgroup.controllers" +#define CGROUP_V1 1 +#define CGROUP_V2 2 #define CURRENT_DIR "." #define PREV_DIR ".." #define OS_DIR_TYPE 4 #define DOCKER_LONG_ID_LEN 64 #define DOCKER_SHORT_ID_LEN 12 + +#define DOCKER_CGROUP_V2_LONG_ID_LEN 77 /* docker-CONTAINERID.scope: 7 + 64 + 6 */ + +/* Files from sysfs containing metrics (cgroups v1) */ #define DOCKER_CGROUP_V1_MEM_DIR "/sys/fs/cgroup/memory/docker" #define DOCKER_CGROUP_V1_CPU_DIR "/sys/fs/cgroup/cpu/docker" #define DOCKER_CGROUP_V1_MEM_LIMIT_FILE "memory.limit_in_bytes" #define DOCKER_CGROUP_V1_MEM_USAGE_FILE "memory.usage_in_bytes" #define DOCKER_CGROUP_V1_CPU_USAGE_FILE "cpuacct.usage" + +/* Files from sysfs containing metrics (cgroups v2) */ +#define DOCKER_CGROUP_V2_DOCKER_SERVICE_DIR "/sys/fs/cgroup/system.slice" +#define DOCKER_CGROUP_V2_MEM_USAGE_FILE "memory.current" +#define DOCKER_CGROUP_V2_MEM_PEAK_FILE "memory.peak" +#define DOCKER_CGROUP_V2_MEM_STAT_FILE "memory.stat" +#define DOCKER_CGROUP_V2_MEM_MAX_FILE "memory.max" +#define DOCKER_CGROUP_V2_CPU_USAGE_FILE "cpu.stat" +#define DOCKER_CGROUP_V2_CPU_USAGE_KEY "usage_usec" +#define DOCKER_CGROUP_V2_CPU_USAGE_TEMPLATE DOCKER_CGROUP_V2_CPU_USAGE_KEY" %lu" + #define DOCKER_LIB_ROOT "/var/lib/docker/containers" #define DOCKER_CONFIG_JSON "config.v2.json" #define DOCKER_NAME_ARG "\"Name\"" @@ -75,6 +97,7 @@ struct cgroup_api { mem_snapshot* (*get_mem_snapshot) (struct flb_docker *, char *); }; int in_docker_set_cgroup_api_v1(struct cgroup_api *api); +int in_docker_set_cgroup_api_v2(struct cgroup_api *api); /* Docker Input configuration & context */ struct flb_docker { @@ -86,6 +109,12 @@ struct flb_docker { struct cgroup_api cgroup_api; struct flb_input_instance *ins; struct flb_log_event_encoder log_encoder; + + /* cgroup version used by host */ + int cgroup_version; + + /* proc and sys paths, overwriting mostly for testing */ + flb_sds_t sysfs_path; }; int in_docker_collect(struct flb_input_instance *i_ins,