diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 15e4fd614dbc..372b8e352448 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -162,6 +162,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] +- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] *Osquerybeat* diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json index f093d13a3e45..216971d16b51 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json @@ -46,7 +46,7 @@ }, "panelIndex": "2", "panelRefName": "panel_2", - "title": "Tablespace Total Size", + "title": "Tablespace Max Total Size", "type": "visualization", "version": "8.0.0-SNAPSHOT" }, diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 603fcc02ec2f..27de5af579a2 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -4,7 +4,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": {} }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "uiStateJSON": {}, "version": 1, "visState": { @@ -24,13 +24,40 @@ "fill": 0.5, "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Tablespace total size", + "label": "Tablespace max total size", "line_width": 1, "metrics": [ { - "field": "oracle.tablespace.space.total.bytes", + "agg_with": "avg", + "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" + }, + { + "agg_with": "avg", + "colors": [ + "#68BC00" + ], + "field": "oracle.tablespace.space.free.bytes", + "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "type": "max" + }, + { + "id": "2cf57800-8b54-41fa-a877-159b49699a50", + "script": "params.used_bytes + params.free_bytes", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "631a44d5-d18a-4743-bea0-6f61930fd65f", + "name": "used_bytes" + }, + { + "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "id": "c255d24c-3a29-4879-b999-77af43d97c6b", + "name": "free_bytes" + } + ] } ], "point_size": 1, @@ -48,7 +75,7 @@ "type": "timeseries", "use_kibana_indexes": false }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "type": "metrics" } }, diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 30c86d766dad..82f93a060b1e 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -16,8 +16,9 @@ import ( // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch // by doing queries to Oracle -func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) { - out = &extractedData{} +func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) { + out := &extractedData{} + var err error if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil { return nil, fmt.Errorf("error getting data_files: %w", err) @@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth return nil, fmt.Errorf("error getting free space data: %w", err) } - return + return out, nil } // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files) -func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { - out = make(map[string]mapstr.M, 0) +func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M { + out := make(map[string]mapstr.M, 0) - for _, dataFile := range in.dataFiles { - m.addDataFileData(&dataFile, out) + for i := range in.dataFiles { + m.addDataFileData(&in.dataFiles[i], out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) m.addTempFreeSpaceData(in.tempFreeSpace, out) - return + return out } func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) { @@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map name := val.(string) if name == "TEMP" { for _, tempFreeSpaceTable := range tempFreeSpaces { - oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace}) } @@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m if name == freeSpaceTable.TablespaceName { oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes}) } } } diff --git a/x-pack/metricbeat/module/oracle/tablespace/data_test.go b/x-pack/metricbeat/module/oracle/tablespace/data_test.go index 02e4bdd4528c..9c6990a25984 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/assert" ) -var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`, +var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`, `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`} + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`} var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`} diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 12348236bccc..9f5800166af1 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) { type happyTempFreeSpaceData struct{} func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) { - return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil + return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil } type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, }, nil } diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index efc8401e2c0b..f4b592d2b75b 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -11,31 +11,24 @@ import ( ) type tempFreeSpace struct { - TablespaceName string - TablespaceSize sql.NullInt64 - UsedSpaceBytes sql.NullInt64 - FreeSpace sql.NullInt64 -} - -func (d *tempFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *tempFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + UsedSpaceBytes sql.NullInt64 + FreeSpace sql.NullInt64 } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE") + rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]tempFreeSpace, 0) for rows.Next() { dest := tempFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index b17b249808d2..e327badcf02d 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -11,30 +11,24 @@ import ( ) type usedAndFreeSpace struct { - TablespaceName string - TotalFreeBytes sql.NullInt64 - TotalUsedBytes sql.NullInt64 -} - -func (d *usedAndFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *usedAndFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + TotalFreeBytes sql.NullInt64 + TotalUsedBytes sql.NullInt64 } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name") + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]usedAndFreeSpace, 0) for rows.Next() { dest := usedAndFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil { return nil, err } results = append(results, dest)