Add support for Elastic 8 (#137)

* Add support for Elastic 8

* fix dockerfile

* increasing test timeout

* remove deprecated metrics

* remove deprecated metrics

* remove old build tag

* address comments

* fix test data

* update changelog

* update changelog

* small change in changelog deprecation notice

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 5.0.0 (2023-01-19)
+### Removed
+- Dropped support for Elastic v5 and v6 (both EOL)
+- Removed metrics since these were deprecated from v7:
+  - "threadpool.indexActive"
+  - "threadpool.indexQueue"
+  - "threadpool.indexRejected"
+  - "threadpool.indexThreads" 
+### Added
+Added support for:
+- Elasticsearch v7
+- Elasticsearch v8
+
 ## 4.5.3 (2022-06-21)
 ### Changed
 - Bump dependencies

README.md

@@ -12,8 +12,7 @@ For installation and usage instructions, see our [documentation web site](https:
 
 ## Compatibility
 
-* Supported OS: No limitations
-* elasticsearch versions: 5.0+
+For compatibility and requirements, see our [documentation web site](https://docs.newrelic.com/docs/infrastructure/host-integrations/host-integrations-list/elasticsearch/elasticsearch-integration/#req).
 
 ## Building
 

build/Dockerfile

@@ -11,5 +11,8 @@ RUN apt-get update \
         unzip \
         zip
 
+# Since the user does not match the owners of the repo "git rev-parse --is-inside-work-tree" fails and goreleaser does not populate projectName
+# https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
+RUN git config --global --add safe.directory '*'
 RUN curl -L https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.deb -o gh_${GH_VERSION}_linux_amd64.deb
 RUN dpkg -i gh_${GH_VERSION}_linux_amd64.deb

spec.csv

@@ -183,14 +183,6 @@ elasticsearch,threadpool.getActive,gauge,true,The number of active threads in th
 elasticsearch,threadpool.getQueue,gauge,true,The number of queued threads in the get pool
 elasticsearch,threadpool.getThreads,gauge,true,The number of threads in the get pool
 elasticsearch,threadpool.getRejected,gauge,true,The number of rejected threads in the get pool
-elasticsearch,threadpool.indexActive,gauge,true,The number of active threads in the index pool
-elasticsearch,threadpool.indexQueue,gauge,true,The number of queued threads in the index pool
-elasticsearch,threadpool.indexThreads,gauge,true,The number of threads in the index pool
-elasticsearch,threadpool.indexRejected,gauge,true,The number of rejected threads in the index pool
-elasticsearch,threadpool.listenerActive,gauge,true,The number of active threads in the listener pool
-elasticsearch,threadpool.listenerQueue,gauge,true,The number of queued threads in the listener pool
-elasticsearch,threadpool.listenerThreads,gauge,true,The number of threads in the listener pool
-elasticsearch,threadpool.listenerRejected,gauge,true,The number of rejected threads in the listener pool
 elasticsearch,threadpool.managementActive,gauge,true,The number of active threads in the management pool
 elasticsearch,threadpool.managementQueue,gauge,true,The number of queued threads in the management pool
 elasticsearch,threadpool.managementThreads,gauge,true,The number of threads in the management pool

src/metric_definition.go

@@ -466,7 +466,6 @@ type NodeThreadPool struct {
 	ForceMerge        *ThreadPoolForceMerge        `json:"force_merge"`
 	Generic           *ThreadPoolGeneric           `json:"generic"`
 	Get               *ThreadPoolGet               `json:"get"`
-	Index             *ThreadPoolIndex             `json:"index"`
 	Listener          *ThreadPoolListener          `json:"listener"`
 	Management        *ThreadPoolManagement        `json:"management"`
 	Merge             *ThreadPoolMerge             `json:"merge"`
@@ -532,14 +531,6 @@ type ThreadPoolGet struct {
 	Rejected *int `json:"rejected" metric_name:"threadpool.getRejected" source_type:"gauge"`
 }
 
-// ThreadPoolIndex struct
-type ThreadPoolIndex struct {
-	Active   *int `json:"active" metric_name:"threadpool.indexActive" source_type:"gauge"`
-	Queue    *int `json:"queue" metric_name:"threadpool.indexQueue" source_type:"gauge"`
-	Threads  *int `json:"threads" metric_name:"threadpool.indexThreads" source_type:"gauge"`
-	Rejected *int `json:"rejected" metric_name:"threadpool.indexRejected" source_type:"gauge"`
-}
-
 // ThreadPoolListener struct
 type ThreadPoolListener struct {
 	Active   *int `json:"active" metric_name:"threadpool.listenerActive" source_type:"gauge"`

src/testdata/nodeStatsMetricsResult.json.golden

@@ -1 +1 @@
-{"activeSearches":0,"activeSearchesInMilliseconds":0,"breakers.estimatedSizeFieldDataCircuitBreakerInBytes":0,"breakers.estimatedSizeParentCircuitBreakerInBytes":0,"breakers.estimatedSizeRequestCircuitBreakerInBytes":0,"breakers.fieldDataCircuitBreakerTripped":0,"breakers.parentCircuitBreakerTripped":0,"breakers.requestCircuitBreakerTripped":0,"cache.cacheSizeIDInBytes":378599,"displayName":"T6Paw3b","entityName":"es-node:T6Paw3b","event_type":"ElasticsearchNodeSample","flush.indexRefreshesTotal":0,"flush.indexRefreshesTotalInMilliseconds":0,"fs.bytesAvailableJVMInBytes":162707742720,"fs.dataWritten":912832,"fs.iOOperations":235195,"fs.readOperations":99,"fs.totalSizeInBytes":164705353728,"fs.unallocatedBytesInBYtes":162707742720,"fs.usedDiskPercent":1.2128391474747815,"fs.writeOperations":235096,"get.currentRequestsRunning":0,"get.requestsDocumentExists":0,"get.requestsDocumentExistsInMilliseconds":0,"get.requestsDocumentMissing":0,"get.requestsDocumentMissingInMilliseconds":0,"get.timeGetRequestsInMilliseconds":0,"get.totalGetRequests":0,"http.currentOpenConnections":2,"http.openedConnections":147,"indexing.docsCurrentlyDeleted":0,"indexing.documentsCurrentlyIndexing":0,"indexing.documentsIndexed":0,"indexing.timeDeletingDocumentsInMilliseconds":0,"indexing.timeIndexingDocumentsInMilliseconds":0,"indexing.totalDocumentsDeleted":0,"indices.indexingOperationsFailed":0,"indices.indexingWaitedThrottlingInMilliseconds":0,"indices.memoryQueryCacheInBytes":0,"indices.numberIndices":0,"indices.queryCacheEvictions":0,"indices.queryCacheHits":0,"indices.queryCacheMisses":0,"indices.recoveryOngoingShardSource":0,"indices.recoveryOngoingShardTarget":0,"indices.recoveryWaitedThrottlingInMilliseconds":0,"indices.requestCacheEvictions":0,"indices.requestCacheHits":0,"indices.requestCacheMemoryInBytes":0,"indices.requestCacheMisses":0,"indices.segmentsIndexShard":0,"indices.segmentsMaxMemoryIndexWriterInBytes":7423798234978,"indices.segmentsMemoryUsedDocValuesInBytes":0,"indices.segmentsMemoryUsedFixedBitSetInBytes":0,"indices.segmentsMemoryUsedIndexSegmentsInBytes":0,"indices.segmentsMemoryUsedIndexWriterInBytes":0,"indices.segmentsMemoryUsedNormsInBytes":0,"indices.segmentsMemoryUsedSegmentVersionMapInBytes":0,"indices.segmentsMemoryUsedStoredFieldsInBytes":0,"indices.segmentsMemoryUsedTermVectorsInBytes":0,"indices.segmentsMemoryUsedTermsInBytes":0,"indices.translogOperations":0,"indices.translogOperationsInBytes":0,"jvm.ThreadsActive":55,"jvm.ThreadsPeak":56,"jvm.gc.collections":100,"jvm.gc.collectionsInMilliseconds":100,"jvm.gc.concurrentMarkSweep":50,"jvm.gc.concurrentMarkSweepInMilliseconds":7238,"jvm.gc.majorCollectionsOldGenerationObjects":2,"jvm.gc.majorCollectionsOldGenerationObjectsInMilliseconds":128,"jvm.gc.minorCollectionsYoungGenerationObjects":108,"jvm.gc.minorCollectionsYoungGenerationObjectsInMilliseconds":1752,"jvm.gc.parallelNewCollections":2189,"jvm.gc.parallelNewCollectionsInMilliseconds":329,"jvm.mem.heapCommittedInBytes":1056309248,"jvm.mem.heapMaxInBytes":1056309248,"jvm.mem.heapUsed":21,"jvm.mem.heapUsedInBytes":225181888,"jvm.mem.maxOldGenerationHeapInBytes":899284992,"jvm.mem.maxSurvivorSpaceInBYtes":17432576,"jvm.mem.maxYoungGenerationHeapInBytes":139591680,"jvm.mem.nonHeapCommittedInBytes":108519424,"jvm.mem.nonHeapUsedInBytes":100466784,"jvm.mem.usedOldGenerationHeapInBytes":127243224,"jvm.mem.usedSurvivorSpaceInBytes":94752,"jvm.mem.usedYoungGenerationHeapInBytes":97843912,"merges.currentActive":0,"merges.docsSegmentMerges":0,"merges.docsSegmentsMerging":0,"merges.mergedSegmentsInBytes":0,"merges.segmentMerges":0,"merges.sizeSegmentsMergingInBytes":0,"merges.totalSegmentMergingInMilliseconds":0,"node.hostname":"10.33.104.186","node.ipAddress":"10.33.104.186:9300,NONE","openFD":213,"queriesTotal":0,"refresh.total":0,"refresh.totalInMilliseconds":0,"reportingEndpoint":":0","searchFetchCurrentlyRunning":0,"searchFetches":0,"sizeStoreInBytes":0,"threadpool.activeFetchShardStarted":0,"threadpool.bulkActive":3,"threadpool.bulkQueue":234,"threadpool.bulkRejected":2,"threadpool.bulkThreads":23,"threadpool.fetchShardStartedQueue":0,"threadpool.fetchShardStartedRejected":0,"threadpool.fetchShardStartedThreads":0,"threadpool.fetchShardStoreActive":0,"threadpool.fetchShardStoreQueue":0,"threadpool.fetchShardStoreRejected":0,"threadpool.fetchShardStoreThreads":0,"threadpool.flushActive":0,"threadpool.flushQueue":0,"threadpool.flushRejected":0,"threadpool.flushThreads":0,"threadpool.forceMergeActive":0,"threadpool.forceMergeQueue":0,"threadpool.forceMergeRejected":0,"threadpool.forceMergeThreads":0,"threadpool.genericActive":0,"threadpool.genericQueue":0,"threadpool.genericRejected":0,"threadpool.genericThreads":5,"threadpool.getActive":0,"threadpool.getQueue":0,"threadpool.getRejected":0,"threadpool.getThreads":0,"threadpool.indexActive":0,"threadpool.indexQueue":0,"threadpool.indexRejected":0,"threadpool.indexThreads":0,"threadpool.listenerActive":0,"threadpool.listenerQueue":0,"threadpool.listenerRejected":0,"threadpool.listenerThreads":0,"threadpool.managementActive":1,"threadpool.managementQueue":0,"threadpool.managementRejected":0,"threadpool.managementThreads":3,"threadpool.mergeActive":3,"threadpool.mergeQueue":39,"threadpool.mergeRejected":837,"threadpool.mergeThreads":9,"threadpool.percolateActive":8,"threadpool.percolateQueue":78,"threadpool.percolateRejected":8,"threadpool.percolateThreads":9,"threadpool.refreshActive":0,"threadpool.refreshQueue":0,"threadpool.refreshRejected":0,"threadpool.refreshThreads":0,"threadpool.searchActive":0,"threadpool.searchQueue":0,"threadpool.searchRejected":0,"threadpool.searchThreads":0,"threadpool.snapshotActive":0,"threadpool.snapshotQueue":0,"threadpool.snapshotRejected":0,"threadpool.snapshotThreads":0,"transport.connectionsOpened":0,"transport.packetsReceived":20,"transport.packetsReceivedInBytes":11691,"transport.packetsSent":20,"transport.packetsSentInBytes":11691}
+{"activeSearches":0,"activeSearchesInMilliseconds":0,"breakers.estimatedSizeFieldDataCircuitBreakerInBytes":0,"breakers.estimatedSizeParentCircuitBreakerInBytes":0,"breakers.estimatedSizeRequestCircuitBreakerInBytes":0,"breakers.fieldDataCircuitBreakerTripped":0,"breakers.parentCircuitBreakerTripped":0,"breakers.requestCircuitBreakerTripped":0,"cache.cacheSizeIDInBytes":378599,"displayName":"T6Paw3b","entityName":"es-node:T6Paw3b","event_type":"ElasticsearchNodeSample","flush.indexRefreshesTotal":0,"flush.indexRefreshesTotalInMilliseconds":0,"fs.bytesAvailableJVMInBytes":162707742720,"fs.dataWritten":912832,"fs.iOOperations":235195,"fs.readOperations":99,"fs.totalSizeInBytes":164705353728,"fs.unallocatedBytesInBYtes":162707742720,"fs.usedDiskPercent":1.2128391474747815,"fs.writeOperations":235096,"get.currentRequestsRunning":0,"get.requestsDocumentExists":0,"get.requestsDocumentExistsInMilliseconds":0,"get.requestsDocumentMissing":0,"get.requestsDocumentMissingInMilliseconds":0,"get.timeGetRequestsInMilliseconds":0,"get.totalGetRequests":0,"http.currentOpenConnections":2,"http.openedConnections":147,"indexing.docsCurrentlyDeleted":0,"indexing.documentsCurrentlyIndexing":0,"indexing.documentsIndexed":0,"indexing.timeDeletingDocumentsInMilliseconds":0,"indexing.timeIndexingDocumentsInMilliseconds":0,"indexing.totalDocumentsDeleted":0,"indices.indexingOperationsFailed":0,"indices.indexingWaitedThrottlingInMilliseconds":0,"indices.memoryQueryCacheInBytes":0,"indices.numberIndices":0,"indices.queryCacheEvictions":0,"indices.queryCacheHits":0,"indices.queryCacheMisses":0,"indices.recoveryOngoingShardSource":0,"indices.recoveryOngoingShardTarget":0,"indices.recoveryWaitedThrottlingInMilliseconds":0,"indices.requestCacheEvictions":0,"indices.requestCacheHits":0,"indices.requestCacheMemoryInBytes":0,"indices.requestCacheMisses":0,"indices.segmentsIndexShard":0,"indices.segmentsMaxMemoryIndexWriterInBytes":7423798234978,"indices.segmentsMemoryUsedDocValuesInBytes":0,"indices.segmentsMemoryUsedFixedBitSetInBytes":0,"indices.segmentsMemoryUsedIndexSegmentsInBytes":0,"indices.segmentsMemoryUsedIndexWriterInBytes":0,"indices.segmentsMemoryUsedNormsInBytes":0,"indices.segmentsMemoryUsedSegmentVersionMapInBytes":0,"indices.segmentsMemoryUsedStoredFieldsInBytes":0,"indices.segmentsMemoryUsedTermVectorsInBytes":0,"indices.segmentsMemoryUsedTermsInBytes":0,"indices.translogOperations":0,"indices.translogOperationsInBytes":0,"jvm.ThreadsActive":55,"jvm.ThreadsPeak":56,"jvm.gc.collections":100,"jvm.gc.collectionsInMilliseconds":100,"jvm.gc.concurrentMarkSweep":50,"jvm.gc.concurrentMarkSweepInMilliseconds":7238,"jvm.gc.majorCollectionsOldGenerationObjects":2,"jvm.gc.majorCollectionsOldGenerationObjectsInMilliseconds":128,"jvm.gc.minorCollectionsYoungGenerationObjects":108,"jvm.gc.minorCollectionsYoungGenerationObjectsInMilliseconds":1752,"jvm.gc.parallelNewCollections":2189,"jvm.gc.parallelNewCollectionsInMilliseconds":329,"jvm.mem.heapCommittedInBytes":1056309248,"jvm.mem.heapMaxInBytes":1056309248,"jvm.mem.heapUsed":21,"jvm.mem.heapUsedInBytes":225181888,"jvm.mem.maxOldGenerationHeapInBytes":899284992,"jvm.mem.maxSurvivorSpaceInBYtes":17432576,"jvm.mem.maxYoungGenerationHeapInBytes":139591680,"jvm.mem.nonHeapCommittedInBytes":108519424,"jvm.mem.nonHeapUsedInBytes":100466784,"jvm.mem.usedOldGenerationHeapInBytes":127243224,"jvm.mem.usedSurvivorSpaceInBytes":94752,"jvm.mem.usedYoungGenerationHeapInBytes":97843912,"merges.currentActive":0,"merges.docsSegmentMerges":0,"merges.docsSegmentsMerging":0,"merges.mergedSegmentsInBytes":0,"merges.segmentMerges":0,"merges.sizeSegmentsMergingInBytes":0,"merges.totalSegmentMergingInMilliseconds":0,"node.hostname":"10.33.104.186","node.ipAddress":"10.33.104.186:9300,NONE","openFD":213,"queriesTotal":0,"refresh.total":0,"refresh.totalInMilliseconds":0,"reportingEndpoint":":0","searchFetchCurrentlyRunning":0,"searchFetches":0,"sizeStoreInBytes":0,"threadpool.activeFetchShardStarted":0,"threadpool.bulkActive":3,"threadpool.bulkQueue":234,"threadpool.bulkRejected":2,"threadpool.bulkThreads":23,"threadpool.fetchShardStartedQueue":0,"threadpool.fetchShardStartedRejected":0,"threadpool.fetchShardStartedThreads":0,"threadpool.fetchShardStoreActive":0,"threadpool.fetchShardStoreQueue":0,"threadpool.fetchShardStoreRejected":0,"threadpool.fetchShardStoreThreads":0,"threadpool.flushActive":0,"threadpool.flushQueue":0,"threadpool.flushRejected":0,"threadpool.flushThreads":0,"threadpool.forceMergeActive":0,"threadpool.forceMergeQueue":0,"threadpool.forceMergeRejected":0,"threadpool.forceMergeThreads":0,"threadpool.genericActive":0,"threadpool.genericQueue":0,"threadpool.genericRejected":0,"threadpool.genericThreads":5,"threadpool.getActive":0,"threadpool.getQueue":0,"threadpool.getRejected":0,"threadpool.getThreads":0,"threadpool.listenerActive":0,"threadpool.listenerQueue":0,"threadpool.listenerRejected":0,"threadpool.listenerThreads":0,"threadpool.managementActive":1,"threadpool.managementQueue":0,"threadpool.managementRejected":0,"threadpool.managementThreads":3,"threadpool.mergeActive":3,"threadpool.mergeQueue":39,"threadpool.mergeRejected":837,"threadpool.mergeThreads":9,"threadpool.percolateActive":8,"threadpool.percolateQueue":78,"threadpool.percolateRejected":8,"threadpool.percolateThreads":9,"threadpool.refreshActive":0,"threadpool.refreshQueue":0,"threadpool.refreshRejected":0,"threadpool.refreshThreads":0,"threadpool.searchActive":0,"threadpool.searchQueue":0,"threadpool.searchRejected":0,"threadpool.searchThreads":0,"threadpool.snapshotActive":0,"threadpool.snapshotQueue":0,"threadpool.snapshotRejected":0,"threadpool.snapshotThreads":0,"transport.connectionsOpened":0,"transport.packetsReceived":20,"transport.packetsReceivedInBytes":11691,"transport.packetsSent":20,"transport.packetsSentInBytes":11691}

tests/integration/docker-compose.yml

@@ -1,12 +1,15 @@
 version: '3.3'
 services:
   elasticsearch:
-    image: elasticsearch:6.4.1
+    image: elasticsearch:8.5.3
     container_name: elasticsearch
     environment:
       - cluster.name=docker-cluster
+      - node.name=elasticsearch
+      - cluster.initial_master_nodes=elasticsearch
+      - discovery.seed_hosts=elasticsearch-replica
       - bootstrap.memory_lock=true
-      - node.master=true
+      - ELASTIC_PASSWORD=elastic
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
       - xpack.security.http.ssl.enabled=true
       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
@@ -26,15 +29,17 @@ services:
     networks:
       - elastic
 
-  elasticsearch2:
-    image: elasticsearch:6.4.1
-    container_name: elasticsearch2
+  elasticsearch-replica:
+    image: elasticsearch:8.5.3
+    container_name: elasticsearch-replica
     environment:
       - cluster.name=docker-cluster
+      - node.name=elasticsearch-replica
+      - cluster.initial_master_nodes=elasticsearch
+      - discovery.seed_hosts=elasticsearch
       - bootstrap.memory_lock=true
-      - node.master=false
+      - ELASTIC_PASSWORD=elastic
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "discovery.zen.ping.unicast.hosts=elasticsearch"
       - xpack.security.http.ssl.enabled=true
       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
       - xpack.security.http.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12