Add Neuron Scraper for scraping neuron monitor metrics

[sam6134]

Description: Adding Neuron Scraper Configs an decorator

CR for initial Reviews on the NeuronScraper

• Rebased the GPU code, and made the decorator generic
• Added a new decorator to add Pod-Attributes to the metric
• Make both the Dcgm scraper and NeuronScraper implement the SimpleScraper

Testing: Deployed on test cluster consumed logs and printed final -

            "Metric_14": {
                "name": "neuroncore_memory_usage_tensors",
                "datapoints": [
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:0 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:0 neuroncore:0 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:1 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:0 neuroncore:1 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:10 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:5 neuroncore:10 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:11 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:5 neuroncore:11 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:12 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:6 neuroncore:12 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:13 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:6 neuroncore:13 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:14 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:7 neuroncore:14 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:15 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:7 neuroncore:15 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:16 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:8 neuroncore:16 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:17 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:8 neuroncore:17 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:18 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:9 neuroncore:18 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:19 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:9 neuroncore:19 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
                        "attributes": "map[ClusterName:my-trn1-cluster ContainerName:trn1-mlp DeviceId:2 FullPodName:trn1-mlp InstanceId:i-09679ee85eb4ec8ee K8sPodName:trn1-mlp Namespace:default PodName:trn1-mlp availability_zone:us-east-1c instance_id:i-09679ee85eb4ec8ee instance_type:trn1n.32xlarge kubernetes:{"container_name":"trn1-mlp","containerd":{"container_id":"7405c4b10700c84fafb45f7f8cc394f0af8f2cf7e0f6b0515b5d180bd4ce1297"},"labels":{"my-label1":"label1-value","my-label2":"label2-value"},"namespace_name":"default","pod_name":"trn1-mlp"} memory_location:None neuron_device_index:1 neuroncore:2 region:us-east-1 runtime_tag:367 subnet_id:subnet-06a7754948e8a000f]",
                        "value": 6.315872e+06,
                    },
                    {
...
....

[straussb]

For any future readers, note that this PR was continued from aditya-purang#1