Merge pull request #228 from FGA-GCES/fix_conflits

Fix conflits

.github/configs/tf-cuda.json

@@ -12,6 +12,10 @@
     {
       "TF": "2.6",
       "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.7",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
     }
   ]
 }

.github/scripts/mr_publish_results.py

@@ -1,4 +1,4 @@
-# Send model regression test results to Segment and Datadog
+# Send model regression test results to Datadog
 # with a summary of all test results.
 # Also write them into a report file.
 import copy
@@ -7,7 +7,6 @@
 import os
 from typing import Any, Dict, List, Text, Tuple
 
-import analytics
 from datadog_api_client.v1 import ApiClient, Configuration
 from datadog_api_client.v1.api.metrics_api import MetricsApi
 from datadog_api_client.v1.model.metrics_payload import MetricsPayload
@@ -28,14 +27,6 @@
     "story_report.json": "story_prediction",
 }
 
-TASK_MAPPING_SEGMENT = {
-    "intent_report.json": "Intent Classification",
-    "CRFEntityExtractor_report.json": "Entity Prediction",
-    "DIETClassifier_report.json": "Entity Prediction",
-    "response_selection_report.json": "Response Selection",
-    "story_report.json": "Story Prediction",
-}
-
 METRICS = {
     "test_run_time": "TEST_RUN_TIME",
     "train_run_time": "TRAIN_RUN_TIME",
@@ -44,7 +35,7 @@
 
 MAIN_TAGS = {
     "config": "CONFIG",
-    "dataset": "DATASET",
+    "dataset": "DATASET_NAME",
 }
 
 OTHER_TAGS = {
@@ -53,6 +44,7 @@
     "accelerator_type": "ACCELERATOR_TYPE",
     "type": "TYPE",
     "index_repetition": "INDEX_REPETITION",
+    "host_name": "HOST_NAME",
 }
 
 GIT_RELATED_TAGS = {
@@ -226,27 +218,6 @@ def send_to_datadog(results: List[Dict[Text, Any]]) -> None:
             print(response)
 
 
-def _send_to_segment(context: Dict[Text, Any]) -> None:
-    jobID = os.environ["GITHUB_RUN_ID"]
-    analytics.identify(
-        jobID, {"name": "model-regression-tests", "created_at": datetime.datetime.now()}
-    )
-
-    analytics.track(
-        jobID,
-        "results",
-        {
-            "config_repository": CONFIG_REPOSITORY,
-            **prepare_datasetrepo_and_external_tags(),
-            **create_dict_of_env(METRICS),
-            **create_dict_of_env(MAIN_TAGS),
-            **create_dict_of_env(OTHER_TAGS),
-            **create_dict_of_env(GIT_RELATED_TAGS),
-            **context,
-        },
-    )
-
-
 def read_results(file: Text) -> Dict[Text, Any]:
     with open(file) as json_file:
         data = json.load(json_file)
@@ -270,12 +241,6 @@ def get_result(file_name: Text, file: Text) -> Dict[Text, Any]:
     return result
 
 
-def _push_results_to_segment(file_name: Text, file: Text) -> None:
-    result = get_result(file_name, file)
-    result["task"] = TASK_MAPPING_SEGMENT[file_name]
-    _send_to_segment(result)
-
-
 def send_all_to_datadog() -> None:
     results = []
     for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
@@ -286,20 +251,9 @@ def send_all_to_datadog() -> None:
     send_to_datadog(results)
 
 
-def send_all_results_to_segment() -> None:
-    analytics.write_key = os.environ["SEGMENT_TOKEN"]
-    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
-        for f in files:
-            if any(
-                f.endswith(valid_name) for valid_name in TASK_MAPPING_SEGMENT.keys()
-            ):
-                _push_results_to_segment(f, os.path.join(dirpath, f))
-    analytics.flush()
-
-
 def generate_json(file: Text, task: Text, data: dict) -> dict:
     config = os.environ["CONFIG"]
-    dataset = os.environ["DATASET"]
+    dataset = os.environ["DATASET_NAME"]
 
     if dataset not in data:
         data = {dataset: {config: []}, **data}
@@ -336,5 +290,4 @@ def create_report_file() -> None:
 
 if __name__ == "__main__":
     send_all_to_datadog()
-    send_all_results_to_segment()
     create_report_file()

.github/scripts/start_dd_agent.sh

@@ -2,6 +2,7 @@
 
 DD_API_KEY=$1
 ACCELERATOR_TYPE=$2
+NVML_INTERVAL_IN_SEC=${3:-15}  # 15 seconds are the default interval
 
 # Install Datadog system agent
 DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$DD_API_KEY DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
@@ -14,7 +15,7 @@ sudo chmod 666 $DATADOG_YAML_PATH
     echo "tags:"
     echo "- service:rasa"
     echo "- accelerator_type:${ACCELERATOR_TYPE}"
-    echo "- dataset:${DATASET}"
+    echo "- dataset:${DATASET_NAME}"
     echo "- config:${CONFIG}"
     echo "- dataset_commit:${DATASET_COMMIT}"
     echo "- branch:${BRANCH}"
@@ -29,6 +30,8 @@ sudo chmod 666 $DATADOG_YAML_PATH
     echo "- workflow:${GITHUB_WORKFLOW:-none}"
     echo "- github_run_id:${GITHUB_RUN_ID:-none}"
     echo "- github_event:${GITHUB_EVENT_NAME:-none}"
+    echo "- index_repetition:${INDEX_REPETITION}"
+    echo "- host_name:${HOST_NAME}"
     echo ""
     echo "apm_config:"
     echo "    enabled: true"
@@ -41,10 +44,15 @@ sudo chmod 666 $DATADOG_YAML_PATH
 sudo mv /etc/datadog-agent/conf.d/system_core.d/conf.yaml.example /etc/datadog-agent/conf.d/system_core.d/conf.yaml
 
 if [[ "${ACCELERATOR_TYPE}" == "GPU" ]]; then
-# Install and enable NVML integration
-sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
-sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
-sudo mv /etc/datadog-agent/conf.d/nvml.d/conf.yaml.example /etc/datadog-agent/conf.d/nvml.d/conf.yaml
+    # Install and enable NVML integration
+    sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
+    sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
+    NVML_CONF_FPATH="/etc/datadog-agent/conf.d/nvml.d/conf.yaml"
+    sudo mv "${NVML_CONF_FPATH}.example" ${NVML_CONF_FPATH}
+    if [[ "${NVML_INTERVAL_IN_SEC}" != 15 ]]; then
+        # Append a line to the NVML config file
+        sudo echo "    min_collection_interval: ${NVML_INTERVAL_IN_SEC}" | sudo tee -a ${NVML_CONF_FPATH} > /dev/null
+    fi
 fi
 
 # Apply changes

.github/tests/test_mr_publish_results.py

@@ -9,16 +9,24 @@
     prepare_ml_metrics,
     transform_to_seconds,
     generate_json,
+    prepare_datadog_tags,
 )
 
 EXAMPLE_CONFIG = "Sparse + BERT + DIET(seq) + ResponseSelector(t2t)"
-EXAMPLE_DATASET = "financial-demo"
+EXAMPLE_DATASET_NAME = "financial-demo"
 
 ENV_VARS = {
+    "BRANCH": "my-branch",
+    "PR_ID": "10927",
+    "PR_URL": "https://github.com/RasaHQ/rasa/pull/10856/",
+    "GITHUB_EVENT_NAME": "pull_request",
+    "GITHUB_RUN_ID": "1882718340",
+    "GITHUB_SHA": "abc",
+    "GITHUB_WORKFLOW": "CI - Model Regression",
     "IS_EXTERNAL": "false",
     "DATASET_REPOSITORY_BRANCH": "main",
     "CONFIG": EXAMPLE_CONFIG,
-    "DATASET": EXAMPLE_DATASET,
+    "DATASET_NAME": EXAMPLE_DATASET_NAME,
     "CONFIG_REPOSITORY_BRANCH": "main",
     "DATASET_COMMIT": "52a3ad3eb5292d56542687e23b06703431f15ead",
     "ACCELERATOR_TYPE": "CPU",
@@ -27,16 +35,17 @@
     "TOTAL_RUN_TIME": "5m58s",
     "TYPE": "nlu",
     "INDEX_REPETITION": "0",
+    "HOST_NAME": "github-runner-2223039222-22df222fcd-2cn7d",
 }
 
 
 @mock.patch.dict(os.environ, ENV_VARS, clear=True)
 def test_generate_json():
     f = Path(__file__).parent / "test_data" / "intent_report.json"
     result = generate_json(f, task="intent_classification", data={})
-    assert isinstance(result[EXAMPLE_DATASET][EXAMPLE_CONFIG], list)
+    assert isinstance(result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG], list)
 
-    actual = result[EXAMPLE_DATASET][EXAMPLE_CONFIG][0]["intent_classification"]
+    actual = result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG][0]["intent_classification"]
     expected = {
         "accuracy": 1.0,
         "weighted avg": {
@@ -115,3 +124,9 @@ def test_prepare_ml_model_perf_metrics_simple():
 
     key, value = "Intent Classification.weighted avg.f1-score", 1.0
     assert key in metrics_ml and value == metrics_ml[key]
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_prepare_datadog_tags():
+    tags_list = prepare_datadog_tags()
+    assert "dataset:financial-demo" in tags_list