Merge pull request #5442 from snwoods/private/stevenwo/CP-46151

CP-46151: Productise the observer.py

.codecov.yml

@@ -158,6 +158,31 @@ coverage:
         #
         threshold: 20%
 
+      python3:
+
+        #
+        # The python3 limit applies to:
+        # -----------------------------
+        #
+        # - python3/**
+        # - excluding: **/test_*.py
+        #
+        paths: ["python3/**", "!**/test_*.py"]
+
+        #
+        # For python3/** (excluding tests):
+        #
+        # For python3, coverage should not be reduced compared to its base:
+        #
+        target: auto
+
+        #
+        # Exception: the threshold value given is allowed
+        #
+        # Allows for not covering 20% if the changed lines of the PR:
+        #
+        threshold: 20%
+
       # Checks each Python version separately:
       python-3.11:
         flags: ["python3.11"]
@@ -175,18 +200,26 @@ coverage:
       # Python modules and scripts below scripts/ (excluding tests)
       #
       scripts:
+        paths: ["scripts/**", "!**/test_*.py"]
         target: 48%
         threshold: 2%
-        paths: ["scripts/**", "!**/test_*.py"]
 
       #
-      # Python modules and scripts below ocaml/
+      # Python modules and scripts below ocaml/ (excluding tests)
       #
       ocaml:
         paths: ["ocaml/**", "!**/test_*.py"]
         target: 51%
         threshold: 3%
 
+      #
+      # Python modules and scripts below python3/ (excluding tests)
+      #
+      python3:
+        paths: ["python3/**", "!**/test_*.py"]
+        target: 48%
+        threshold: 2%
+
       #
       # Test files
       #
@@ -239,6 +272,10 @@ component_management:
         - "ocaml/xapi-storage-script/**"
         - "!**/test_*.py"
 
+    - component_id: python3
+      name: python3
+      paths: ["python3/**", "!**/test_*.py"]
+
     - component_id: test_cases
       name: test_cases
       paths: ["**/test_*.py"]

.github/workflows/main.yml

@@ -51,12 +51,13 @@ jobs:
 
       - name: Install dependencies only needed for python 3
         if: ${{ matrix.python-version != '2.7' }}
-        run: pip install pandas pytype toml
+        run: pip install pandas pytype toml wrapt
 
       - name: Install common dependencies for Python ${{matrix.python-version}}
         run: pip install future mock pytest-coverage pytest-mock
 
-      - name: Run Pytest and get code coverage for Codecov
+      - name: Run Pytest for python 2 and get code coverage for Codecov
+        if: ${{ matrix.python-version == '2.7' }}
         run: >
           pytest
           --cov=scripts --cov=ocaml/xcp-rrdd
@@ -67,6 +68,18 @@ jobs:
         env:
           PYTHONDEVMODE: yes
 
+      - name: Run Pytest for python 3 and get code coverage for Codecov
+        if: ${{ matrix.python-version != '2.7' }}
+        run: >
+          pytest
+          --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
+          scripts/ ocaml/xcp-rrdd python3/ -vv -rA
+          --junitxml=.git/pytest${{matrix.python-version}}.xml
+          --cov-report term-missing
+          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
+        env:
+          PYTHONDEVMODE: yes
+
       - name: Upload Python ${{matrix.python-version}} coverage report to Codecov
         uses: codecov/codecov-action@v3
         with:

Makefile

@@ -147,6 +147,7 @@ install: build doc sdk doc-json
 	mkdir -p $(DESTDIR)/etc/bash_completion.d
 # ocaml/xapi
 	make -C scripts install
+	make -C python3 install
 	cp -f _build/install/default/bin/xapi $(DESTDIR)$(OPTDIR)/bin/xapi
 	scripts/install.sh 755 ocaml/quicktest/quicktest $(DESTDIR)$(OPTDIR)/debug
 	cp -f _build/install/default/bin/quicktestbin $(DESTDIR)$(OPTDIR)/debug/quicktestbin

ocaml/libs/tracing/tracing.ml

@@ -143,7 +143,7 @@ end
 module SpanContext = struct
   type t = {trace_id: string; span_id: string} [@@deriving rpcty]
 
-  let to_traceparent t = Printf.sprintf "00-%s-%s-00" t.trace_id t.span_id
+  let to_traceparent t = Printf.sprintf "00-%s-%s-01" t.trace_id t.span_id
 
   let of_traceparent traceparent =
     let elements = String.split_on_char '-' traceparent in
@@ -722,6 +722,8 @@ module Export = struct
 
       let set_trace_log_dir dir = trace_log_dir := dir
 
+      let get_trace_log_dir () = !trace_log_dir
+
       let set_max_file_size size = max_file_size := size
 
       let set_compress_tracing_files enabled = compress_tracing_files := enabled

ocaml/libs/tracing/tracing.mli

@@ -153,6 +153,8 @@ module Export : sig
 
       val set_trace_log_dir : string -> unit
 
+      val get_trace_log_dir : unit -> string
+
       val set_compress_tracing_files : bool -> unit
     end
 

ocaml/tests/test_observer.ml

@@ -82,7 +82,13 @@ module TracerProvider = struct
           ()
         with _ -> Alcotest.failf "Missing mandatory attribute: %s" x
       )
-      ["xs.pool.uuid"; "xs.host.name"; "xs.host.uuid"; "xs.observer.name"]
+      [
+        "xs.pool.uuid"
+      ; "xs.host.name"
+      ; "xs.host.uuid"
+      ; "xs.observer.name"
+      ; "service.name"
+      ]
 
   let check_endpoints ~name ~endpoints =
     let provider = find_provider_exn ~name in
@@ -287,6 +293,7 @@ let verify_json_fields_and_values ~json =
             ; ("xs.observer.name", `String "test-observer")
             ; ("xs.host.uuid", `String _)
             ; ("xs.host.name", `String _)
+            ; ("service.name", `String _)
             ]
         )
       ; ("annotations", `List _)

ocaml/xapi/xapi_observer.ml

@@ -220,6 +220,19 @@ module Xapi_cluster = struct
   end
 end
 
+let default_attributes ~__context ~host ~name_label ~component =
+  let pool = Helpers.get_pool ~__context in
+  let host_label = Db.Host.get_name_label ~__context ~self:host in
+  let host_uuid = Db.Host.get_uuid ~__context ~self:host in
+  let pool_uuid = Db.Pool.get_uuid ~__context ~self:pool in
+  [
+    ("xs.pool.uuid", pool_uuid)
+  ; ("xs.host.name", host_label)
+  ; ("xs.host.uuid", host_uuid)
+  ; ("xs.observer.name", name_label)
+  ; ("service.name", to_string component)
+  ]
+
 module ObserverConfig = struct
   type t = {
       otel_service_name: string
@@ -235,7 +248,7 @@ module ObserverConfig = struct
   let rec bugtool_endpoint endpoints =
     match endpoints with
     | x :: _ when x = Tracing.bugtool_name ->
-        Some x
+        Some (Tracing.Export.Destination.File.get_trace_log_dir ())
     | _ :: t ->
         bugtool_endpoint t
     | [] ->
@@ -247,12 +260,18 @@ module ObserverConfig = struct
       attrs
 
   let config_of_observer ~__context ~component ~observer =
+    (* In the future this should be updated so that the config is read
+       from and updated instead of being regenerated. *)
     let endpoints = Db.Observer.get_endpoints ~__context ~self:observer in
+    let host = Helpers.get_localhost ~__context in
+    let name_label = Db.Observer.get_name_label ~__context ~self:observer in
     {
-      otel_service_name= component
+      otel_service_name= to_string component
     ; otel_resource_attributes=
         attributes_to_W3CBaggage
-          (Db.Observer.get_attributes ~__context ~self:observer)
+          (Db.Observer.get_attributes ~__context ~self:observer
+          @ default_attributes ~__context ~host ~name_label ~component
+          )
     ; xs_exporter_zipkin_endpoints= zipkin_endpoints endpoints
     ; xs_exporter_bugtool_endpoint= bugtool_endpoint endpoints
     }
@@ -292,8 +311,7 @@ module Dom0ObserverConfig (ObserverComponent : OBSERVER_COMPONENT) :
     if Db.Observer.get_enabled ~__context ~self:observer then (
       let observer_config =
         ObserverConfig.config_of_observer ~__context
-          ~component:(to_string ObserverComponent.component)
-          ~observer
+          ~component:ObserverComponent.component ~observer
       in
       Xapi_stdext_unix.Unixext.mkdir_rec dir_name 0o755 ;
       let file_name = observer_conf_path_of ~uuid in
@@ -429,22 +447,10 @@ let assert_valid_attributes attributes =
     )
     attributes
 
-let default_attributes ~__context ~host ~name_label =
-  let pool = Helpers.get_pool ~__context in
-  let host_label = Db.Host.get_name_label ~__context ~self:host in
-  let host_uuid = Db.Host.get_uuid ~__context ~self:host in
-  let pool_uuid = Db.Pool.get_uuid ~__context ~self:pool in
-  [
-    ("xs.pool.uuid", pool_uuid)
-  ; ("xs.host.name", host_label)
-  ; ("xs.host.uuid", host_uuid)
-  ; ("xs.observer.name", name_label)
-  ]
-
 let register_component ~__context ~self ~host ~component =
   let name_label = Db.Observer.get_name_label ~__context ~self in
   let attributes =
-    default_attributes ~__context ~host ~name_label
+    default_attributes ~__context ~host ~name_label ~component
     @ Db.Observer.get_attributes ~__context ~self
   in
   let uuid = Db.Observer.get_uuid ~__context ~self in
@@ -603,13 +609,15 @@ let set_attributes ~__context ~self ~value =
   let uuid = Db.Observer.get_uuid ~__context ~self in
   let host = Helpers.get_localhost ~__context in
   let name_label = Db.Observer.get_name_label ~__context ~self in
-  let default_attributes = default_attributes ~__context ~host ~name_label in
   let observation_fn () =
     List.iter
       (fun c ->
         let module Forwarder = (val get_forwarder c : ObserverInterface) in
         Forwarder.set_attributes ~__context ~uuid
-          ~attributes:(default_attributes @ value)
+          ~attributes:
+            (default_attributes ~__context ~host ~name_label ~component:c
+            @ value
+            )
       )
       (Db.Observer.get_components ~__context ~self
       |> List.map of_string

pyproject.toml

@@ -39,8 +39,8 @@ ensure_newline_before_comments = false
 # Note mypy has no config setting for PYTHONPATH, so you need to call it with:
 # PYTHONPATH="scripts/examples/python:.:scripts:scripts/plugins:scripts/examples"
 files = [
+    "python3",
     "scripts/usb_reset.py",
-    "scripts/unit_tests",
 ]
 pretty = true
 error_summary = true
@@ -113,6 +113,7 @@ inputs = [
     "scripts/examples/python",
     "scripts/yum-plugins",
     "scripts/*.py",
+    "python3/packages/*.py",
 
     # To be added later,
     # when converted to Python3-compatible syntax:

python3/Makefile

@@ -0,0 +1,9 @@
+include ../config.mk
+
+SITE3_DIR=$(shell python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
+
+IDATA=install -m 644
+
+install:
+	mkdir -p $(DESTDIR)$(SITE3_DIR)
+	$(IDATA) packages/observer.py $(DESTDIR)$(SITE3_DIR)/

python3/README.md

@@ -0,0 +1,12 @@
+# Python3 Scripts
+
+This directory is intended for scripts that only run on python3. As more scripts are
+ported to python3 this directory should start to fill up. The intended structure of
+the directory is as follows:
+
+- bin: This contains files to be installed in bin and are meant to be run by users
+- libexec: This contains files to be installed in libexec and are meant to only be
+run by xapi and other daemons.
+- packages: This contains files to be installed in python's site-packages and are meant
+to be modules and packages to be imported by other scripts or executed via python3 -m
+- plugins: This contains files that are meant to be xapi plugins