Support pool.sync_updates from remote_pool repo (xapi-project#6108)

CP-50787 CP-51347: Support pool.sync_updates from remote_pool repo

When a remote_pool type repository, which points to the enabled
repository in the remote pool coordinator, is set as the enabled
repository of the pool, updates can be synced from it with API
pool.sync_updates.

The username password of the remote pool coordinator is required as
parameters for pool.sync_updates to login the remote pool.

And the remote pool coordinator's host server certificate needs to be
configured in the remote_pool repository, it will be used to verify the
remote end when sending out username passwords and syncing updates from
it.

A new yum/dnf plugin "xapitoken" is introduced to set xapi token as HTTP
cookie: "session_id" for each HTTP request which downloads files from the
remote_pool repository.


CP-52245: Temp disable repo_gpgcheck when syncing from remote_pool repo

Will re-enable repo_gpgcheck by reverting this commit after CP-51429 is done.

ocaml/idl/datamodel_errors.ml

@@ -1926,6 +1926,16 @@ let _ =
       "If the bundle repository or remote_pool repository is enabled, it \
        should be the only one enabled repository of the pool."
     () ;
+  error Api_errors.update_syncing_remote_pool_coordinator_connection_failed []
+    ~doc:
+      "There was an error connecting to the remote pool coordinator while \
+       syncing updates from it."
+    () ;
+  error Api_errors.update_syncing_remote_pool_coordinator_service_failed []
+    ~doc:
+      "There was an error connecting to the server while syncing updates from \
+       it. The service contacted didn't reply properly."
+    () ;
   error Api_errors.repository_is_in_use [] ~doc:"The repository is in use." () ;
   error Api_errors.repository_cleanup_failed []
     ~doc:"Failed to clean up local repository on coordinator." () ;

ocaml/idl/datamodel_pool.ml

@@ -1282,6 +1282,20 @@ let sync_updates =
         ; param_release= numbered_release "1.329.0"
         ; param_default= Some (VString "")
         }
+      ; {
+          param_type= String
+        ; param_name= "username"
+        ; param_doc= "The username of the remote pool"
+        ; param_release= numbered_release "24.39.0-next"
+        ; param_default= Some (VString "")
+        }
+      ; {
+          param_type= String
+        ; param_name= "password"
+        ; param_doc= "The password of the remote pool"
+        ; param_release= numbered_release "24.39.0-next"
+        ; param_default= Some (VString "")
+        }
       ]
     ~result:(String, "The SHA256 hash of updateinfo.xml.gz")
     ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)

ocaml/idl/datamodel_repository.ml

@@ -105,7 +105,9 @@ let introduce_remote_pool =
       ; ( String
         , "binary_url"
         , "Base URL of binary packages in the local repository of this remote \
-           pool in https://<coordinator-ip>/repository format"
+           pool in https://<coordinator-ip>"
+          ^ Constants.get_enabled_repository_uri
+          ^ " format"
         )
       ; ( String
         , "certificate"

ocaml/xapi-cli-server/cli_frontend.ml

@@ -511,7 +511,7 @@ let rec cmdtable_data : (string * cmd_spec) list =
   ; ( "pool-sync-updates"
     , {
         reqd= []
-      ; optn= ["force"; "token"; "token-id"]
+      ; optn= ["force"; "token"; "token-id"; "username"; "password"]
       ; help= "Sync updates from remote YUM repository, pool-wide."
       ; implementation= No_fd Cli_operations.pool_sync_updates
       ; flags= []

ocaml/xapi-cli-server/cli_operations.ml

@@ -1833,8 +1833,11 @@ let pool_sync_updates printer rpc session_id params =
   let force = get_bool_param params "force" in
   let token = get_param params "token" ~default:"" in
   let token_id = get_param params "token-id" ~default:"" in
+  let username = get_param params "username" ~default:"" in
+  let password = get_param params "password" ~default:"" in
   let hash =
     Client.Pool.sync_updates ~rpc ~session_id ~self:pool ~force ~token ~token_id
+      ~username ~password
   in
   printer (Cli_printer.PList [hash])
 

ocaml/xapi-consts/api_errors.ml

@@ -1330,6 +1330,12 @@ let can_not_periodic_sync_updates = add_error "CAN_NOT_PERIODIC_SYNC_UPDATES"
 let repo_should_be_single_one_enabled =
   add_error "REPO_SHOULD_BE_SINGLE_ONE_ENABLED"
 
+let update_syncing_remote_pool_coordinator_connection_failed =
+  add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_CONNECTION_FAILED"
+
+let update_syncing_remote_pool_coordinator_service_failed =
+  add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_SERVICE_FAILED"
+
 let repository_is_in_use = add_error "REPOSITORY_IS_IN_USE"
 
 let repository_cleanup_failed = add_error "REPOSITORY_CLEANUP_FAILED"

ocaml/xapi/helpers.ml

@@ -2030,19 +2030,24 @@ let with_temp_file ?mode prefix suffix f =
   let path, channel = Filename.open_temp_file ?mode prefix suffix in
   finally (fun () -> f (path, channel)) (fun () -> Unix.unlink path)
 
+let with_temp_file_of_content ?mode prefix suffix content f =
+  let@ temp_file, temp_out_ch = with_temp_file ?mode prefix suffix in
+  Xapi_stdext_pervasives.Pervasiveext.finally
+    (fun () -> output_string temp_out_ch content)
+    (fun () -> close_out temp_out_ch) ;
+  f temp_file
+
 let with_temp_out_ch_of_temp_file ?mode prefix suffix f =
   let@ path, channel = with_temp_file ?mode prefix suffix in
   f (path, channel |> with_temp_out_ch)
 
-let make_external_host_verified_rpc ~__context ext_host_address ext_host_cert
-    xml =
-  let@ temp_file, temp_out_ch = with_temp_file "external-host-cert" ".pem" in
-  Xapi_stdext_pervasives.Pervasiveext.finally
-    (fun () -> output_string temp_out_ch ext_host_cert)
-    (fun () -> close_out temp_out_ch) ;
+let make_external_host_verified_rpc ~__context host_address host_cert xml =
+  let@ cert_file =
+    with_temp_file_of_content "external-host-cert-" ".pem" host_cert
+  in
   make_remote_rpc ~__context
-    ~verify_cert:(Stunnel_client.external_host temp_file)
-    ext_host_address xml
+    ~verify_cert:(Stunnel_client.external_host cert_file)
+    host_address xml
 
 module FileSys : sig
   (* bash-like interface for manipulating files *)

ocaml/xapi/pool_periodic_update_sync.ml

@@ -140,7 +140,8 @@ let rec update_sync () =
                 ignore
                   (Client.Pool.sync_updates ~rpc ~session_id
                      ~self:(Helpers.get_pool ~__context)
-                     ~force:false ~token:"" ~token_id:""
+                     ~force:false ~token:"" ~token_id:"" ~username:""
+                     ~password:""
                   )
               with e ->
                 let exc = Printexc.to_string e in

ocaml/xapi/repository.ml

@@ -149,26 +149,78 @@ let get_proxy_params ~__context repo_name =
   | _ ->
       ("", "", "")
 
-let sync ~__context ~self ~token ~token_id =
+let sync ~__context ~self ~token ~token_id ~username ~password =
   try
     let repo_name = get_remote_repository_name ~__context ~self in
     remove_repo_conf_file repo_name ;
-    let binary_url, source_url =
-      match Db.Repository.get_origin ~__context ~self with
+    let origin = Db.Repository.get_origin ~__context ~self in
+
+    let ( binary_url
+        , source_url
+        , repo_gpgcheck
+        , use_proxy
+        , client_auth
+        , server_auth ) =
+      match origin with
       | `remote ->
+          let plugin = "accesstoken" in
           ( Db.Repository.get_binary_url ~__context ~self
           , Some (Db.Repository.get_source_url ~__context ~self)
+          , true
+          , true
+          , CdnTokenAuth {token_id; token; plugin}
+          , DefaultAuth
           )
       | `bundle ->
           let uri =
             Uri.make ~scheme:"file" ~path:!Xapi_globs.bundle_repository_dir ()
           in
-          (Uri.to_string uri, None)
+          (Uri.to_string uri, None, true, false, NoAuth, NoAuth)
       | `remote_pool ->
-          (* TODO: sync with Stunnel.with_client_proxy as otherwise yum
-             reposync will fail when checking the self signed certificate on
-             the remote pool. *)
-          ("", None)
+          let cert = Db.Repository.get_certificate ~__context ~self in
+          let repo_binary_url = Db.Repository.get_binary_url ~__context ~self in
+          let remote_addr =
+            repo_binary_url |> Repository_helpers.get_remote_pool_coordinator_ip
+          in
+          let verified_rpc =
+            try
+              Helpers.make_external_host_verified_rpc ~__context remote_addr
+                cert
+            with Xmlrpc_client.Connection_reset ->
+              raise
+                (Api_errors.Server_error
+                   ( Api_errors
+                     .update_syncing_remote_pool_coordinator_connection_failed
+                   , []
+                   )
+                )
+          in
+          let session_id =
+            try
+              Client.Client.Session.login_with_password ~rpc:verified_rpc
+                ~uname:username ~pwd:password
+                ~version:Datamodel_common.api_version_string
+                ~originator:Xapi_version.xapi_user_agent
+            with
+            | Http_client.Http_request_rejected _ | Http_client.Http_error _ ->
+              raise
+                (Api_errors.Server_error
+                   ( Api_errors
+                     .update_syncing_remote_pool_coordinator_service_failed
+                   , []
+                   )
+                )
+          in
+          let xapi_token = session_id |> Ref.string_of in
+          let plugin = "xapitoken" in
+          ( repo_binary_url
+          , None
+          , false
+          , true
+          , PoolExtHostAuth {xapi_token; plugin}
+          , StunnelClientProxyAuth
+              {cert; remote_addr; remote_port= Constants.default_ssl_port}
+          )
     in
     let gpgkey_path =
       match Db.Repository.get_gpgkey_path ~__context ~self with
@@ -177,69 +229,96 @@ let sync ~__context ~self ~token ~token_id =
       | s ->
           s
     in
-    let write_initial_yum_config () =
-      write_yum_config ~source_url ~binary_url ~repo_gpgcheck:true ~gpgkey_path
+    let write_initial_yum_config ~binary_url =
+      write_yum_config ~source_url ~binary_url ~repo_gpgcheck ~gpgkey_path
         ~repo_name
     in
-    write_initial_yum_config () ;
-    clean_yum_cache repo_name ;
-    (* Remove imported YUM repository GPG key *)
-    if Pkgs.manager = Yum then
-      Xapi_stdext_unix.Unixext.rm_rec (get_repo_config repo_name "gpgdir") ;
     Xapi_stdext_pervasives.Pervasiveext.finally
       (fun () ->
-        with_access_token ~token ~token_id @@ fun token_path ->
-        (* Configure proxy and token *)
-        let token_param =
-          match token_path with
-          | Some p ->
-              Printf.sprintf "--setopt=%s.accesstoken=file://%s" repo_name p
-          | None ->
-              ""
-        in
-        let proxy_url_param, proxy_username_param, proxy_password_param =
-          get_proxy_params ~__context repo_name
-        in
-        let Pkg_mgr.{cmd; params} =
-          [
-            "--save"
-          ; proxy_url_param
-          ; proxy_username_param
-          ; proxy_password_param
-          ; token_param
-          ]
-          |> fun config -> Pkgs.config_repo ~repo_name ~config
+        let config_repo params =
+          let Pkg_mgr.{cmd; params} =
+            "--save" :: params |> fun config ->
+            Pkgs.config_repo ~repo_name ~config
+          in
+          ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params)
         in
-        ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params) ;
 
-        (* Import YUM repository GPG key to check metadata in reposync *)
-        let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in
-        ignore (Helpers.call_script cmd params) ;
+        let make_cache () =
+          (* Import YUM repository GPG key to check metadata in reposync *)
+          let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in
+          ignore (Helpers.call_script cmd params)
+        in
 
         (* Sync with remote repository *)
-        let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in
-        Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ;
+        let sync_repo () =
+          let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in
+          Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ;
+          clean_yum_cache repo_name ;
+          ignore (Helpers.call_script cmd params)
+        in
+
+        with_sync_client_auth client_auth @@ fun client_auth ->
+        with_sync_server_auth server_auth @@ fun binary_url' ->
+        write_initial_yum_config
+          ~binary_url:(Option.value binary_url' ~default:binary_url) ;
         clean_yum_cache repo_name ;
-        ignore (Helpers.call_script cmd params)
+        (* Remove imported YUM repository GPG key *)
+        if Pkgs.manager = Yum then
+          Xapi_stdext_unix.Unixext.rm_rec (get_repo_config repo_name "gpgdir") ;
+        let auth_params =
+          match client_auth with
+          | Some (auth_file, plugin) ->
+              let token_param =
+                Printf.sprintf "--setopt=%s.%s=%s" repo_name plugin
+                  (Uri.make ~scheme:"file" ~path:auth_file () |> Uri.to_string)
+              in
+              [token_param]
+          | None ->
+              []
+        in
+        let proxy_params =
+          match use_proxy with
+          | true ->
+              let proxy_url_param, proxy_username_param, proxy_password_param =
+                get_proxy_params ~__context repo_name
+              in
+              [proxy_url_param; proxy_username_param; proxy_password_param]
+          | false ->
+              []
+        in
+        config_repo (auth_params @ proxy_params) ;
+        make_cache () ;
+        sync_repo ()
       )
       (fun () ->
-        (* Rewrite repo conf file as initial content to remove credential related info,
-         * I.E. proxy username/password and temporary token file path.
+        (* Rewrite repo conf file as initial content to remove credential
+         * related info, I.E. proxy username/password and temporary token file
+         * path.
+         * One thing to note: for remote_repo, the binary_url used to
+         * re-initial yum repo is the url configed in the remote_pool repo,
+         * which is not the correct one for stunnel client proxy, while as we
+         * will always write_initial_yum_config every time before syncing repo,
+         * this should be ok.
          *)
-        write_initial_yum_config ()
+        write_initial_yum_config ~binary_url
       ) ;
-    (* The custom yum-utils will fully download repository metadata.*)
-    let repodata_dir =
+    (* The custom yum-utils will fully download repository metadata including
+     * the repo gpg signature.
+     *)
+    let repo_gpg_signature =
       !Xapi_globs.local_pool_repo_dir
       // repo_name
       // "repodata"
       // "repomd.xml.asc"
     in
-    Sys.file_exists repodata_dir
-  with e ->
-    error "Failed to sync with remote YUM repository: %s"
-      (ExnHelper.string_of_exn e) ;
-    raise Api_errors.(Server_error (reposync_failed, []))
+    Sys.file_exists repo_gpg_signature
+  with
+  | Api_errors.Server_error (_, _) as e ->
+      raise e
+  | e ->
+      error "Failed to sync with remote YUM repository: %s"
+        (ExnHelper.string_of_exn e) ;
+      raise Api_errors.(Server_error (reposync_failed, []))
 
 let http_get_host_updates_in_json ~__context ~host ~installed =
   let host_session_id =

ocaml/xapi/repository.mli

@@ -48,6 +48,8 @@ val sync :
   -> self:[`Repository] API.Ref.t
   -> token:string
   -> token_id:string
+  -> username:string
+  -> password:string
   -> bool
 
 val create_pool_repository :