diff --git a/ocaml/idl/datamodel_errors.ml b/ocaml/idl/datamodel_errors.ml index d1c3bf0ac0c..1c4f19dacff 100644 --- a/ocaml/idl/datamodel_errors.ml +++ b/ocaml/idl/datamodel_errors.ml @@ -1926,6 +1926,16 @@ let _ = "If the bundle repository or remote_pool repository is enabled, it \ should be the only one enabled repository of the pool." () ; + error Api_errors.update_syncing_remote_pool_coordinator_connection_failed [] + ~doc: + "There was an error connecting to the remote pool coordinator while \ + syncing updates from it." + () ; + error Api_errors.update_syncing_remote_pool_coordinator_service_failed [] + ~doc: + "There was an error connecting to the server while syncing updates from \ + it. The service contacted didn't reply properly." + () ; error Api_errors.repository_is_in_use [] ~doc:"The repository is in use." () ; error Api_errors.repository_cleanup_failed [] ~doc:"Failed to clean up local repository on coordinator." () ; diff --git a/ocaml/idl/datamodel_pool.ml b/ocaml/idl/datamodel_pool.ml index ab0d1669788..a86a3ea152c 100644 --- a/ocaml/idl/datamodel_pool.ml +++ b/ocaml/idl/datamodel_pool.ml @@ -1282,6 +1282,20 @@ let sync_updates = ; param_release= numbered_release "1.329.0" ; param_default= Some (VString "") } + ; { + param_type= String + ; param_name= "username" + ; param_doc= "The username of the remote pool" + ; param_release= numbered_release "24.39.0-next" + ; param_default= Some (VString "") + } + ; { + param_type= String + ; param_name= "password" + ; param_doc= "The password of the remote pool" + ; param_release= numbered_release "24.39.0-next" + ; param_default= Some (VString "") + } ] ~result:(String, "The SHA256 hash of updateinfo.xml.gz") ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT) diff --git a/ocaml/idl/datamodel_repository.ml b/ocaml/idl/datamodel_repository.ml index e2a8f03b0c5..d1daa55e85c 100644 --- a/ocaml/idl/datamodel_repository.ml +++ b/ocaml/idl/datamodel_repository.ml @@ -105,7 +105,9 @@ let introduce_remote_pool = ; ( String , "binary_url" , "Base URL of binary packages in the local repository of this remote \ - pool in https:///repository format" + pool in https://" + ^ Constants.get_enabled_repository_uri + ^ " format" ) ; ( String , "certificate" diff --git a/ocaml/xapi-cli-server/cli_frontend.ml b/ocaml/xapi-cli-server/cli_frontend.ml index c4a5a4a5dc2..4b5fa9476ae 100644 --- a/ocaml/xapi-cli-server/cli_frontend.ml +++ b/ocaml/xapi-cli-server/cli_frontend.ml @@ -511,7 +511,7 @@ let rec cmdtable_data : (string * cmd_spec) list = ; ( "pool-sync-updates" , { reqd= [] - ; optn= ["force"; "token"; "token-id"] + ; optn= ["force"; "token"; "token-id"; "username"; "password"] ; help= "Sync updates from remote YUM repository, pool-wide." ; implementation= No_fd Cli_operations.pool_sync_updates ; flags= [] diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index 62a655b9564..8c9a1dbaf2b 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -1833,8 +1833,11 @@ let pool_sync_updates printer rpc session_id params = let force = get_bool_param params "force" in let token = get_param params "token" ~default:"" in let token_id = get_param params "token-id" ~default:"" in + let username = get_param params "username" ~default:"" in + let password = get_param params "password" ~default:"" in let hash = Client.Pool.sync_updates ~rpc ~session_id ~self:pool ~force ~token ~token_id + ~username ~password in printer (Cli_printer.PList [hash]) diff --git a/ocaml/xapi-consts/api_errors.ml b/ocaml/xapi-consts/api_errors.ml index 6e9b7fdbe06..04912fb4932 100644 --- a/ocaml/xapi-consts/api_errors.ml +++ b/ocaml/xapi-consts/api_errors.ml @@ -1330,6 +1330,12 @@ let can_not_periodic_sync_updates = add_error "CAN_NOT_PERIODIC_SYNC_UPDATES" let repo_should_be_single_one_enabled = add_error "REPO_SHOULD_BE_SINGLE_ONE_ENABLED" +let update_syncing_remote_pool_coordinator_connection_failed = + add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_CONNECTION_FAILED" + +let update_syncing_remote_pool_coordinator_service_failed = + add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_SERVICE_FAILED" + let repository_is_in_use = add_error "REPOSITORY_IS_IN_USE" let repository_cleanup_failed = add_error "REPOSITORY_CLEANUP_FAILED" diff --git a/ocaml/xapi/helpers.ml b/ocaml/xapi/helpers.ml index aff7f383e46..6bd81a84fa1 100644 --- a/ocaml/xapi/helpers.ml +++ b/ocaml/xapi/helpers.ml @@ -2030,19 +2030,24 @@ let with_temp_file ?mode prefix suffix f = let path, channel = Filename.open_temp_file ?mode prefix suffix in finally (fun () -> f (path, channel)) (fun () -> Unix.unlink path) +let with_temp_file_of_content ?mode prefix suffix content f = + let@ temp_file, temp_out_ch = with_temp_file ?mode prefix suffix in + Xapi_stdext_pervasives.Pervasiveext.finally + (fun () -> output_string temp_out_ch content) + (fun () -> close_out temp_out_ch) ; + f temp_file + let with_temp_out_ch_of_temp_file ?mode prefix suffix f = let@ path, channel = with_temp_file ?mode prefix suffix in f (path, channel |> with_temp_out_ch) -let make_external_host_verified_rpc ~__context ext_host_address ext_host_cert - xml = - let@ temp_file, temp_out_ch = with_temp_file "external-host-cert" ".pem" in - Xapi_stdext_pervasives.Pervasiveext.finally - (fun () -> output_string temp_out_ch ext_host_cert) - (fun () -> close_out temp_out_ch) ; +let make_external_host_verified_rpc ~__context host_address host_cert xml = + let@ cert_file = + with_temp_file_of_content "external-host-cert-" ".pem" host_cert + in make_remote_rpc ~__context - ~verify_cert:(Stunnel_client.external_host temp_file) - ext_host_address xml + ~verify_cert:(Stunnel_client.external_host cert_file) + host_address xml module FileSys : sig (* bash-like interface for manipulating files *) diff --git a/ocaml/xapi/pool_periodic_update_sync.ml b/ocaml/xapi/pool_periodic_update_sync.ml index a9755d0cf1e..07bb44965d7 100644 --- a/ocaml/xapi/pool_periodic_update_sync.ml +++ b/ocaml/xapi/pool_periodic_update_sync.ml @@ -140,7 +140,8 @@ let rec update_sync () = ignore (Client.Pool.sync_updates ~rpc ~session_id ~self:(Helpers.get_pool ~__context) - ~force:false ~token:"" ~token_id:"" + ~force:false ~token:"" ~token_id:"" ~username:"" + ~password:"" ) with e -> let exc = Printexc.to_string e in diff --git a/ocaml/xapi/repository.ml b/ocaml/xapi/repository.ml index f9af339734b..8d62a27a84e 100644 --- a/ocaml/xapi/repository.ml +++ b/ocaml/xapi/repository.ml @@ -149,26 +149,78 @@ let get_proxy_params ~__context repo_name = | _ -> ("", "", "") -let sync ~__context ~self ~token ~token_id = +let sync ~__context ~self ~token ~token_id ~username ~password = try let repo_name = get_remote_repository_name ~__context ~self in remove_repo_conf_file repo_name ; - let binary_url, source_url = - match Db.Repository.get_origin ~__context ~self with + let origin = Db.Repository.get_origin ~__context ~self in + + let ( binary_url + , source_url + , repo_gpgcheck + , use_proxy + , client_auth + , server_auth ) = + match origin with | `remote -> + let plugin = "accesstoken" in ( Db.Repository.get_binary_url ~__context ~self , Some (Db.Repository.get_source_url ~__context ~self) + , true + , true + , CdnTokenAuth {token_id; token; plugin} + , DefaultAuth ) | `bundle -> let uri = Uri.make ~scheme:"file" ~path:!Xapi_globs.bundle_repository_dir () in - (Uri.to_string uri, None) + (Uri.to_string uri, None, true, false, NoAuth, NoAuth) | `remote_pool -> - (* TODO: sync with Stunnel.with_client_proxy as otherwise yum - reposync will fail when checking the self signed certificate on - the remote pool. *) - ("", None) + let cert = Db.Repository.get_certificate ~__context ~self in + let repo_binary_url = Db.Repository.get_binary_url ~__context ~self in + let remote_addr = + repo_binary_url |> Repository_helpers.get_remote_pool_coordinator_ip + in + let verified_rpc = + try + Helpers.make_external_host_verified_rpc ~__context remote_addr + cert + with Xmlrpc_client.Connection_reset -> + raise + (Api_errors.Server_error + ( Api_errors + .update_syncing_remote_pool_coordinator_connection_failed + , [] + ) + ) + in + let session_id = + try + Client.Client.Session.login_with_password ~rpc:verified_rpc + ~uname:username ~pwd:password + ~version:Datamodel_common.api_version_string + ~originator:Xapi_version.xapi_user_agent + with + | Http_client.Http_request_rejected _ | Http_client.Http_error _ -> + raise + (Api_errors.Server_error + ( Api_errors + .update_syncing_remote_pool_coordinator_service_failed + , [] + ) + ) + in + let xapi_token = session_id |> Ref.string_of in + let plugin = "xapitoken" in + ( repo_binary_url + , None + , false + , true + , PoolExtHostAuth {xapi_token; plugin} + , StunnelClientProxyAuth + {cert; remote_addr; remote_port= Constants.default_ssl_port} + ) in let gpgkey_path = match Db.Repository.get_gpgkey_path ~__context ~self with @@ -177,69 +229,96 @@ let sync ~__context ~self ~token ~token_id = | s -> s in - let write_initial_yum_config () = - write_yum_config ~source_url ~binary_url ~repo_gpgcheck:true ~gpgkey_path + let write_initial_yum_config ~binary_url = + write_yum_config ~source_url ~binary_url ~repo_gpgcheck ~gpgkey_path ~repo_name in - write_initial_yum_config () ; - clean_yum_cache repo_name ; - (* Remove imported YUM repository GPG key *) - if Pkgs.manager = Yum then - Xapi_stdext_unix.Unixext.rm_rec (get_repo_config repo_name "gpgdir") ; Xapi_stdext_pervasives.Pervasiveext.finally (fun () -> - with_access_token ~token ~token_id @@ fun token_path -> - (* Configure proxy and token *) - let token_param = - match token_path with - | Some p -> - Printf.sprintf "--setopt=%s.accesstoken=file://%s" repo_name p - | None -> - "" - in - let proxy_url_param, proxy_username_param, proxy_password_param = - get_proxy_params ~__context repo_name - in - let Pkg_mgr.{cmd; params} = - [ - "--save" - ; proxy_url_param - ; proxy_username_param - ; proxy_password_param - ; token_param - ] - |> fun config -> Pkgs.config_repo ~repo_name ~config + let config_repo params = + let Pkg_mgr.{cmd; params} = + "--save" :: params |> fun config -> + Pkgs.config_repo ~repo_name ~config + in + ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params) in - ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params) ; - (* Import YUM repository GPG key to check metadata in reposync *) - let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in - ignore (Helpers.call_script cmd params) ; + let make_cache () = + (* Import YUM repository GPG key to check metadata in reposync *) + let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in + ignore (Helpers.call_script cmd params) + in (* Sync with remote repository *) - let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in - Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ; + let sync_repo () = + let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in + Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ; + clean_yum_cache repo_name ; + ignore (Helpers.call_script cmd params) + in + + with_sync_client_auth client_auth @@ fun client_auth -> + with_sync_server_auth server_auth @@ fun binary_url' -> + write_initial_yum_config + ~binary_url:(Option.value binary_url' ~default:binary_url) ; clean_yum_cache repo_name ; - ignore (Helpers.call_script cmd params) + (* Remove imported YUM repository GPG key *) + if Pkgs.manager = Yum then + Xapi_stdext_unix.Unixext.rm_rec (get_repo_config repo_name "gpgdir") ; + let auth_params = + match client_auth with + | Some (auth_file, plugin) -> + let token_param = + Printf.sprintf "--setopt=%s.%s=%s" repo_name plugin + (Uri.make ~scheme:"file" ~path:auth_file () |> Uri.to_string) + in + [token_param] + | None -> + [] + in + let proxy_params = + match use_proxy with + | true -> + let proxy_url_param, proxy_username_param, proxy_password_param = + get_proxy_params ~__context repo_name + in + [proxy_url_param; proxy_username_param; proxy_password_param] + | false -> + [] + in + config_repo (auth_params @ proxy_params) ; + make_cache () ; + sync_repo () ) (fun () -> - (* Rewrite repo conf file as initial content to remove credential related info, - * I.E. proxy username/password and temporary token file path. + (* Rewrite repo conf file as initial content to remove credential + * related info, I.E. proxy username/password and temporary token file + * path. + * One thing to note: for remote_repo, the binary_url used to + * re-initial yum repo is the url configed in the remote_pool repo, + * which is not the correct one for stunnel client proxy, while as we + * will always write_initial_yum_config every time before syncing repo, + * this should be ok. *) - write_initial_yum_config () + write_initial_yum_config ~binary_url ) ; - (* The custom yum-utils will fully download repository metadata.*) - let repodata_dir = + (* The custom yum-utils will fully download repository metadata including + * the repo gpg signature. + *) + let repo_gpg_signature = !Xapi_globs.local_pool_repo_dir // repo_name // "repodata" // "repomd.xml.asc" in - Sys.file_exists repodata_dir - with e -> - error "Failed to sync with remote YUM repository: %s" - (ExnHelper.string_of_exn e) ; - raise Api_errors.(Server_error (reposync_failed, [])) + Sys.file_exists repo_gpg_signature + with + | Api_errors.Server_error (_, _) as e -> + raise e + | e -> + error "Failed to sync with remote YUM repository: %s" + (ExnHelper.string_of_exn e) ; + raise Api_errors.(Server_error (reposync_failed, [])) let http_get_host_updates_in_json ~__context ~host ~installed = let host_session_id = diff --git a/ocaml/xapi/repository.mli b/ocaml/xapi/repository.mli index 5e1c78690fb..3049c003400 100644 --- a/ocaml/xapi/repository.mli +++ b/ocaml/xapi/repository.mli @@ -48,6 +48,8 @@ val sync : -> self:[`Repository] API.Ref.t -> token:string -> token_id:string + -> username:string + -> password:string -> bool val create_pool_repository : diff --git a/ocaml/xapi/repository_helpers.ml b/ocaml/xapi/repository_helpers.ml index 4016a158237..ea7495b32b1 100644 --- a/ocaml/xapi/repository_helpers.ml +++ b/ocaml/xapi/repository_helpers.ml @@ -231,18 +231,22 @@ let assert_gpgkey_path_is_valid path = raise Api_errors.(Server_error (invalid_gpgkey_path, [path])) ) -let assert_remote_pool_url_is_valid ~url = +let get_remote_pool_coordinator_ip url = let uri = Uri.of_string url in match (Uri.scheme uri, Uri.host uri, Uri.path uri) with | Some "https", Some host, path when path = Constants.get_enabled_repository_uri && Helpers.is_valid_ip `ipv4or6 host -> - () + host | _ -> error "Invalid url: %s, expected url format: %s" url ("https://" ^ Constants.get_enabled_repository_uri) ; raise Api_errors.(Server_error (invalid_base_url, [url])) +let assert_remote_pool_url_is_valid ~url = + get_remote_pool_coordinator_ip url + |> Xapi_stdext_pervasives.Pervasiveext.ignore_string + let with_pool_repositories f = Xapi_stdext_pervasives.Pervasiveext.finally (fun () -> @@ -1284,26 +1288,69 @@ let get_single_enabled_update_repository ~__context = in get_singleton enabled_update_repositories -let with_access_token ~token ~token_id f = - match (token, token_id) with - | t, tid when t <> "" && tid <> "" -> - info "sync updates with token_id: %s" tid ; - let json = `Assoc [("token", `String t); ("token_id", `String tid)] in - let tmpfile, tmpch = - Filename.open_temp_file ~mode:[Open_text] "accesstoken" ".json" +type client_auth = + | CdnTokenAuth (* remote *) of { + token_id: string + ; token: string + ; plugin: string + } + | NoAuth (* bundle *) + | PoolExtHostAuth (* remote_pool *) of {xapi_token: string; plugin: string} + +let with_sync_client_auth auth f = + let go_with_client_plugin cred plugin = + let ( let@ ) g x = g x in + let@ temp_file = + Helpers.with_temp_file_of_content ~mode:[Open_text] "token-" ".json" cred + in + f (Some (temp_file, plugin)) + in + match auth with + | CdnTokenAuth {token_id; token; _} when token_id = "" && token = "" -> + f None + | CdnTokenAuth {token_id; token; plugin} -> + let cred = + `Assoc [("token", `String token); ("token_id", `String token_id)] + |> Yojson.Basic.to_string in - Xapi_stdext_pervasives.Pervasiveext.finally - (fun () -> - output_string tmpch (Yojson.Basic.to_string json) ; - close_out tmpch ; - f (Some tmpfile) - ) - (fun () -> Unixext.unlink_safe tmpfile) - | t, tid when t = "" && tid = "" -> + go_with_client_plugin cred plugin + | PoolExtHostAuth {xapi_token; plugin} -> + let cred = + `Assoc [("xapitoken", `String xapi_token)] |> Yojson.Basic.to_string + in + go_with_client_plugin cred plugin + | NoAuth -> f None - | _ -> - let msg = Printf.sprintf "%s: The token or token_id is empty" __LOC__ in - raise Api_errors.(Server_error (internal_error, [msg])) + +type server_auth = + | DefaultAuth (* remote *) + | NoAuth (* bundle *) + | StunnelClientProxyAuth (* remote_pool *) of { + cert: string + ; remote_addr: string + ; remote_port: int + } + +let with_sync_server_auth auth f = + match auth with + | DefaultAuth | NoAuth -> + f None + | StunnelClientProxyAuth {cert; remote_addr; remote_port} -> + let local_host = "127.0.0.1" in + let local_port = !Xapi_globs.local_yum_repo_port in + let ( let@ ) f x = f x in + let@ temp_file = + Helpers.with_temp_file_of_content "external-host-cert-" ".pem" cert + in + let binary_url = + Uri.make ~scheme:"http" ~host:local_host ~port:local_port + ~path:Constants.get_enabled_repository_uri () + |> Uri.to_string + in + Stunnel.with_client_proxy + ~verify_cert:(Stunnel_client.external_host temp_file) + ~remote_host:remote_addr ~remote_port ~local_host ~local_port + @@ fun () -> f (Some binary_url) let prune_updateinfo_for_livepatches latest_lps updateinfo = let livepatches = diff --git a/ocaml/xapi/xapi_pool.ml b/ocaml/xapi/xapi_pool.ml index 5eec626c601..9b7953306dc 100644 --- a/ocaml/xapi/xapi_pool.ml +++ b/ocaml/xapi/xapi_pool.ml @@ -3559,12 +3559,15 @@ let remove_repository ~__context ~self ~value = if Db.Pool.get_repositories ~__context ~self = [] then Db.Pool.set_last_update_sync ~__context ~self ~value:Date.epoch -let sync_repos ~__context ~self ~repos ~force ~token ~token_id = +let sync_repos ~__context ~self ~repos ~force ~token ~token_id ~username + ~password = let open Repository in repos |> List.iter (fun repo -> if force then cleanup_pool_repo ~__context ~self:repo ; - let complete = sync ~__context ~self:repo ~token ~token_id in + let complete = + sync ~__context ~self:repo ~token ~token_id ~username ~password + in (* Dnf and custom yum-utils sync all the metadata including updateinfo, * Thus no need to re-create pool repository *) if Pkgs.manager = Yum && complete = false then @@ -3574,14 +3577,14 @@ let sync_repos ~__context ~self ~repos ~force ~token ~token_id = Db.Pool.set_last_update_sync ~__context ~self ~value:(Date.now ()) ; checksum -let sync_updates ~__context ~self ~force ~token ~token_id = +let sync_updates ~__context ~self ~force ~token ~token_id ~username ~password = Pool_features.assert_enabled ~__context ~f:Features.Updates ; Xapi_pool_helpers.with_pool_operation ~__context ~self ~doc:"pool.sync_updates" ~op:`sync_updates @@ fun () -> let repos = Repository_helpers.get_enabled_repositories ~__context in assert_can_sync_updates ~__context ~repos ; - sync_repos ~__context ~self ~repos ~force ~token ~token_id + sync_repos ~__context ~self ~repos ~force ~token ~token_id ~username ~password let check_update_readiness ~__context ~self:_ ~requires_reboot = (* Pool license check *) @@ -3956,7 +3959,7 @@ let put_bundle_handler (req : Request.t) s _ = (fun () -> try sync_repos ~__context ~self:pool ~repos:[repo] ~force:true - ~token:"" ~token_id:"" + ~token:"" ~token_id:"" ~username:"" ~password:"" |> ignore with _ -> raise Api_errors.(Server_error (bundle_sync_failed, [])) diff --git a/ocaml/xapi/xapi_pool.mli b/ocaml/xapi/xapi_pool.mli index 835a356f782..494a486032b 100644 --- a/ocaml/xapi/xapi_pool.mli +++ b/ocaml/xapi/xapi_pool.mli @@ -360,6 +360,8 @@ val sync_updates : -> force:bool -> token:string -> token_id:string + -> username:string + -> password:string -> string val check_update_readiness : diff --git a/python3/dnf_plugins/accesstoken.py b/python3/dnf_plugins/accesstoken.py index 2537d2a6721..97635fa160b 100644 --- a/python3/dnf_plugins/accesstoken.py +++ b/python3/dnf_plugins/accesstoken.py @@ -10,7 +10,7 @@ class InvalidToken(Exception): - """Token is invlaid""" + """Token is invalid""" def __init__(self, token): super().__init__(f"Invalid token: {token}") diff --git a/python3/dnf_plugins/ptoken.py b/python3/dnf_plugins/ptoken.py index c2ea73fccc8..35b6f9aef70 100644 --- a/python3/dnf_plugins/ptoken.py +++ b/python3/dnf_plugins/ptoken.py @@ -24,6 +24,8 @@ def config(self): for repo_name in self.base.repos: repo = self.base.repos[repo_name] + # Only include the ptoken for repos with a localhost URL, for added safety. + # These will be proxied to the coordinator through stunnel, set up by xapi. if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ and repo.ptoken: secret = "pool_secret=" + ptoken diff --git a/python3/dnf_plugins/xapitoken.py b/python3/dnf_plugins/xapitoken.py new file mode 100644 index 00000000000..377fe33964e --- /dev/null +++ b/python3/dnf_plugins/xapitoken.py @@ -0,0 +1,49 @@ +"""dnf plugin to set xapitoken http header for enabled repos""" +import json +import logging +# Disable the error, it can be import in production env +# and mocked out in unitttest +# pylint: disable=import-error +# pytype: disable=import-error +import dnf +import urlgrabber + + +class InvalidToken(Exception): + """Token is invalid""" + def __init__(self, token): + super().__init__(f"Invalid token: {token}") + + +#pylint: disable=too-few-public-methods +class XapiToken(dnf.Plugin): + """dnf xapitoken plugin class""" + + name = "xapitoken" + + def config(self): + """ DNF plugin config hook, + refer to https://dnf.readthedocs.io/en/latest/api_plugins.html""" + + for repo_name in self.base.repos: + repo = self.base.repos[repo_name] + + token_url = repo.xapitoken + if not token_url or token_url == '': + continue + try: + token_str = urlgrabber.urlopen(token_url).read().strip() + token = json.loads(token_str) + except Exception: #pylint: disable=broad-except + logging.debug("Failed to load token from: %s", token_url) + continue + + if not token.get('xapitoken'): + raise InvalidToken(token) + + # Only include the xapitoken for repos with a localhost URL, for added safety. + # These will be proxied to the remote pool coordinator through stunnel, set up by xapi. + if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ + and repo.xapitoken: + secret = "session_id=" + str(token["xapitoken"]) + repo.set_http_headers([f'cookie:{secret}']) diff --git a/python3/tests/test_dnf_plugins.py b/python3/tests/test_dnf_plugins.py index 2f82b1eb5cb..895317f8778 100644 --- a/python3/tests/test_dnf_plugins.py +++ b/python3/tests/test_dnf_plugins.py @@ -1,4 +1,4 @@ -"""Test module for dnf accesstoken""" +"""Test module for dnf accesstoken, ptoken and xapitoken""" import unittest import sys import json @@ -17,14 +17,16 @@ accesstoken = import_file_as_module("python3/dnf_plugins/accesstoken.py") ptoken = import_file_as_module("python3/dnf_plugins/ptoken.py") +xapitoken = import_file_as_module("python3/dnf_plugins/xapitoken.py") REPO_NAME = "testrepo" -def _mock_repo(a_token=None, p_token=None, baseurl=None): +def _mock_repo(a_token=None, p_token=None, xapi_token=None, baseurl=None): mock_repo = MagicMock() mock_repo.accesstoken = a_token mock_repo.ptoken = p_token + mock_repo.xapitoken = xapi_token mock_repo.baseurl = baseurl mock_base = MagicMock() mock_base.repos = {REPO_NAME: mock_repo} @@ -103,3 +105,60 @@ def test_local_repo_does_not_enable_ptoken_should_ignore_ptoken(self, mock_open) mock_repo = _mock_repo(p_token=False, baseurl=["http://127.0.0.1/some_local_path"]) ptoken.Ptoken(mock_repo.base, MagicMock()).config() assert not mock_repo.set_http_headers.called + +@patch("xapitoken.urlgrabber") +class TestXapitoken(unittest.TestCase): + """Test class for xapitoken dnf plugin""" + + def test_set_http_header_with_xapi_token(self, mock_grabber): + """test config succeed with xapitokan""" + mock_repo = _mock_repo(xapi_token="file:///mock_xapitoken_url", + baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + mock_repo.set_http_headers.assert_called_with( + ['cookie:session_id=valid_token'] + ) + + def test_repo_without_xapi_token(self, mock_grabber): + """If repo has not xapitoken, it should not be blocked""" + mock_repo = _mock_repo() + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_ignore_invalid_token_url(self, mock_grabber): + """If repo provided an invalid token url, it should be ignored""" + mock_repo = _mock_repo(xapi_token="Not_existed") + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_invalid_token_raise_exception(self, mock_grabber): + """Token with right json format, bad content should raise""" + mock_repo = _mock_repo(xapi_token="file:///file_contain_invalid_token", + baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "bad_token": "I am bad guy" + }) + with self.assertRaises(xapitoken.InvalidToken): + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + + def test_remote_repo_ignore_xapitoken(self, mock_grabber): + """non-local repo should just ignore the xapitoken""" + mock_repo = _mock_repo(xapi_token=True, + baseurl=["http://some_remote_token/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_local_repo_does_not_enable_xapitoken_should_ignore_xapitoken(self, mock_grabber): + """local repo which has not enabled xapitoken should just ignore the xapitoken""" + mock_repo = _mock_repo(xapi_token=False, baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called diff --git a/scripts/Makefile b/scripts/Makefile index 503e7838546..6bb740e9df8 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -143,6 +143,8 @@ install: # YUM plugins $(IPROG) yum-plugins/accesstoken.py $(DESTDIR)$(YUMPLUGINDIR) $(IDATA) yum-plugins/accesstoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) + $(IPROG) yum-plugins/xapitoken.py $(DESTDIR)$(YUMPLUGINDIR) + $(IDATA) yum-plugins/xapitoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) $(IPROG) yum-plugins/ptoken.py $(DESTDIR)$(YUMPLUGINDIR) $(IDATA) yum-plugins/ptoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) # maillanguages diff --git a/scripts/yum-plugins/accesstoken.py b/scripts/yum-plugins/accesstoken.py index 0f549f27121..a83d7bf2ce9 100644 --- a/scripts/yum-plugins/accesstoken.py +++ b/scripts/yum-plugins/accesstoken.py @@ -11,16 +11,16 @@ # The content of the file referred by the looks like: # { 'token': '...', 'token_id': '...' } +import json from yum import config from yum.plugins import TYPE_CORE -import json import urlgrabber requires_api_version = '2.5' plugin_type = (TYPE_CORE,) -def config_hook(conduit): +def config_hook(conduit): # pylint: disable=unused-argument config.RepoConf.accesstoken = config.UrlOption() def init_hook(conduit): @@ -35,11 +35,11 @@ def init_hook(conduit): try: token_str = urlgrabber.urlopen(token_url).read().strip() token = json.loads(token_str) - except: + except Exception: #pylint: disable=broad-except continue if not (token['token'] and token['token_id']): - raise Exception("Invalid token or token_id") + raise Exception("Invalid token or token_id") #pylint: disable=broad-exception-raised repo.http_headers['X-Access-Token'] = str(token['token']) repo.http_headers['Referer'] = str(token['token_id']) diff --git a/scripts/yum-plugins/ptoken.py b/scripts/yum-plugins/ptoken.py index 74536e19ee8..0bc0cca0a6d 100755 --- a/scripts/yum-plugins/ptoken.py +++ b/scripts/yum-plugins/ptoken.py @@ -25,7 +25,7 @@ def init_hook(conduit): for name in repos.repos: repo = repos.repos[name] # Only include the ptoken for repos with a localhost URL, for added safety. - # These may be proxied to the coordinator through stunnel, set up by xapi. + # These will be proxied to the coordinator through stunnel, set up by xapi. if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ and repo.getConfigOption('ptoken'): repo.http_headers['cookie'] = "pool_secret=" + ptoken diff --git a/scripts/yum-plugins/xapitoken.conf b/scripts/yum-plugins/xapitoken.conf new file mode 100644 index 00000000000..8e4d76c728b --- /dev/null +++ b/scripts/yum-plugins/xapitoken.conf @@ -0,0 +1,2 @@ +[main] +enabled=1 diff --git a/scripts/yum-plugins/xapitoken.py b/scripts/yum-plugins/xapitoken.py new file mode 100644 index 00000000000..6b959c74462 --- /dev/null +++ b/scripts/yum-plugins/xapitoken.py @@ -0,0 +1,48 @@ +#!/usr/bin/python + +# Drop this file into /usr/lib/yum-plugins/ +# Enable it by creating conf file /etc/yum/pluginconf.d/xapitoken.conf: +# [main] +# enabled=1 +# +# Configure it by: +# yum-config-manager --setopt=.xapitoken=file:// --save + +# The content of the file referred by the looks like: +# { 'xapitoken': '...' } + +import json +from yum import config +from yum.plugins import TYPE_CORE +import urlgrabber + + +requires_api_version = '2.5' +plugin_type = (TYPE_CORE,) + +def config_hook(conduit): # pylint: disable=unused-argument + config.RepoConf.xapitoken = config.UrlOption() + +def init_hook(conduit): + repos = conduit.getRepos() + for name in repos.repos: + repo = repos.repos[name] + token_url = repo.getConfigOption('xapitoken') + if not token_url or token_url == '': + continue + + token = {} + try: + token_str = urlgrabber.urlopen(token_url).read().strip() + token = json.loads(token_str) + except Exception: #pylint: disable=broad-except + continue + + if not token['xapitoken']: + raise Exception("Invalid xapitoken") #pylint: disable=broad-exception-raised + + # Only include the xapitoken for repos with a localhost URL, for added safety. + # These will be proxied to the remote pool coordinator through stunnel, set up by xapi. + if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ + and repo.getConfigOption('xapitoken'): + repo.http_headers['cookie'] = "session_id=" + str(token['xapitoken'])