From aa714001ddce35a81bf1711ac6bc52d1987ff81b Mon Sep 17 00:00:00 2001 From: Stephen Cheng Date: Wed, 26 Feb 2025 07:04:48 +0000 Subject: [PATCH] CP-53444: For XenServer 9, remove python dnf plugins DNF has been updated to dnf5 in XenServer 9 and no longer provides `dnf` python module. We've rewritten the plugins using C++ for dnf5 and built them as rpm packages Signed-off-by: Stephen Cheng --- python3/Makefile | 4 - python3/README.md | 1 - python3/dnf_plugins/__init__.py | 0 python3/dnf_plugins/accesstoken.py | 46 -------- python3/dnf_plugins/ptoken.py | 32 ------ python3/dnf_plugins/xapitoken.py | 49 --------- python3/stubs/dnf.py | 12 --- python3/tests/test_dnf_plugins.py | 164 ----------------------------- 8 files changed, 308 deletions(-) delete mode 100644 python3/dnf_plugins/__init__.py delete mode 100644 python3/dnf_plugins/accesstoken.py delete mode 100644 python3/dnf_plugins/ptoken.py delete mode 100644 python3/dnf_plugins/xapitoken.py delete mode 100644 python3/stubs/dnf.py delete mode 100644 python3/tests/test_dnf_plugins.py diff --git a/python3/Makefile b/python3/Makefile index 1a46b50b164..fb13068ca0e 100644 --- a/python3/Makefile +++ b/python3/Makefile @@ -5,7 +5,6 @@ IPROG=../scripts/install.sh 755 IDATA=../scripts/install.sh 644 SITE3_DIR=$(shell python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -DNF_PLUGIN_DIR=dnf-plugins install: # Create destination directories using install -m 755 -d: @@ -16,12 +15,9 @@ install: install -m 755 -d $(DESTDIR)/etc/sysconfig install -m 755 -d $(DESTDIR)/usr/lib/systemd/system install -m 755 -d $(DESTDIR)$(EXTENSIONDIR) - install -m 755 -d $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR) $(IDATA) packages/inventory.py $(DESTDIR)$(SITE3_DIR)/ $(IDATA) packages/observer.py $(DESTDIR)$(SITE3_DIR)/ - $(IDATA) dnf_plugins/accesstoken.py $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR)/ - $(IDATA) dnf_plugins/ptoken.py $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR)/ $(IPROG) libexec/metrics.py $(DESTDIR)$(OPTDIR)/debug $(IPROG) libexec/metricsgraph.py $(DESTDIR)$(OPTDIR)/debug diff --git a/python3/README.md b/python3/README.md index 9d4e43e3dcd..e2bf8dcc464 100644 --- a/python3/README.md +++ b/python3/README.md @@ -10,4 +10,3 @@ run by xapi and other daemons. - packages: This contains files to be installed in python's site-packages and are meant to be modules and packages to be imported by other scripts or executed via python3 -m - plugins: This contains files that are meant to be xapi plugins -- dnf-plugins: This contains dnf-plugins and are meant to be called automatically by dnf diff --git a/python3/dnf_plugins/__init__.py b/python3/dnf_plugins/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python3/dnf_plugins/accesstoken.py b/python3/dnf_plugins/accesstoken.py deleted file mode 100644 index 97635fa160b..00000000000 --- a/python3/dnf_plugins/accesstoken.py +++ /dev/null @@ -1,46 +0,0 @@ -"""dnf plugin to set accesstoken http header for enabled repos""" -import json -import logging -# Disable the error, it can be import in production env -# and mocked out in unitttest -# pylint: disable=import-error -# pytype: disable=import-error -import dnf -import urlgrabber - - -class InvalidToken(Exception): - """Token is invalid""" - def __init__(self, token): - super().__init__(f"Invalid token: {token}") - - -#pylint: disable=too-few-public-methods -class AccessToken(dnf.Plugin): - """dnf accesstoken plugin class""" - - name = "accesstoken" - - def config(self): - """ DNF plugin config hook, - refer to https://dnf.readthedocs.io/en/latest/api_plugins.html""" - - for repo_name in self.base.repos: - repo = self.base.repos[repo_name] - - token_url = repo.accesstoken - if not token_url or token_url == '': - continue - try: - token_str = urlgrabber.urlopen(token_url).read().strip() - token = json.loads(token_str) - except Exception: #pylint: disable=broad-except - logging.debug("Failed to load token from: %s", token_url) - continue - - if not (token.get('token') and token.get('token_id')): - raise InvalidToken(token) - - access_token = f'X-Access-Token:{str(token["token"])}' - referer = f'Referer:{str(token["token_id"])}' - repo.set_http_headers([access_token, referer]) diff --git a/python3/dnf_plugins/ptoken.py b/python3/dnf_plugins/ptoken.py deleted file mode 100644 index 35b6f9aef70..00000000000 --- a/python3/dnf_plugins/ptoken.py +++ /dev/null @@ -1,32 +0,0 @@ -"""dnf plugin to add ptoken for repos""" -# pytype: disable=import-error -import logging -import dnf - -PTOKEN_PATH = "/etc/xensource/ptoken" - -#pylint: disable=too-few-public-methods -class Ptoken(dnf.Plugin): - """ptoken plugin class""" - - name = "ptoken" - - def config(self): - """ DNF plugin config hook, - refer to https://dnf.readthedocs.io/en/latest/api_plugins.html""" - try: - with open(PTOKEN_PATH, encoding="utf-8") as file: - ptoken = file.read().strip() - except Exception: #pylint: disable=broad-exception-caught - logging.error("Failed to open %s", PTOKEN_PATH) - raise - - for repo_name in self.base.repos: - repo = self.base.repos[repo_name] - - # Only include the ptoken for repos with a localhost URL, for added safety. - # These will be proxied to the coordinator through stunnel, set up by xapi. - if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ - and repo.ptoken: - secret = "pool_secret=" + ptoken - repo.set_http_headers([f'cookie:{secret}']) diff --git a/python3/dnf_plugins/xapitoken.py b/python3/dnf_plugins/xapitoken.py deleted file mode 100644 index 377fe33964e..00000000000 --- a/python3/dnf_plugins/xapitoken.py +++ /dev/null @@ -1,49 +0,0 @@ -"""dnf plugin to set xapitoken http header for enabled repos""" -import json -import logging -# Disable the error, it can be import in production env -# and mocked out in unitttest -# pylint: disable=import-error -# pytype: disable=import-error -import dnf -import urlgrabber - - -class InvalidToken(Exception): - """Token is invalid""" - def __init__(self, token): - super().__init__(f"Invalid token: {token}") - - -#pylint: disable=too-few-public-methods -class XapiToken(dnf.Plugin): - """dnf xapitoken plugin class""" - - name = "xapitoken" - - def config(self): - """ DNF plugin config hook, - refer to https://dnf.readthedocs.io/en/latest/api_plugins.html""" - - for repo_name in self.base.repos: - repo = self.base.repos[repo_name] - - token_url = repo.xapitoken - if not token_url or token_url == '': - continue - try: - token_str = urlgrabber.urlopen(token_url).read().strip() - token = json.loads(token_str) - except Exception: #pylint: disable=broad-except - logging.debug("Failed to load token from: %s", token_url) - continue - - if not token.get('xapitoken'): - raise InvalidToken(token) - - # Only include the xapitoken for repos with a localhost URL, for added safety. - # These will be proxied to the remote pool coordinator through stunnel, set up by xapi. - if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ - and repo.xapitoken: - secret = "session_id=" + str(token["xapitoken"]) - repo.set_http_headers([f'cookie:{secret}']) diff --git a/python3/stubs/dnf.py b/python3/stubs/dnf.py deleted file mode 100644 index 81a773041b8..00000000000 --- a/python3/stubs/dnf.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This is a stub module for dnf.base - -This module is introduced to decouple the dependencies from dnf -modules during unittest, as the github CI ubuntu container has -issues with the python-dnf and python3.11 -""" -#pylint: disable=too-few-public-methods -class Plugin: - """Dnf plugin interface""" - def __init__(self, base, cli): - self.base = base - self.cli = cli diff --git a/python3/tests/test_dnf_plugins.py b/python3/tests/test_dnf_plugins.py deleted file mode 100644 index 895317f8778..00000000000 --- a/python3/tests/test_dnf_plugins.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Test module for dnf accesstoken, ptoken and xapitoken""" -import unittest -import sys -import json -from unittest.mock import MagicMock, patch -from python3.tests.import_helper import import_file_as_module - -sys.modules["urlgrabber"] = MagicMock() - -# Disable wrong import postition as need to mock some sys modules first -#pylint: disable=wrong-import-position - -# Disable unused-argument as some mock obj is not used -#pylint: disable=unused-argument - -# Some test case does not use self - -accesstoken = import_file_as_module("python3/dnf_plugins/accesstoken.py") -ptoken = import_file_as_module("python3/dnf_plugins/ptoken.py") -xapitoken = import_file_as_module("python3/dnf_plugins/xapitoken.py") - -REPO_NAME = "testrepo" - - -def _mock_repo(a_token=None, p_token=None, xapi_token=None, baseurl=None): - mock_repo = MagicMock() - mock_repo.accesstoken = a_token - mock_repo.ptoken = p_token - mock_repo.xapitoken = xapi_token - mock_repo.baseurl = baseurl - mock_base = MagicMock() - mock_base.repos = {REPO_NAME: mock_repo} - mock_repo.base = mock_base - return mock_repo - - -@patch("accesstoken.urlgrabber") -class TestAccesstoken(unittest.TestCase): - """Test class for dnf access plugin""" - - def test_set_http_header_with_access_token(self, mock_grabber): - """test config succeed with accesstokan""" - mock_repo = _mock_repo(a_token="file:///mock_accesstoken_url") - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "token": "valid_token", - "token_id": "valid_token_id", - }) - accesstoken.AccessToken(mock_repo.base, MagicMock()).config() - mock_repo.set_http_headers.assert_called_with( - ['X-Access-Token:valid_token','Referer:valid_token_id'] - ) - - def test_repo_without_access_token(self, mock_grabber): - """If repo has not accestoken, it should not be blocked""" - mock_repo = _mock_repo() - accesstoken.AccessToken(mock_repo.base, MagicMock()).config() - mock_repo.set_http_headers.assert_not_called() - - def test_ignore_invalid_token_url(self, mock_grabber): - """If repo provided an invalid token url, it should be ignored""" - mock_repo = _mock_repo(a_token="Not_existed") - mock_grabber.urlopen.side_effect = FileNotFoundError('') - accesstoken.AccessToken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - - def test_invalid_token_raise_exception(self, mock_grabber): - """Token with right json format, bad content should raise""" - mock_repo = _mock_repo(a_token="file:///file_contain_invalid_token") - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "bad_token": "I am bad guy" - }) - with self.assertRaises(accesstoken.InvalidToken): - accesstoken.AccessToken(mock_repo.base, MagicMock()).config() - - -class TestPtoken(unittest.TestCase): - """Test class for ptoken dnf plugin""" - def test_failed_to_open_ptoken_file(self): - """Exception should raised if the system does not have PTOKEN_PATH""" - # Disable pyright warning as we need to set the PTOKEN_PATH to test the exception - ptoken.PTOKEN_PATH = "/some/not/exist/path" # pyright: ignore[reportAttributeAccessIssue] - with self.assertRaises(Exception): - ptoken.Ptoken(MagicMock(), MagicMock()).config() - - @patch("builtins.open") - def test_set_ptoken_to_http_header(self, mock_open): - """Local repo with ptoken enabled should set the ptoken to its http header""" - mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken" - mock_repo = _mock_repo(p_token=True, baseurl=["http://127.0.0.1/some_local_path"]) - ptoken.Ptoken(mock_repo.base, MagicMock()).config() - mock_repo.set_http_headers.assert_called_with(["cookie:pool_secret=valid_ptoken"]) - - @patch("builtins.open") - def test_remote_repo_ignore_ptoken(self, mock_open): - """non-local repo should just ignore the ptoken""" - mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken" - mock_repo = _mock_repo(p_token=True, baseurl=["http://some_remote_token/some_local_path"]) - ptoken.Ptoken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - - @patch("builtins.open") - def test_local_repo_does_not_enable_ptoken_should_ignore_ptoken(self, mock_open): - """local repo which has not enabled ptoken should just ignore the ptoken""" - mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken" - mock_repo = _mock_repo(p_token=False, baseurl=["http://127.0.0.1/some_local_path"]) - ptoken.Ptoken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - -@patch("xapitoken.urlgrabber") -class TestXapitoken(unittest.TestCase): - """Test class for xapitoken dnf plugin""" - - def test_set_http_header_with_xapi_token(self, mock_grabber): - """test config succeed with xapitokan""" - mock_repo = _mock_repo(xapi_token="file:///mock_xapitoken_url", - baseurl=["http://127.0.0.1/some_local_path"]) - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "xapitoken": "valid_token", - }) - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - mock_repo.set_http_headers.assert_called_with( - ['cookie:session_id=valid_token'] - ) - - def test_repo_without_xapi_token(self, mock_grabber): - """If repo has not xapitoken, it should not be blocked""" - mock_repo = _mock_repo() - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - - def test_ignore_invalid_token_url(self, mock_grabber): - """If repo provided an invalid token url, it should be ignored""" - mock_repo = _mock_repo(xapi_token="Not_existed") - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - - def test_invalid_token_raise_exception(self, mock_grabber): - """Token with right json format, bad content should raise""" - mock_repo = _mock_repo(xapi_token="file:///file_contain_invalid_token", - baseurl=["http://127.0.0.1/some_local_path"]) - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "bad_token": "I am bad guy" - }) - with self.assertRaises(xapitoken.InvalidToken): - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - - def test_remote_repo_ignore_xapitoken(self, mock_grabber): - """non-local repo should just ignore the xapitoken""" - mock_repo = _mock_repo(xapi_token=True, - baseurl=["http://some_remote_token/some_local_path"]) - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "xapitoken": "valid_token", - }) - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called - - def test_local_repo_does_not_enable_xapitoken_should_ignore_xapitoken(self, mock_grabber): - """local repo which has not enabled xapitoken should just ignore the xapitoken""" - mock_repo = _mock_repo(xapi_token=False, baseurl=["http://127.0.0.1/some_local_path"]) - mock_grabber.urlopen.return_value.read.return_value = json.dumps({ - "xapitoken": "valid_token", - }) - xapitoken.XapiToken(mock_repo.base, MagicMock()).config() - assert not mock_repo.set_http_headers.called