Merge pull request juju#12807 from wallyworld/merge-2.9-20210324

juju#12807

Merge 2.9

juju#12761 New controller config for migration-agent-wait-time
juju#12768 Use system instead of core for writing proxy settings
juju#12767 Migration-master worker uses configured minion report timeout
juju#12765 Add MinionReportTimeout to migration-master API
juju#12772 Add link-local field to netplan.Interface
juju#12773 Pull resources for app from store
juju#12771 Upgrade series graph/FSM
juju#12738 Support dynamic instance type discovery on GCE
juju#12787 Make errors from API2Resource users friendly
juju#12805 Support older agent upgrades
juju#12804 Update Pebble: environment variable and "startup: enabled/disabled" fix
juju#12791 Verify charmhub finish series
juju#12801 Move fan logic to core/network
juju#12790 Use os name for indexing juju agents in simplestreams
juju#12798 Update to latest Pebble version
juju#12793 Fix wrong mount volume for init containers;
juju#12792 Remove the force when destroying models
juju#12785 Change 'dialled' to 'dialed'.
juju#12786 Verify charmhub subordinates
juju#12788 Enable RBAC in microk8s
juju#12783 Replace systems with bases for metadata v2

Conflicts mostly due to import changes

Also updated the charm/v9 dependency and remove the version number compatability shims.

```
# Conflicts:
# api/common/charms/common_test.go
# apiserver/facades/client/client/backend.go
# apiserver/facades/client/client/client.go
# apiserver/facades/client/machinemanager/types_mock_test.go
# apiserver/facades/client/machinemanager/upgrade_series.go
# apiserver/gui.go
# apiserver/gui_test.go
# caas/broker.go
# caas/kubernetes/provider/cloud_test.go
# caas/kubernetes/provider/mocks/crd_getter_mock.go
# caas/kubernetes/provider/providerconfig.go
# caas/mocks/broker_mock.go
# cmd/juju/application/refresh.go
# go.mod
# go.sum
# migration/migration.go
# migration/migration_test.go
# provider/common/mocks/zoned_environ.go
# scripts/win-installer/setup.iss
# snap/snapcraft.yaml
# state/action.go
# state/action_test.go
# state/gui.go
# state/migration_export_test.go
# state/migration_import_test.go
# state/migrations/remoteapplications.go
# state/upgrades.go
# testing/base.go
# tools/tools.go
# version/version.go
# worker/caasfirewallerembedded/applicationworker_test.go
# worker/caasoperator/client.go
# worker/upgradedatabase/mocks/agent.go
# worker/upgradedatabase/mocks/package.go
```

## QA steps

See PRs

.github/workflows/microk8s-tests.yml

@@ -83,6 +83,8 @@ jobs:
           set -eux
           git clone https://github.com/juju-solutions/bundle-kubeflow.git
           cd bundle-kubeflow
+          # TODO: tmp fix, remove me later once current kubeflow master branch published.
+          git reset --hard 5e0b6fcb
           git clone git://git.launchpad.net/canonical-osm
           cp -r canonical-osm/charms/interfaces/juju-relation-mysql mysql
 

acceptancetests/assess_caas_deploy_charms.py

@@ -17,14 +17,14 @@
 from time import sleep
 
 import requests
+
 from deploy_stack import BootstrapManager
+from jujupy.k8s_provider import K8sProviderType, providers
+from jujupy.utility import until_timeout
 from utility import (
     JujuAssertionError, add_basic_testing_arguments, configure_logging,
 )
 
-from jujupy.k8s_provider import K8sProviderType, providers
-from jujupy.utility import until_timeout
-
 __metaclass__ = type
 
 
@@ -147,6 +147,11 @@ def parse_args(argv):
         action='store_true',
         help='Bootstrap to k8s cluster or not.'
     )
+    parser.add_argument(
+        '--enable-rbac',
+        action='store_true',
+        help='Deploy workload with RBAC enabled.'
+    )
 
     add_basic_testing_arguments(parser, existing=False)
     return parser.parse_args(argv)
@@ -159,7 +164,11 @@ def main(argv=None):
     k8s_provider = providers[args.caas_provider]
     bs_manager = BootstrapManager.from_args(args)
 
-    with k8s_provider(bs_manager, cluster_name=args.temp_env_name).substrate_context() as caas_client:
+    with k8s_provider(
+        bs_manager,
+        cluster_name=args.temp_env_name,
+        enable_rbac=args.enable_rbac,
+    ).substrate_context() as caas_client:
         # add-k8s --local
         is_mk8s = args.caas_provider == K8sProviderType.MICROK8S.name
         if args.k8s_controller and not is_mk8s:

acceptancetests/assess_caas_deploy_kubeflow.py

@@ -10,25 +10,23 @@
 from __future__ import print_function
 
 import argparse
+import contextlib
 import json
 import logging
-import contextlib
 import os
 import shutil
-import time
+import subprocess
 import sys
 import textwrap
-import subprocess
-from pprint import pformat
+import time
 from time import sleep
 
 from deploy_stack import BootstrapManager
+from jujupy.k8s_provider import K8sProviderType, providers
+from jujupy.utility import until_timeout
 from utility import (
     JujuAssertionError, add_basic_testing_arguments, configure_logging,
 )
-from jujupy.k8s_provider import K8sProviderType, providers
-from jujupy.utility import until_timeout
-
 
 __metaclass__ = type
 log = logging.getLogger("assess_caas_kubeflow_deployment")
@@ -295,27 +293,27 @@ def get_pub_addr(caas_client, model_name):
 
 
 def prepare(caas_client, caas_provider, build):
+
+    for dep in [
+        "charm",
+        "juju-helpers",
+        "juju-wait",
+    ]:
+        if shutil.which(dep):
+            continue
+        caas_client.sh('sudo', 'snap', 'install', dep, '--classic')
+
     if caas_provider == K8sProviderType.MICROK8S.name:
         caas_client.enable_microk8s_addons(
             [
                 "dns", "storage", "dashboard", "ingress", "metallb:10.64.140.43-10.64.140.49",
-                # "rbac",  # TODO: enable `RBAC`?
             ],
         )
         caas_client.kubectl(
             "wait", "--for=condition=available",
             "-nkube-system", "deployment/coredns", "deployment/hostpath-provisioner", "--timeout=10m",
         )
 
-    for dep in [
-        "charm",
-        "juju-helpers",
-        "juju-wait",
-    ]:
-        if shutil.which(dep):
-            continue
-        caas_client.sh('sudo', 'snap', 'install', dep, '--classic')
-
     caas_client.sh('sudo', 'apt', 'update')
     caas_client.sh('sudo', 'apt', 'install', '-y', 'libssl-dev', 'python3-setuptools')
 
@@ -340,7 +338,7 @@ def prepare(caas_client, caas_provider, build):
         )
 
 
-def run_test(caas_provider, k8s_model, bundle, build):
+def run_test(caas_provider, caas_client, k8s_model, bundle, build):
     if caas_provider != K8sProviderType.MICROK8S.name:
         # tests/run.sh only works for microk8s.
         log.info("%s/tests/run.sh is skipped for %s k8s provider", KUBEFLOW_DIR, caas_provider)
@@ -352,6 +350,10 @@ def run_test(caas_provider, k8s_model, bundle, build):
     os.environ['JUJU_DATA'] = k8s_model.env.juju_home
 
     with jump_dir(KUBEFLOW_DIR):
+        if not build:
+            # TODO: tmp fix, remove me later once current kubeflow master branch published.
+            caas_client.sh('git', 'reset', '--hard', '5e0b6fcb')
+
         run("sg", "microk8s", "-c", f"{KUBEFLOW_DIR}/tests/run.sh -m {bundle}")
 
 
@@ -416,7 +418,7 @@ def fail_hook():
         log.info("sleeping for 30 seconds to let everything start up")
         sleep(30)
 
-        run_test(caas_provider, k8s_model, bundle, build)
+        run_test(caas_provider, caas_client, k8s_model, bundle, build)
         k8s_model.juju(k8s_model._show_status, ('--format', 'tabular'))
         success_hook()
     except:  # noqa: E722
@@ -469,7 +471,11 @@ def main(argv=None):
     k8s_provider = providers[args.caas_provider]
     bs_manager = BootstrapManager.from_args(args)
 
-    with k8s_provider(bs_manager, cluster_name=args.temp_env_name).substrate_context() as caas_client:
+    with k8s_provider(
+        bs_manager,
+        cluster_name=args.temp_env_name,
+        enable_rbac=args.enable_rbac,
+    ).substrate_context() as caas_client:
         # add-k8s --local
         if args.k8s_controller and args.caas_provider != K8sProviderType.MICROK8S.name:
             # microk8s is built-in cloud, no need run add-k8s for bootstrapping.

acceptancetests/assess_upgrade.py

@@ -60,7 +60,7 @@
 log = logging.getLogger("assess_upgrade")
 
 
-VersionParts = namedtuple('VersionParts', ['version', 'series', 'arch'])
+VersionParts = namedtuple('VersionParts', ['version', 'release', 'arch'])
 
 
 def assess_upgrade_from_stable_to_develop(args, stable_bsm, devel_client):
@@ -88,9 +88,9 @@ def assess_upgrade_passing_agent_stream(args, devel_client):
     stable_client = stable_bsm.client
     devel_version_parts = get_version_parts(devel_client.version)
     forced_devel_client = devel_client.clone(
-        version='{vers}-{series}-{arch}'.format(
+        version='{vers}-{release}-{arch}'.format(
             vers=increment_version(devel_client.version),
-            series=devel_version_parts.series,
+            release=devel_version_parts.release,
             arch=devel_version_parts.arch))
     with temp_dir() as base_dir:
         stream_server = StreamServer(base_dir)
@@ -176,14 +176,12 @@ def setup_agent_metadata(
         stream,
         version_parts.version,
         version_parts.arch,
-        client.env.get_option('default-series'),
         agent_details)
     # Trusty needed for wikimedia charm.
     stream_server.add_product(
         stream,
         version_parts.version,
         version_parts.arch,
-        'trusty',
         agent_details)
 
 

acceptancetests/jujupy/k8s_provider/base.py

@@ -85,6 +85,8 @@ class Base(object):
     default_storage_class_name = None
     kubeconfig_cluster_name = None
 
+    enable_rbac = None
+
     def _ensure_cluster_stack(self):
         """ensures or checks if stack/infrastructure is ready to use.
         - ensures kubectl/apiserver is functioning.
@@ -114,7 +116,7 @@ def _tear_down_substrate(self):
         """
         raise NotImplementedError()
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
         self.client = bs_manager.client
         self.bs_manager = bs_manager
         # register cleanup_hook.
@@ -126,6 +128,7 @@ def __init__(self, bs_manager, cluster_name=None, timeout=1800):
             self.cluster_name += f'-{suffix}'
         self.cluster_name = self.cluster_name.lower()
 
+        self.enable_rbac = enable_rbac
         self.timeout = timeout
         old_environment = bs_manager.client.env.environment
 

acceptancetests/jujupy/k8s_provider/microk8s.py

@@ -19,23 +19,20 @@
 
 from __future__ import print_function
 
+import json
 import logging
-import shutil
 import os
-import json
-import yaml
+import shutil
 from pprint import pformat
 from time import sleep
 
 import dns.resolver
+import yaml
 
 from jujupy.utility import until_timeout
-from .base import (
-    Base,
-    K8sProviderType,
-)
-from .factory import register_provider
 
+from .base import Base, K8sProviderType
+from .factory import register_provider
 
 logger = logging.getLogger(__name__)
 
@@ -93,6 +90,12 @@ def _microk8s_status(self, wait_ready=False):
     def enable_microk8s_addons(self, addons=None):
         # addons are required to be enabled.
         addons = addons or ['storage', 'dns', 'ingress']
+        if self.enable_rbac:
+            if 'rbac' not in addons:
+                addons.append('rbac')
+        else:
+            addons = [addon for addon in addons if addon != 'rbac']
+            logger.info('disabling rbac -> ', self.sh('microk8s.disable', 'rbac'))
 
         def wait_until_ready(timeout, checker):
             for _ in until_timeout(timeout):

acceptancetests/jujupy/stream_server.py

@@ -65,21 +65,19 @@ def __init__(self, working_dir):
 
         os.makedirs(self._agent_path)
 
-    def add_product(self, content_id, version, arch, series, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'release')
         :param version: Juju version string for product (i.e. '2.3.3')
         :param arch: Architecture string of this product (e.g. 's390x','amd64')
-        :param series: Series string that appears in item_name
-          (e.g. 'bionic', 'xenial', 'centos')
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
         """
         shutil.copy(agent_tgz_path, self._agent_path)
         product_dict = _generate_product_json(
-            content_id, version, arch, series, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path)
         self.products.append(product_dict)
 
     def generate_stream_data(self):
@@ -108,20 +106,18 @@ def __init__(self, base_dir, stream_data_type=_JujuStreamData):
         self.base_dir = base_dir
         self.stream_data = stream_data_type(base_dir)
 
-    def add_product(self, content_id, version, arch, series, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'released')
         :param version: Juju version string for product (i.e. '2.3.3')
         :param arch: Architecture string of this product (e.g. 's390x','amd64')
-        :param series: Series string that appears in item_name
-          (e.g. 'bionic', 'xenial', 'centos')
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
         """
         self.stream_data.add_product(
-            content_id, version, arch, series, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path)
         # Re-generate when adding a product allows updating the server while
         # running.
         # Can be noisey in the logs, if a lot of products need to be added can
@@ -154,7 +150,7 @@ def server(self):
 
 
 def agent_tgz_from_juju_binary(
-        juju_bin_path, tmp_dir, series=None, force_version=None):
+        juju_bin_path, tmp_dir, release=None, force_version=None):
     """
     Create agent tarball with jujud found with provided juju binary.
 
@@ -163,19 +159,10 @@ def agent_tgz_from_juju_binary(
 
     :param juju_bin_path: The path to the juju bin in use.
     :param tmp_dir: Location to store the generated agent file.
-    :param series: String series to use instead of that of the passed binary.
-      Allows one to overwrite the series of the juju client.
+    :param release: String release (os name) to use instead of that of the passed binary.
+      Allows one to overwrite the os name of the juju client.
     :returns: String path to generated
     """
-    def _series_lookup(series):
-        # Handle the inconsistencies with agent series names.
-        if series is None:
-            return None
-        if series.startswith('centos'):
-            return series
-        if series.startswith('win'):
-            return 'win2012'
-        return 'ubuntu'
 
     bin_dir = os.path.dirname(juju_bin_path)
     try:
@@ -188,8 +175,7 @@ def _series_lookup(series):
     try:
         version_output = subprocess.check_output(
             [jujud_path, 'version']).rstrip(str.encode('\n'))
-        version, bin_series, arch = get_version_string_parts(version_output)
-        bin_agent_series = _series_lookup(bin_series)
+        version, bin_agent_release, arch = get_version_string_parts(version_output)
     except subprocess.CalledProcessError as e:
         raise RuntimeError(
             'Unable to query jujud for version details: {}'.format(e))
@@ -199,9 +185,9 @@ def _series_lookup(series):
             'string: {}'.format(version_output))
 
     version = force_version or version
-    agent_tgz_name = 'juju-{version}-{series}-{arch}.tgz'.format(
+    agent_tgz_name = 'juju-{version}-{release}-{arch}.tgz'.format(
         version=version,
-        series=series if series else bin_agent_series,
+        release=release if release else bin_agent_release,
         arch=arch
     )
 
@@ -224,25 +210,22 @@ def _series_lookup(series):
 
 def _generate_product_json(content_id, version, arch, series, agent_tgz_path):
     """Return dict containing product metadata from provided args."""
-    series_name, series_code = _get_series_details(series)
     tgz_name = os.path.basename(agent_tgz_path)
     file_details = _get_tgz_file_details(agent_tgz_path)
-    item_name = '{version}-{series}-{arch}'.format(
+    item_name = '{version}-ubuntu-{arch}'.format(
         version=version,
-        series='{}:{}'.format(series_name, series_code),
         arch=arch)
     return dict(
         arch=arch,
-        content_id='com.ubuntu.juju:{}:tools'.format(content_id),
+        content_id='com.ubuntu.juju:{}:agents'.format(content_id),
         format='products:1.0',
         ftype='tar.gz',
         item_name=item_name,
         md5=file_details['md5'],
         path=os.path.join('agent', tgz_name),
-        product_name='com.ubuntu.juju:{series_code}:{arch}'.format(
-            series_code=series_code,
+        product_name='com.ubuntu.juju:ubuntu:{arch}'.format(
             arch=arch),
-        release=series_name,
+        release='ubuntu',
         sha256=file_details['sha256'],
         size=file_details['size'],
         version=version,