Merge pull request #237 from telefonicaid/task/upgrade_xena

Task/upgrade xena
telefonicaid · Mar 12, 2024 · 0db2c29 · 0db2c29
2 parents 14cbc11 + 0d63dac
commit 0db2c29
Show file tree

Hide file tree

Showing 16 changed files with 155 additions and 114 deletions.
diff --git a/README.md b/README.md
@@ -21,7 +21,8 @@ a recover procedure password, a second factor authentication (2FA)  and so on.
 - 1.7.0 uses keystone Pike
 - 1.8.0 uses keystone Queens
 - 1.9.0 uses keystone Rocky
-- 1.10.0 and further uses keystone Stein
+- 1.10.0 to 1.17.0 uses keystone Stein
+- 1.18.0 uses keystone Xena
 
 
 ## Installing and Configuration
@@ -120,7 +121,8 @@ https://hub.docker.com/repository/docker/telefonicaiot/fiware-keystone-spassword
 There are some [env vars  for configuration](docs/DOCKER.md)
 
 #### Upgrade from a older version:
-How to upgrade to latest (1.13.0) docker release:
+How to upgrade to latest (1.18.0) docker release:
+Before upgrade to 1.18.0 verison you should upgrade first to 1.17.0 version. Depending on the starting version some steps should be performed before.
 
 Normal procedure is stop container, update version in docker-compose and then up container; then container will be recreated.
 But, if starting version is between 1.4.X and 1.6.X then some steps should be done to achieve that.
@@ -130,6 +132,13 @@ mysqldump -u root -p keystone  > keystone_backup.sql
 ```
 And check in each step of migration that keystone works properly (i.e. is able to authenticate)
 
+Another way to create a copy of keystone database could be:
+```
+create database new_keystone" | mysql -u root -p
+mysqldump -u root --password=<pwd> keystone | mysql -u root -p new_keystone
+```
+
+
 ##### Upgrade from 1.4.X
 -> needs be upgrade to 1.5.4 version before and then perform the steps described for that version.
 In this step is important to use and SQL schema created by Keystone, not just recover from the scratch a sql dump backup, since and sql dump backup has not all required data to migration 1.4.x to 1.5.x will be successfully executed. More info about this issue is found at https://github.com/telefonicaid/fiware-keystone-spassword/issues/194
@@ -158,7 +167,10 @@ mysql -u root -p keystone < table_spassword.sql
 ```
 Restart again keystone container
 
-##### Upgrade from 1.7.0,  1.8.0, 1.9.0
+##### Upgrade from 1.7.0 or upper to 1.17.0
+-> no workaround needed
+
+##### Upgrade from 1.17.0 to 1.18.0
 -> no workaround needed
 
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,11 +1,11 @@
-FROM centos:7.9.2009
+FROM centos:8.4.2105
 
 MAINTAINER IoT team
 
 ENV DB_HOST localhost
 ENV KEYSTONE_ADMIN_PASSWORD 4pass1w0rd
-ENV KEYSTONE_SCIM_VERSION 1.6.0
-ENV KEYSTONE_SPASSWORD_VERSION 1.17.0
+ENV KEYSTONE_SCIM_VERSION 1.7.0
+ENV KEYSTONE_SPASSWORD_VERSION 1.18.0
 
 COPY ./*.sh /opt/keystone/
 COPY ./*.patch /opt/keystone/
@@ -16,34 +16,58 @@ COPY ./keystone-all /opt/keystone/
 WORKDIR /opt/keystone
 
 RUN \
-    # Install dependencies
+    # Install dependencie
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* && \
     yum update -y && yum install -y curl && \
+    # rpm -qa && \
     yum install -y epel-release && yum update -y epel-release && \
-    yum localinstall -y --nogpgcheck qpid-proton-c-0.34.0-2.el7.x86_64.rpm && \
-    yum localinstall -y --nogpgcheck python2-qpid-proton-0.34.0-2.el7.x86_64.rpm && \
     # Install MySQL client
     yum -y install mysql git && \
     # Install keystone dependencies
     yum -y install rpm-build tar findutils && \
-    yum -y install python cronie && \
-    curl -s --insecure -L 'https://repos.fedorapeople.org/openstack/openstack-stein/rdo-release-stein-3.noarch.rpm' > rdo-release-stein-3.noarch.rpm && \
-    yum localinstall -y --nogpgcheck rdo-release-stein-3.noarch.rpm && \
+    yum -y install python3 cronie && \
+    curl -s --insecure -L 'https://repos.fedorapeople.org/openstack/archived/openstack-xena/rdo-release-xena-2.el8.noarch.rpm' > rdo-release-xena-2.el8.noarch.rpm && \
+    yum localinstall -y --nogpgcheck rdo-release-xena-2.el8.noarch.rpm && \
     # Set Centos mirror to ensure openstack version
-    sed -i 's/http:\/\/mirror.centos.org/https:\/\/buildlogs.centos.org/g' /etc/yum.repos.d/rdo-release.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/rdo-release.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://buildlogs.centos.org|g' /etc/yum.repos.d/rdo-release.repo && \
     sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/rdo-release.repo && \
+    sed -i 's/$stream/8-stream/g' /etc/yum.repos.d/rdo-release.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/messaging.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/messaging.repo && \
+    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/messaging.repo && \
+    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/messaging.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/advanced-virtualization.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/advanced-virtualization.repo && \
+    sed -i 's|advancedvirt-common|advanced-virtualization|g' /etc/yum.repos.d/advanced-virtualization.repo && \
+    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/advanced-virtualization.repo && \
+    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/advanced-virtualization.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/nfv-openvswitch.repo && \
+    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
+    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/nfv-openvswitch.repo && \
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/ceph-pacific.repo && \
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/ceph-pacific.repo && \
+    sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/ceph-pacific.repo && \
+    sed -i 's/$stream/8.4.2105/g' /etc/yum.repos.d/ceph-pacific.repo && \
+    yum update -y && \
     # Install keystone dependencies
-    yum -y install openstack-utils openstack-keystone python-keystoneclient  && \
-    yum -y install python-openstackclient && \
-    yum -y install httpd mod_wsgi && \
-    # https://docs.openstack.org/keystone/queens/install/
+    yum -y install crudini openstack-keystone python3-keystoneclient python3-openstackclient && \
+    yum -y install httpd python3-mod_wsgi && \
     sed -i 's/#ServerName www.example.com:80/ServerName 127.0.0.1 /g' /etc/httpd/conf/httpd.conf && \
     sed -i 's/5000/5001/g' /usr/share/keystone/wsgi-keystone.conf && \
     ln -s /usr/share/keystone/wsgi-keystone.conf /etc/httpd/conf.d && \
     systemctl enable httpd.service && \
     cp /opt/keystone/keystone-all /usr/bin && \
     chmod 755 /usr/bin/keystone-all && \
-    yum -y install unzip tcping jq python2-redis && \
-    yum -y install cracklib cracklib-python && \
+    cp /opt/keystone/openstack-config.sh /usr/bin/openstack-config && \
+    chmod 755 /usr/bin/openstack-config && \
+    yum -y install unzip tcping jq python3-redis && \
+    echo "INFO: install cracklib...\n" && \
+    yum -y install cracklib gcc python3-devel && \
+    yum localinstall -y --nogpgcheck cracklib-devel-2.9.6-15.el8.x86_64.rpm && \
+    pip3 install cracklib && \
     # Set keystone configuration
     openstack-config --set /etc/keystone/keystone.conf \
     DEFAULT public_port 5001  && \
@@ -70,8 +94,8 @@ RUN \
     # Keystone postconfig file
     chmod 755 /opt/keystone/postlaunchconfig.sh /opt/keystone/postlaunchconfig_update.sh /opt/keystone/keystone-entrypoint.sh && \
     ln -s /etc/keystone/keystone-paste.ini /usr/share/keystone/keystone-dist-paste.ini && \
-    mkdir -p /usr/lib/python2.7/site-packages/keystone/contrib && \
-    touch /usr/lib/python2.7/site-packages/keystone/contrib/__init__.py && \    
+    mkdir -p /usr/lib/python3.6/site-packages/keystone/contrib && \
+    touch /usr/lib/python3.6/site-packages/keystone/contrib/__init__.py && \    
     # Install Keystone-SCIM
     tag=$KEYSTONE_SCIM_VERSION && \
     user="telefonicaid" && \
@@ -82,7 +106,7 @@ RUN \
     rm -fR $dir && mkdir -p $dir && \
     curl -s --insecure -L "${url_scim}" | tar zxvf - -C ${dir} && \
     cd ${dir}/${repo_scim}-${tag} && \
-    source ./${pack_scim} --with-python27 --with-version $KEYSTONE_SCIM_VERSION --with-release 0 && \
+    source ./${pack_scim} --with-python36 --with-version $KEYSTONE_SCIM_VERSION --with-release 0 && \
     find . -name "*.rpm" -exec rpm -Uvh {} \; && \
     # Install Keystone-SPASSWORD
     cd /opt/keystone && \
@@ -95,7 +119,7 @@ RUN \
     rm -fR $dir && mkdir -p $dir && \
     curl -s --insecure -L "${url_spassword}" | tar zxvf - -C ${dir} && \
     cd ${dir}/${repo_spassword}-${tag} && \
-    source ./${pack_spassword} --with-python27 --with-version $KEYSTONE_SPASSWORD_VERSION --with-release 0 && \
+    source ./${pack_spassword} --with-python36 --with-version $KEYSTONE_SPASSWORD_VERSION --with-release 0 && \
     find . -name "*.rpm" -exec rpm -Uvh {} \; && \
     # LDAP pre-support
     yum -y install openldap-clients && \
@@ -110,45 +134,46 @@ RUN \
     # Federation by SAML2 pre-support
     yum install -y xmlsec1 && \
     # Patching ...
-    ln -s /usr/lib/python2.7/site-packages/keystone/contrib/scim/scim.py  /usr/lib/python2.7/site-packages/keystone/api && \
-    ln -s /usr/lib/python2.7/site-packages/keystone/contrib/spassword/spassword.py  /usr/lib/python2.7/site-packages/keystone/api && \
-    rm -f /usr/lib/python2.7/site-packages/keystone/contrib/spassword/controllers.* && \
-    rm -f /usr/lib/python2.7/site-packages/keystone/contrib/spassword/routers.* && \
-    rm -f /usr/lib/python2.7/site-packages/keystone/contrib/scim/controllers.* && \
-    rm -f /usr/lib/python2.7/site-packages/keystone/contrib/scim/routers.* && \
-    cd /usr/lib/python2.7/site-packages/keystone/identity && \
+    ln -s /usr/lib/python3.6/site-packages/keystone/contrib/scim/scim.py  /usr/lib/python3.6/site-packages/keystone/api && \
+    ln -s /usr/lib/python3.6/site-packages/keystone/contrib/spassword/spassword.py  /usr/lib/python3.6/site-packages/keystone/api && \
+    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/spassword/controllers.* && \
+    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/spassword/routers.* && \
+    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/scim/controllers.* && \
+    rm -f /usr/lib/python3.6/site-packages/keystone/contrib/scim/routers.* && \
+    cd /usr/lib/python3.6/site-packages/keystone/identity && \
     patch -f -p0 < /opt/keystone/core.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/common && \
+    cd /usr/lib/python3.6/site-packages/keystone/common && \
     patch -f -p0 < /opt/keystone/manager.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/auth && \
+    cd /usr/lib/python3.6/site-packages/keystone/auth && \
     patch -f -p0 < /opt/keystone/core_auth.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/server && \
+    cd /usr/lib/python3.6/site-packages/keystone/server && \
     patch -f -p0 < /opt/keystone/backends.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/auth/plugins && \
+    cd /usr/lib/python3.6/site-packages/keystone/auth/plugins && \
     patch -f -p0 < /opt/keystone/auth_plugins_token.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/common && \
+    cd /usr/lib/python3.6/site-packages/keystone/common && \
     patch -f -p0 < /opt/keystone/common_render_token.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/models && \
+    cd /usr/lib/python3.6/site-packages/keystone/models && \
     patch -f -p0 < /opt/keystone/token_model.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/token && \
+    cd /usr/lib/python3.6/site-packages/keystone/token && \
     patch -f -p0 < /opt/keystone/token_provider.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/api && \
+    cd /usr/lib/python3.6/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/apis.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/api && \
+    cd /usr/lib/python3.6/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/api_role_assignments.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/api && \
+    cd /usr/lib/python3.6/site-packages/keystone/api && \
     patch -f -p0 < /opt/keystone/projects.patch && \
-    cd /usr/lib/python2.7/site-packages/keystone/api/_shared && \
+    cd /usr/lib/python3.6/site-packages/keystone/api/_shared && \
     patch -f -p0 < /opt/keystone/api_auth.patch && \
-    find /usr/lib/python2.7/site-packages/keystone -name "*.pyc" -delete && \
-    find /usr/lib/python2.7/site-packages/keystone -name "*.pyo" -delete && \
+    find /usr/lib/python3.6/site-packages/keystone -name "*.pyc" -delete && \
+    find /usr/lib/python3.6/site-packages/keystone -name "*.pyo" -delete && \
     sed -ri \
               -e 's!^(\s*CustomLog)\s+\S+!\1 /proc/1/fd/1!g' \
               -e 's!^(\s*ErrorLog)\s+\S+!\1 /proc/1/fd/2!g' \
               /etc/httpd/conf.d/wsgi-keystone.conf && \
     rm /etc/keystone/logging.conf && \
     # Cleaning unused files...
-    rpm -e --nodeps redhat-logos || true && yum -y erase libss && \
+    yum -y remove gcc python3-devel && \
+    yum -y erase libss && \
     yum clean all && rm -rf /var/lib/yum/yumdb && \
     rm -rf /var/lib/yum/history && find /usr/share/locale -mindepth 1 -maxdepth 1 ! -name 'en' ! -name 'es' ! -name 'es_ES' | xargs rm -r && \
     rm -rf ~/fiware-keystone && rm -f /var/log/*log && rm -f /opt/keystone/*.rpm

diff --git a/docker/core_auth.patch b/docker/core_auth.patch
@@ -1,14 +1,12 @@
 --- core.py   2020-03-04 13:25:28.475608664 +0000
 +++ core.py_new 2020-03-04 13:25:02.063686461 +0000
-@@ -13,5 +13,7 @@ from functools import partial
- import sys
+@@ -13,3 +13,5 @@ from functools import partial
 
  from oslo_log import log
 +from oslo_log import versionutils
 +from oslo_utils import importutils
- import six
  import stevedore
-@@ -42,6 +44,23 @@ def load_auth_method(method):
+@@ -40,6 +42,23 @@ def load_auth_method(method):
     plugin_name = CONF.auth.get(method) or 'default'
     namespace = 'keystone.auth.%s' % method
 -    driver_manager = _get_auth_driver_manager(namespace, plugin_name)

diff --git a/docker/cracklib-devel-2.9.6-15.el8.x86_64.rpm b/docker/cracklib-devel-2.9.6-15.el8.x86_64.rpm
diff --git a/docker/keystone-entrypoint.sh b/docker/keystone-entrypoint.sh
@@ -37,8 +37,10 @@ if [ "$DB_HOST_ARG" == "-dbhost" ]; then
         # Check if previos DB data exists
         mysql -h $DB_HOST_NAME --port $DB_HOST_PORT -u root --password=$MYSQL_PASSWORD_VALUE -e 'use keystone'
         if [ "$?" == "1" ]; then
+            rm -f /var/log/keystone/keystone.log
             /opt/keystone/postlaunchconfig.sh $DB_HOST_ARG $DB_HOST_VALUE $DEFAULT_PASSWORD_ARG $DEFAULT_PASSWORD_VALUE $MYSQL_PASSWORD_ARG $MYSQL_PASSWORD_VALUE $TOKEN_EXPIRATION_TIME_ARG $TOKEN_EXPIRATION_TIME_VALUE
         else
+            rm -f /var/log/keystone/keystone.log
             /opt/keystone/postlaunchconfig_update.sh $DB_HOST_ARG $DB_HOST_VALUE $DEFAULT_PASSWORD_ARG $DEFAULT_PASSWORD_VALUE $MYSQL_PASSWORD_ARG $MYSQL_PASSWORD_VALUE $TOKEN_EXPIRATION_TIME_ARG $TOKEN_EXPIRATION_TIME_VALUE
         fi
     fi
@@ -51,9 +53,11 @@ tail -17 /etc/keystone/keystone.conf
 touch /var/log/keystone/keystone.log
 chmod 777 /var/log/keystone/
 chmod 777 /var/log/keystone/keystone.log
-ln -snf /dev/stdout /var/log/keystone/keystone.log
 echo "[ keystone-entrypoint - keystone-all ] "
 /usr/bin/keystone-all &
+sleep 5
+rm -f /var/log/keystone/keystone.log
+ln -snf /dev/stdout /var/log/keystone/keystone.log
 
 sleep infinity
 

diff --git a/docker/manager.patch b/docker/manager.patch
@@ -1,14 +1,13 @@
 --- manager.py   2020-03-04 13:25:28.475608664 +0000
 +++ manager.py_new 2020-03-04 13:25:02.063686461 +0000
-@@ -18,5 +18,7 @@ import time
+@@ -18,4 +18,6 @@ import types
  import types
 
  from oslo_log import log
 +from oslo_log import versionutils
 +from oslo_utils import importutils
- import six
  import stevedore
-@@ -72,9 +74,20 @@ def load_driver(namespace, driver_name, *args):
+@@ -71,9 +73,20 @@ def load_driver(namespace, driver_name, *args):
                                                   invoke_on_load=True,
                                                   invoke_args=args)
          return driver_manager.driver

diff --git a/docker/openstack-config.sh b/docker/openstack-config.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Copyright (C) 2013, Red Hat, Inc.
+# Pádraig Brady <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+# This functionality is now implemented in the more general crudini
+
+exec crudini "$@"