From cb01b0549468df266c2970345649ec7b9352da5b Mon Sep 17 00:00:00 2001 From: Jonathan Martinez Montes Date: Wed, 7 Apr 2021 05:03:07 +0000 Subject: [PATCH] DAOS-4265 build: Update fio to 3.26 Signed-off-by: Jonathan Martinez Montes --- Makefile | 9 +- fio.spec | 11 +- packaging/Dockerfile.centos.7 | 4 +- packaging/Dockerfile.mockbuild | 11 +- packaging/Dockerfile.ubuntu.18.04 | 29 --- packaging/Dockerfile.ubuntu.20.04 | 39 +++ packaging/Makefile_distro_vars.mk | 108 ++++++++ packaging/Makefile_packaging.mk | 399 +++++++++++++----------------- packaging/rpm_chrootbuild | 47 ++++ 9 files changed, 388 insertions(+), 269 deletions(-) delete mode 100644 packaging/Dockerfile.ubuntu.18.04 create mode 100644 packaging/Dockerfile.ubuntu.20.04 create mode 100644 packaging/Makefile_distro_vars.mk create mode 100755 packaging/rpm_chrootbuild diff --git a/Makefile b/Makefile index e5b6e17..7ced4e4 100644 --- a/Makefile +++ b/Makefile @@ -2,4 +2,11 @@ NAME := fio SRC_EXT := bz2 SOURCE = http://brick.kernel.dk/snaps/$(NAME)-$(VERSION).tar.$(SRC_EXT) -include packaging/Makefile_packaging.mk \ No newline at end of file +include source_deps.mk + +include packaging/Makefile_packaging.mk + +source_deps.mk: + for s in $(SOURCES); do \ + echo $${s##*/}:; \ + done > $@ diff --git a/fio.spec b/fio.spec index fa7637e..7875f72 100644 --- a/fio.spec +++ b/fio.spec @@ -1,17 +1,19 @@ Name: fio -Version: 3.3 -Release: 5%{?dist} +Version: 3.26 +Release: 1%{?dist} Summary: Multithreaded IO generation tool Group: Applications/System License: GPLv2 URL: http://git.kernel.dk/?p=fio.git;a=summary Source: http://brick.kernel.dk/snaps/%{name}-%{version}.tar.bz2 +Patch0: https://github.com/axboe/fio/compare/267b164..c363fdd.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildRequires: libaio-devel BuildRequires: zlib-devel +BuildRequires: libuuid-devel %ifarch x86_64 BuildRequires: libpmem-devel BuildRequires: libpmemblk-devel @@ -50,6 +52,7 @@ FIO devel %prep %setup -q +%patch0 -p1 -b .267b164..c363fdd.patch find . -type f > source_files.txt # fix hard-coded /usr/bin/bash sed -i -e "s|/usr/bin/bash|$(command -v bash)|g" tools/genfio @@ -86,6 +89,10 @@ rm -rf $RPM_BUILD_ROOT %{_usrsrc}/%{name}-%{version} %changelog +* Wed Apr 07 2021 Jonathan Martinez Montes - 3.26-1 +- New upstream version +- Add 267b164..c363fdd patch that adds DAOS support + * Tue Jul 30 2019 Brian J. Murrell 3.3-5 - SUSE really wants source files in -devel, not -src diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 70ba804..86febdc 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,5 +1,5 @@ # -# Copyright 2018-2019, Intel Corporation +# Copyright 2018-2021, Intel Corporation # # 'recipe' for Docker to build an RPM # @@ -28,7 +28,7 @@ RUN echo "$USER:$PASSWD" | chpasswd RUN usermod -a -G mock $USER # mock in Docker needs to use the old-chroot option -RUN grep use_nspawn || \ +RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg RUN chmod g+w /etc/mock/default.cfg diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index aa660b5..b098dd0 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,5 +1,5 @@ # -# Copyright 2018-2019, Intel Corporation +# Copyright 2018-2021, Intel Corporation # # 'recipe' for Docker to build an RPM # @@ -12,8 +12,9 @@ MAINTAINER daos-stack ARG UID=1000 # Install basic tools -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros +RUN dnf -y install mock make \ + rpm-build curl createrepo rpmlint redhat-lsb-core git \ + python-srpm-macros rpmdevtools # Add build user (to keep rpmbuild happy) ENV USER build @@ -24,7 +25,7 @@ RUN echo "$USER:$PASSWD" | chpasswd RUN usermod -a -G mock $USER # mock in Docker needs to use the old-chroot option -RUN grep use_nspawn || \ +RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg -RUN chmod g+w /etc/mock/* \ No newline at end of file +RUN chmod g+w /etc/mock/* diff --git a/packaging/Dockerfile.ubuntu.18.04 b/packaging/Dockerfile.ubuntu.18.04 deleted file mode 100644 index 2180c9a..0000000 --- a/packaging/Dockerfile.ubuntu.18.04 +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright 2019, Intel Corporation -# -# 'recipe' for Docker to build an Debian package -# -# Pull base image -FROM ubuntu:18.04 -Maintainer daos-stack - -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 - -# Install basic tools -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ - git git-buildpackage locales make patch pbuilder rpm wget - -# Add build user (to keep chrootbuild happy) -ENV USER build -RUN useradd -u $UID -ms /bin/bash $USER - -# need to run the build command as root, as it needs to chroot -RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ - echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ - fi; \ - echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" > /etc/sudoers.d/build; \ - chmod 0440 /etc/sudoers.d/build; \ - visudo -c; \ - sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 new file mode 100644 index 0000000..893ad2d --- /dev/null +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -0,0 +1,39 @@ +# +# Copyright 2019-2021, Intel Corporation +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +FROM ubuntu:20.04 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 +ARG REPO_URL="" +ARG REPO_UBUNTU_20_04="" + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ + git git-buildpackage locales make patch pbuilder rpm wget \ + ca-certificates scons python3-distutils pkg-config \ + python3-dev python3-distro + +# rpmdevtools +RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + rpmdevtools + +# Add build user (to keep chrootbuild happy) +ENV USER build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk new file mode 100644 index 0000000..c126c81 --- /dev/null +++ b/packaging/Makefile_distro_vars.mk @@ -0,0 +1,108 @@ +DOT := . + +DOCKER := docker + +# Find out what we are +ID_LIKE := $(shell . /etc/os-release; echo $$ID_LIKE) +# Of course that does not work for SLES-12 +ID := $(shell . /etc/os-release; echo $$ID) +VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID) +VERSION_CODENAME := $(shell . /etc/os-release; echo $$VERSION_CODENAME) + +ifeq ($(ID_LIKE),debian) +SPECTOOL := spectool +UBUNTU_VERS := $(shell . /etc/os-release; echo $$VERSION) +DISTRO_ID_OPT := --distribution $(VERSION_CODENAME) +DISTRO_ID := ubuntu$(VERSION_ID) +VERSION_ID_STR := $(subst $(DOT),_,$(VERSION_ID)) +DISTRO_BASE := UBUNTU_$(VERSION_ID_STR) +endif +ifeq ($(ID),fedora) +DOCKER := podman +SPECTOOL := spectool +# a Fedora-based mock builder +# derive the the values of: +# VERSION_ID (i.e. 7) +# DISTRO_ID (i.e. el7) +# DISTRO_BASE (i.e. EL_7) +# from the CHROOT_NAME +ifeq ($(CHROOT_NAME),epel-7-x86_64) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 7 +DISTRO_ID := el7 +DISTRO_BASE := EL_7 +SED_EXPR := 1s/$(DIST)//p +endif +ifeq ($(CHROOT_NAME),epel-8-x86_64) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 8 +DISTRO_ID := el8 +DISTRO_BASE := EL_8 +SED_EXPR := 1s/$(DIST)//p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.1-x86_64) +VERSION_ID := 15.1 +DISTRO_ID := sl15.1 +DISTRO_BASE := LEAP_15 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) +VERSION_ID := 15.2 +DISTRO_ID := sl15.2 +DISTRO_BASE := LEAP_15 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),leap-42.3-x86_64) +# TBD if support is ever resurrected +endif +ifeq ($(CHROOT_NAME),sles-12.3-x86_64) +# TBD if support is ever resurrected +endif +endif +ifeq ($(ID),centos) +DISTRO_ID := el$(VERSION_ID) +DISTRO_BASE := $(basename EL_$(VERSION_ID)) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +SED_EXPR := 1s/$(DIST)//p +SPECTOOL := spectool +define install_repo + if yum-config-manager --add-repo=$(1); then \ + repo_file=$$(ls -tar /etc/yum.repos.d/*.repo | tail -1); \ + sed -i -e 1d -e '$$s/^/gpgcheck=False/' $$repo_file; \ + else \ + exit 1; \ + fi +endef +endif +ifeq ($(findstring opensuse,$(ID)),opensuse) +ID_LIKE := suse +DISTRO_ID := sl$(VERSION_ID) +DISTRO_BASE := $(basename LEAP_$(VERSION_ID)) +endif +ifeq ($(ID),sles) +# SLES-12 or 15 detected. +ID_LIKE := suse +DISTRO_ID := sle$(VERSION_ID) +DISTRO_BASE := $(basename SLES_$(VERSION_ID)) +endif +ifeq ($(ID_LIKE),suse) +SPECTOOL := rpmdev-spectool +define install_repo + zypper --non-interactive ar $(1) +endef +endif +ifeq ($(ID_LIKE),debian) +ifndef LANG +export LANG = C.UTF-8 +endif +ifndef LC_ALL +export LC_ALL = C.UTF-8 +endif +else +ifndef LANG +export LANG = C.utf8 +endif +ifndef LC_ALL +export LC_ALL = C.utf8 +endif +endif diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index e859476..c7dcb0d 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -1,134 +1,93 @@ # Common Makefile for including -# Needs the following variables set at a minium: +# Needs the following variables set at a minimum: # NAME := # SRC_EXT := -# SOURCE = + +# force bash (looking at you Ubuntu) +SHELL=/bin/bash # Put site overrides (i.e. REPOSITORY_URL, DAOS_STACK_*_LOCAL_REPO) in here -include Makefile.local +# default to Leap 15 distro for chrootbuild +CHROOT_NAME ?= opensuse-leap-15.2-x86_64 +include packaging/Makefile_distro_vars.mk + ifeq ($(DEB_NAME),) DEB_NAME := $(NAME) endif CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) +TOPDIR ?= $(CURDIR) +BUILD_PREFIX ?= . + DOT := . RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) -# Find out what we are -ID_LIKE := $(shell . /etc/os-release; echo $$ID_LIKE) -# Of course that does not work for SLES-12 -ID := $(shell . /etc/os-release; echo $$ID) -VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID) -ifeq ($(ID_LIKE),debian) -UBUNTU_VERS := $(shell . /etc/os-release; echo $$VERSION) -ifeq ($(VERSION_ID),19.04) -# Bug - distribution is set to "devel" -DISTRO_ID_OPT = --distribution disco -endif -DISTRO_ID := ubuntu$(VERSION_ID) -DISTRO_BASE = $(basename UBUNTU_$(VERSION_ID)) -VERSION_ID_STR := $(subst $(DOT),_,$(VERSION_ID)) -endif -ifeq ($(ID),fedora) -# a Fedora-based mock builder -# derive the the values of: -# VERSION_ID (i.e. 7) -# DISTRO_ID (i.e. el7) -# DISTRO_BASE (i.e. EL_7) -# from the CHROOT_NAME -ifeq ($(CHROOT_NAME),epel-7-x86_64) -VERSION_ID := 7 -DISTRO_ID := el7 -DISTRO_BASE := EL_7 -endif -ifeq ($(CHROOT_NAME),opensuse-leap-15.1-x86_64) -VERSION_ID := 15.1 -DISTRO_ID := sl15.1 -DISTRO_BASE := LEAP_15 -endif -ifeq ($(CHROOT_NAME),leap-42.3-x86_64) -# TBD if support is ever resurrected -endif -ifeq ($(CHROOT_NAME),sles-12.3-x86_64) -# TBD if support is ever resurrected -endif -endif -ifeq ($(ID),centos) -DISTRO_ID := el$(VERSION_ID) -DISTRO_BASE := $(basename EL_$(VERSION_ID)) -define install_repo - if yum-config-manager --add-repo=$(1); then \ - repo_file=$$(ls -tar /etc/yum.repos.d/*.repo | tail -1); \ - sed -i -e 1d -e '$$s/^/gpgcheck=False/' $$repo_file; \ - else \ - exit 1; \ - fi -endef -endif -ifeq ($(findstring opensuse,$(ID)),opensuse) -ID_LIKE := suse -DISTRO_ID := sl$(VERSION_ID) -DISTRO_BASE := $(basename LEAP_$(VERSION_ID)) -endif -ifeq ($(ID),sles) -# SLES-12 or 15 detected. -ID_LIKE := suse -DISTRO_ID := sle$(VERSION_ID) -DISTRO_BASE := $(basename SLES_$(VERSION_ID)) -endif -ifeq ($(ID_LIKE),suse) -define install_repo - zypper --non-interactive ar $(1) -endef -endif -BUILD_OS ?= leap.15 -CHROOT_NAME ?= opensuse-leap-15.1-x86_64 +# some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging -COMMON_RPM_ARGS := --define "%_topdir $$PWD/_topdir" $(BUILD_DEFINES) -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -ifeq ($(DIST),) -SED_EXPR := 1p -else -SED_EXPR := 1s/$(DIST)//p -endif -SPEC := $(NAME).spec -VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p') -DEB_VERS := $(subst rc,~rc,$(VERSION)) -DEB_RVERS := $(subst $(DOT),\$(DOT),$(DEB_VERS)) -DEB_BVERS := $(basename $(subst ~rc,$(DOT)rc,$(DEB_VERS))) -RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)') -SRPM := _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm -RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC)))) -DEB_TOP := _topdir/BUILD -DEB_BUILD := $(DEB_TOP)/$(NAME)-$(DEB_VERS) -DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(DEB_VERS) -SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES)) +LOCAL_REPOS ?= true +TEST_PACKAGES ?= ${NAME} + +# unfortunately we cannot always name the repo the same as the project +REPO_NAME ?= $(NAME) + +PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') +EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') +UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') + +COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) +SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) +VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) +DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) +DEB_BVERS := $(basename $(subst ~rc,$(DOT)rc,$(VERSION))) +RELEASE = $(eval RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)'))$(RELEASE) +SRPM = _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm +RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC)))))$(RPMS) +DEB_TOP := _topdir/BUILD +DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) +DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) +SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\\\#/\\\\\#/g' -e 's/.*: *//'))$(SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) -DEBS := $(addsuffix _$(DEB_VERS)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' debian/control)) -DEB_PREV_RELEASE := $(shell dpkg-parsechangelog -S version) -DEB_DSC := $(DEB_NAME)_$(DEB_PREV_RELEASE)$(GIT_INFO).dsc -#Ubuntu Containers do not set a UTF-8 environment by default. -ifndef LANG -export LANG = C.UTF-8 -endif -ifndef LC_ALL -export LC_ALL = C.UTF-8 +DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) +DEB_PREV_RELEASE := $(shell cd $(TOPDIR) && dpkg-parsechangelog -S version) +ifneq ($(GIT_SHORT),) +GIT_INFO ?= .$(GIT_NUM_COMMITS).g$(GIT_SHORT) endif +DEB_DSC := $(DEB_NAME)_$(DEB_PREV_RELEASE)$(GIT_INFO).dsc TARGETS := $(DEBS) else -# CentOS/Suse packages that want a locale set need this. -ifndef LANG -export LANG = en_US.utf8 -endif -ifndef LC_ALL -export LC_ALL = en_US.utf8 -endif TARGETS := $(RPMS) $(SRPM) endif +define distro_map + case $(DISTRO_ID) in \ + el7) distro="centos7" \ + ;; \ + el8) distro="centos8" \ + ;; \ + sle12.3) distro="sles12.3" \ + ;; \ + sl42.3) distro="leap42.3" \ + ;; \ + sl15.*) distro="leap15" \ + ;; \ + ubuntu*) distro="$(DISTRO_ID)" \ + ;; \ + esac; +endef + define install_repos + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl); \ + done for repo in $($(DISTRO_BASE)_PR_REPOS) \ $(PR_REPOS) $(1); do \ branch="master"; \ @@ -141,16 +100,7 @@ define install_repos branch="$${branch%:*}"; \ fi; \ fi; \ - case $(DISTRO_ID) in \ - el7) distro="centos7"; \ - ;; \ - sle12.3) distro="sles12.3"; \ - ;; \ - sl42.3) distro="leap42.3"; \ - ;; \ - sl15.1) distro="leap15"; \ - ;; \ - esac; \ + $(call distro_map) \ baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/; \ baseurl+=$$build_number/artifact/artifacts/$$distro/; \ $(call install_repo,$$baseurl); \ @@ -181,24 +131,31 @@ _topdir/SOURCES/%: % | _topdir/SOURCES/ # At least one spec file, SLURM (sles), has a different version for the # download file than the version in the spec file. ifeq ($(DL_VERSION),) -DL_VERSION = $(VERSION) +DL_VERSION = $(subst ~,,$(VERSION)) +endif +ifeq ($(DL_NAME),) +DL_NAME = $(NAME) endif -$(NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(NAME)-*.tar.{gz,bz*,xz}.asc - curl -f -L -O '$(SOURCE).asc' +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.asc + spectool -g $(SPEC) + +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).sig: $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.sig + spectool -g $(SPEC) -$(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(NAME)-*.tar.{gz,bz*,xz} - curl -f -L -O '$(SOURCE)' +$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) + rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz} + spectool -g $(SPEC) v$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./v*.tar.{gz,bz*,xz} - curl -f -L -O '$(SOURCE)' + spectool -g $(SPEC) $(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) rm -f ./*.tar.{gz,bz*,xz} - curl -f -L -O '$(SOURCE)' + spectool -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ @@ -207,7 +164,7 @@ $(DEB_BUILD)/%: % | $(DEB_BUILD)/ $(DEB_BUILD).tar.$(SRC_EXT): $(notdir $(SOURCE)) | $(DEB_TOP)/ ln -f $< $@ -$(DEB_TARBASE).orig.tar.$(SRC_EXT) : $(DEB_BUILD).tar.$(SRC_EXT) +$(DEB_TARBASE).orig.tar.$(SRC_EXT): $(DEB_BUILD).tar.$(SRC_EXT) rm -f $(DEB_TOP)/*.orig.tar.* ln -f $< $@ @@ -250,19 +207,20 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ # Move the debian files into the Debian directory. ifeq ($(ID_LIKE),debian) -$(DEB_TOP)/.deb_files : $(shell find debian -type f) deb_detar | \ +$(DEB_TOP)/.deb_files: $(shell find $(TOPDIR)/debian -type f) deb_detar | \ $(DEB_BUILD)/debian/ - find debian -maxdepth 1 -type f -exec cp '{}' '$(DEB_BUILD)/{}' ';' - if [ -e debian/source ]; then \ - cp -r debian/source $(DEB_BUILD)/debian; fi - if [ -e debian/local ]; then \ - cp -r debian/local $(DEB_BUILD)/debian; fi - if [ -e debian/examples ]; then \ - cp -r debian/examples $(DEB_BUILD)/debian; fi - if [ -e debian/upstream ]; then \ - cp -r debian/upstream $(DEB_BUILD)/debian; fi - if [ -e debian/tests ]; then \ - cp -r debian/tests $(DEB_BUILD)/debian; fi + cd $(TOPDIR)/ && \ + find debian -maxdepth 1 -type f -exec cp '{}' '$(BUILD_PREFIX)/$(DEB_BUILD)/{}' ';' + if [ -e $(TOPDIR)/debian/source ]; then \ + cp -r $(TOPDIR)/debian/source $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/local ]; then \ + cp -r $(TOPDIR)/debian/local $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/examples ]; then \ + cp -r $(TOPDIR)/debian/examples $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/upstream ]; then \ + cp -r $(TOPDIR)/debian/upstream $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/tests ]; then \ + cp -r $(TOPDIR)/debian/tests $(DEB_BUILD)/debian; fi rm -f $(DEB_BUILD)/debian/*.ex $(DEB_BUILD)/debian/*.EX rm -f $(DEB_BUILD)/debian/*.orig ifneq ($(GIT_INFO),) @@ -289,7 +247,7 @@ $(subst deb,%,$(DEBS)): $(DEB_BUILD).tar.$(SRC_EXT) \ cd $(DEB_BUILD); debuild -- clean git status rm -rf $(DEB_TOP)/$(NAME)-tmp - lfile1=$(shell echo $(DEB_TOP)/$(NAME)[0-9]*_$(DEB_VERS)-1_amd64.deb);\ + lfile1=$(shell echo $(DEB_TOP)/$(NAME)[0-9]*_$(VERSION)-1_amd64.deb);\ lfile=$$(ls $${lfile1}); \ lfile2=$${lfile##*/}; lname=$${lfile2%%_*}; \ dpkg-deb -R $${lfile} \ @@ -330,99 +288,58 @@ ls: $(TARGETS) # *_LOCAL_* repos are locally built packages. # *_GROUP_* repos are a local mirror of a group of upstream repos. # *_GROUP_* repos may not supply a repomd.xml.key. +ifeq ($(LOCAL_REPOS),true) ifneq ($(REPOSITORY_URL),) ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ -endif -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS += $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/ -endif -endif - ifeq ($(ID_LIKE),debian) -ifneq ($(DAOS_STACK_REPO_SUPPORT),) -TEST_STR := $(DAOS_STACK_REPO_UBUNTU_$(VERSION_ID_STR)_LIST) -ifneq ($(TEST_STR),) -UBUNTU_REPOS := $(shell curl $(DAOS_STACK_REPO_SUPPORT)$(TEST_STR)) -# Additional repos can be added but must be separated by a | character. -UBUNTU_ADD_REPOS = --othermirror "$(UBUNTU_REPOS)" -else -ifneq ($(DAOS_STACK_REPO_UBUNTU_ROLLING_LIST),) -UBUNTU_REPOS := $(shell curl $(DAOS_STACK_REPO_SUPPORT)$(DAOS_STACK_REPO_UBUNTU_ROLLING_LIST)) -# Additional repos can be added but must be separated by a | character. -UBUNTU_ADD_REPOS = --othermirror "$(UBUNTU_REPOS)" +# $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists +# of values with spaces as environment variables +$(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] endif +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ endif -# Need to figure out how to support multiple keys, such as for IPMCTL -ifneq ($(DAOS_STACK_REPO_PUB_KEY),) -HAVE_DAOS_STACK_KEY := TRUE - -$(DAOS_STACK_REPO_PUB_KEY): - curl -f -L -O '$(DAOS_STACK_REPO_SUPPORT)$(DAOS_STACK_REPO_PUB_KEY)' +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)| +ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) +$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/| endif endif - -chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(DAOS_STACK_REPO_PUB_KEY) - sudo pbuilder create \ - --extrapackages "gnupg ca-certificates" $(DISTRO_ID_OPT) -ifneq ($(HAVE_DAOS_STACK_KEY),) - printf "apt-key add - <> /etc/mock/$(CHROOT_NAME).cfg; \ - case $(DISTRO_ID) in \ - el7) distro="centos7"; \ - ;; \ - sle12.3) distro="sles12.3"; \ - ;; \ - sl42.3) distro="leap42.3"; \ - ;; \ - sl15.1) distro="leap15"; \ - ;; \ - esac; \ - for repo in $($(DISTRO_BASE)_PR_REPOS) $(PR_REPOS); do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $$repo = *@* ]]; then \ - branch="$${repo#*@}"; \ - repo="$${repo%@*}"; \ - if [[ $$branch = *:* ]]; then \ - build_number="$${branch#*:}"; \ - branch="$${branch%:*}"; \ - fi; \ - fi; \ - echo -e "[$$repo:$$branch:$$build_number]\n\ -name=$$repo:$$branch:$$build_number\n\ -baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/$$build_number/artifact/artifacts/$$distro/\n\ -enabled=1\n\ -gpgcheck=False\n" >> /etc/mock/$(CHROOT_NAME).cfg; \ - done; \ - for repo in $($(DISTRO_BASE)_LOCAL_REPOS) $($(DISTRO_BASE)_REPOS); do \ - repo_name=$${repo##*://}; \ - repo_name=$${repo_name//\//_}; \ - echo -e "[$$repo_name]\n\ -name=$${repo_name}\n\ -baseurl=$${repo}\n\ -enabled=1\n" >> /etc/mock/$(CHROOT_NAME).cfg; \ - done; \ - echo "\"\"\"" >> /etc/mock/$(CHROOT_NAME).cfg; \ - else \ - echo "Unable to update /etc/mock/$(CHROOT_NAME).cfg."; \ - echo "You need to make sure it has the needed repos in it yourself."; \ - fi - mock -r $(CHROOT_NAME) $(MOCK_OPTIONS) $(RPM_BUILD_OPTIONS) $< + $(call distro_map) \ + DISTRO="$$distro" \ + CHROOT_NAME="$(CHROOT_NAME)" \ + PR_REPOS="$(PR_REPOS)" \ + DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ + JENKINS_URL="$${JENKINS_URL}" \ + JOB_REPOS="$(JOB_REPOS)" \ + DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ + MOCK_OPTIONS="$(MOCK_OPTIONS)" \ + RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ + TARGET="$<" \ + packaging/rpm_chrootbuild endif docker_chrootbuild: - docker build --build-arg UID=$$(id -u) -t $(BUILD_OS)-chrootbuild \ - -f packaging/Dockerfile.$(BUILD_OS) . - docker run --privileged=true -w $$PWD -v=$$PWD:$$PWD \ - -it $(BUILD_OS)-chrootbuild bash -c "make chrootbuild" + $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ + -f packaging/Dockerfile.mockbuild . + $(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \ + -it chrootbuild bash -c "make -C $(CURDIR) \ + CHROOT_NAME=$(CHROOT_NAME) chrootbuild" rpmlint: $(SPEC) rpmlint $< @@ -443,6 +360,8 @@ packaging_check: --exclude \*.tar.\* \ --exclude \*.code-workspace \ --exclude install \ + --exclude packaging \ + --exclude utils \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi @@ -456,35 +375,55 @@ ifndef DEBFULLNAME endif test: - @echo "No test defined for this module" + # Test the rpmbuild by installing the built RPM + $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) + yum -y install $(TEST_PACKAGES) + +show_spec: + @echo '$(SPEC)' + +show_build_defines: + @echo '$(BUILD_DEFINES)' + +show_common_rpm_args: + @echo '$(COMMON_RPM_ARGS)' show_version: - @echo $(VERSION) + @echo '$(VERSION)' + +show_dl_version: + @echo '$(DL_VERSION)' show_release: - @echo $(RELEASE) + @echo '$(RELEASE)' show_rpms: - @echo $(RPMS) + @echo '$(RPMS)' show_source: - @echo $(SOURCE) + @echo '$(SOURCE)' + +show_patches: + @echo '$(PATCHES)' show_sources: - @echo $(SOURCES) + @echo '$(SOURCES)' + +show_other_sources: + @echo '$(OTHER_SOURCES)' show_targets: - @echo $(TARGETS) + @echo '$(TARGETS)' show_makefiles: - @echo $(MAKEFILE_LIST) + @echo '$(MAKEFILE_LIST)' show_calling_makefile: - @echo $(CALLING_MAKEFILE) + @echo '$(CALLING_MAKEFILE)' show_git_metadata: - @echo $(GIT_SHA1):$(GIT_SHORT):$(GIT_NUM_COMMITS) + @echo '$(GIT_SHA1):$(GIT_SHORT):$(GIT_NUM_COMMITS)' .PHONY: srpm rpms debs deb_detar ls chrootbuild rpmlint FORCE \ show_version show_release show_rpms show_source show_sources \ - show_targets check-env show_git_metadata + show_targets check-env show_git_metadata diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild new file mode 100755 index 0000000..e2b201d --- /dev/null +++ b/packaging/rpm_chrootbuild @@ -0,0 +1,47 @@ +#!/bin/bash + +set -uex + +# shellcheck disable=SC2153 +IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" + +if [ -w /etc/mock/"$CHROOT_NAME".cfg ]; then + # Remove the distro repos + if [[ $CHROOT_NAME =~ opensuse-leap-* ]]; then + ed /etc/mock/"$CHROOT_NAME".cfg << EOF +/^# repos$/+2;/^"""$/-1d +wq +EOF + fi + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/"$CHROOT_NAME".cfg + for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + echo -e "[$repo:$branch:$build_number]\n\ +name=$repo:$branch:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ +enabled=1\n\ +gpgcheck=False\n" >> /etc/mock/"$CHROOT_NAME".cfg + done + for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + echo -e "[${repo_name//@/_}]\n\ +name=${repo_name}\n\ +baseurl=${repo}\n\ +enabled=1\n" >> /etc/mock/"$CHROOT_NAME".cfg + done + echo "\"\"\"" >> /etc/mock/"$CHROOT_NAME".cfg +else + echo "Unable to update /etc/mock/$CHROOT_NAME.cfg." + echo "You need to make sure it has the needed repos in it yourself." +fi +eval mock -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"