From d28541b6dd81dfa21aabe650f7640063fc66e2b4 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 13 Mar 2025 20:12:40 +0000 Subject: [PATCH 1/9] DAOS-17287 build: Remove separate rpm build for raft Just include the submodule in the daos sources Signed-off-by: Jeff Olivier --- utils/rpms/Makefile | 8 +++----- utils/rpms/daos.spec | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/rpms/Makefile b/utils/rpms/Makefile index f5ab698e26e..3fe910b47c3 100644 --- a/utils/rpms/Makefile +++ b/utils/rpms/Makefile @@ -27,13 +27,11 @@ include packaging/Makefile_packaging.mk PACKAGING_CHECK_DIR ?= ../../../rpm/packaging -$(NAME)-$(VERSION).tar.gz: $(shell git ls-files :/:) +$(NAME)-$(VERSION).tar.gz: $(shell git ls-files --recurse-submodules :/:) echo Creating $@ echo $(basename $@) - cd ../../ && \ - git archive --format tar --prefix $(NAME)-$(VERSION)/ \ - -o $$OLDPWD/$(basename $@) HEAD ./ rm -f $@ - gzip $(basename $@) + git submodule update --init --recursive + cd ../../ && git ls-files --recurse-submodules | tar --transform='s/^/$(NAME)-$(VERSION)\//' --hard-dereference -hczf utils/rpms/$(NAME)-$(VERSION).tar.gz -T - tarball: $(NAME)-$(VERSION).tar.gz diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 156ce952e94..7201ccf35b1 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -71,7 +71,6 @@ BuildRequires: libisa-l_crypto-devel BuildRequires: libisal-devel BuildRequires: libisal_crypto-devel %endif -BuildRequires: daos-raft-devel = 0.11.0-1.416.g12dbc15%{?dist} BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel From df5f3cc5a71f10fd62d3ee70f2b6af8ea590cef4 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 13 Mar 2025 18:54:02 -0600 Subject: [PATCH 2/9] Oops, need to change the change logs Signed-off-by: Jeff Olivier --- debian/changelog | 6 ++++++ debian/control | 1 - utils/rpms/daos.spec | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 18a0bfd0294..d00e271bd0e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.7.101-6) unstable; urgency=medium + [ Jeff Olivier ] + * Remove raft as an external dependence + + -- Jeff Olivier Thu, 13 Mar 2025 19:31:00 +0000 + daos (2.7.101-5) unstable; urgency=medium [ Jan Michalski ] * Add ddb_ut and dtx_ut to the server-tests package diff --git a/debian/control b/debian/control index 28cb2d6830e..d3e961761a4 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,6 @@ Build-Depends: debhelper (>= 10), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, - libraft-dev (= 0.11.0-1416.g12dbc15), python3-tabulate, liblz4-dev, libaio-dev, diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 7201ccf35b1..7784ea0b549 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -16,7 +16,7 @@ Name: daos Version: 2.7.101 -Release: 5%{?relval}%{?dist} +Release: 6%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -575,7 +575,6 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %doc README.md # set daos_firmware_helper to be setuid root in order to perform privileged tasks %attr(4750,root,daos_server) %{_bindir}/daos_firmware_helper - %files serialize %doc README.md %{_libdir}/libdaos_serialize.so @@ -593,6 +592,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Thu Mar 13 2025 Jeff Olivier 2.7.101-6 +- Remove raft as external dependency + * Wed Jan 22 2025 Jan Michalski 2.7.101-5 - Add ddb_ut and dtx_ut to the server-tests package From 5c81c4ef7f62c181d24ef89b9f3954f98a12dfdb Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 13 Mar 2025 18:55:21 -0600 Subject: [PATCH 3/9] Accidental whitespace removal Signed-off-by: Jeff Olivier --- utils/rpms/daos.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 7784ea0b549..c955d761a4d 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -575,6 +575,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %doc README.md # set daos_firmware_helper to be setuid root in order to perform privileged tasks %attr(4750,root,daos_server) %{_bindir}/daos_firmware_helper + %files serialize %doc README.md %{_libdir}/libdaos_serialize.so From 85aa12c9d05bd0b3e784f8224a6c97a2416d0aaa Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 18:54:18 -0600 Subject: [PATCH 4/9] Add Li Wei's script Signed-off-by: Jeff Olivier --- utils/rpms/Makefile | 5 +++-- utils/rpms/archive.sh | 46 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100755 utils/rpms/archive.sh diff --git a/utils/rpms/Makefile b/utils/rpms/Makefile index 3fe910b47c3..c57eb434dc4 100644 --- a/utils/rpms/Makefile +++ b/utils/rpms/Makefile @@ -31,7 +31,8 @@ $(NAME)-$(VERSION).tar.gz: $(shell git ls-files --recurse-submodules :/:) echo Creating $@ echo $(basename $@) rm -f $@ - git submodule update --init --recursive - cd ../../ && git ls-files --recurse-submodules | tar --transform='s/^/$(NAME)-$(VERSION)\//' --hard-dereference -hczf utils/rpms/$(NAME)-$(VERSION).tar.gz -T - + ./archive.sh ../../ $(NAME) $(VERSION) tar + mv ../../$(NAME)-$(VERSION).tar . + gzip $(NAME)-$(VERSION).tar tarball: $(NAME)-$(VERSION).tar.gz diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh new file mode 100755 index 00000000000..eba8f90e757 --- /dev/null +++ b/utils/rpms/archive.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Usage: archive.sh +# +# Create an archive of the HEAD commit of a local Git repository, including +# submodules. +# +# - Must run at the root of the Git repository. +# - Must have all submodules initialized and updated. +# - Whitespaces are prohibited in any of the parameters. +# - Submodules within submodules are not supported. +# +# For example, +# +# archive.sh /a/b/c daos 1.2.3 tar +# +# produces /a/b/c/daos-1.2.3.tar. +# + +set -e + +dir=$1 +name=$2 +version=$3 +ext=$4 + +# Use a temporary directory for all intermediate files. +unset tmp +trap 'if [ -n "${tmp}" ]; then rm -rf "${tmp}"; fi' EXIT +tmp=$(mktemp -d) + +file_extless="${name}-${version}" +file="${file_extless}.${ext}" +sm_file_prefix="${file_extless}-submodule" + +# Create an archive, which doesn't include any submodule. +git archive --prefix "${name}-${version}/" -o "${tmp}/${file}" HEAD +tar -tvf "${tmp}/${file}" + +# Add all submodules to the archive. +git submodule --quiet foreach --recursive \ + "git archive --prefix ${name}-${version}/\${sm_path}/ -o ${tmp}/${sm_file_prefix}-\${name}.${ext} \${sha1}" +tar -Af "${tmp}/${file}" "${tmp}/${sm_file_prefix}-"*."${ext}" + +# Publish the archive. +mv ${tmp}/${file} ${dir}/${file} From 58631d83229e8c4be412d522f2d512bd2fd530d2 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 19:24:55 -0600 Subject: [PATCH 5/9] Add Li Wei's script Co-authored-by: Li Wei Signed-off-by: Jeff Olivier --- utils/rpms/Makefile | 3 +-- utils/rpms/archive.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/rpms/Makefile b/utils/rpms/Makefile index c57eb434dc4..3269a081cd2 100644 --- a/utils/rpms/Makefile +++ b/utils/rpms/Makefile @@ -31,8 +31,7 @@ $(NAME)-$(VERSION).tar.gz: $(shell git ls-files --recurse-submodules :/:) echo Creating $@ echo $(basename $@) rm -f $@ - ./archive.sh ../../ $(NAME) $(VERSION) tar - mv ../../$(NAME)-$(VERSION).tar . + cd ../../ && utils/rpms/archive.sh utils/rpms $(NAME) $(VERSION) tar gzip $(NAME)-$(VERSION).tar tarball: $(NAME)-$(VERSION).tar.gz diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh index eba8f90e757..405b5211ad4 100755 --- a/utils/rpms/archive.sh +++ b/utils/rpms/archive.sh @@ -38,8 +38,8 @@ git archive --prefix "${name}-${version}/" -o "${tmp}/${file}" HEAD tar -tvf "${tmp}/${file}" # Add all submodules to the archive. -git submodule --quiet foreach --recursive \ - "git archive --prefix ${name}-${version}/\${sm_path}/ -o ${tmp}/${sm_file_prefix}-\${name}.${ext} \${sha1}" +git submodule --quiet foreach \ + "git archive --prefix ${name}-${version}/\$sm_path/ -o ${tmp}/${sm_file_prefix}-\$name.${ext} \$sha1" tar -Af "${tmp}/${file}" "${tmp}/${sm_file_prefix}-"*."${ext}" # Publish the archive. From f155a815a93448a169b5da2af5d8b6fb144300cc Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 20:14:33 -0600 Subject: [PATCH 6/9] Remove debug Signed-off-by: Jeff Olivier --- utils/rpms/archive.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh index 405b5211ad4..e7e476389ad 100755 --- a/utils/rpms/archive.sh +++ b/utils/rpms/archive.sh @@ -35,7 +35,6 @@ sm_file_prefix="${file_extless}-submodule" # Create an archive, which doesn't include any submodule. git archive --prefix "${name}-${version}/" -o "${tmp}/${file}" HEAD -tar -tvf "${tmp}/${file}" # Add all submodules to the archive. git submodule --quiet foreach \ From 8a4e36532595596aa4983fee14c0a0c74e93a5ca Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 20:16:04 -0600 Subject: [PATCH 7/9] Shellcheck Signed-off-by: Jeff Olivier --- utils/rpms/archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh index e7e476389ad..2e49b195063 100755 --- a/utils/rpms/archive.sh +++ b/utils/rpms/archive.sh @@ -42,4 +42,4 @@ git submodule --quiet foreach \ tar -Af "${tmp}/${file}" "${tmp}/${sm_file_prefix}-"*."${ext}" # Publish the archive. -mv ${tmp}/${file} ${dir}/${file} +mv "${tmp}/${file}" "${dir}/${file}" From a84e7d34e3aff5f24a3b49e0307b566235ab27c7 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 21:12:45 -0600 Subject: [PATCH 8/9] Fix archive.sh for recursive submodules Signed-off-by: Jeff Olivier --- utils/rpms/archive.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh index 2e49b195063..ef7f497a1ef 100755 --- a/utils/rpms/archive.sh +++ b/utils/rpms/archive.sh @@ -37,9 +37,11 @@ sm_file_prefix="${file_extless}-submodule" git archive --prefix "${name}-${version}/" -o "${tmp}/${file}" HEAD # Add all submodules to the archive. -git submodule --quiet foreach \ - "git archive --prefix ${name}-${version}/\$sm_path/ -o ${tmp}/${sm_file_prefix}-\$name.${ext} \$sha1" -tar -Af "${tmp}/${file}" "${tmp}/${sm_file_prefix}-"*."${ext}" +# shellcheck disable=SC2086,SC2016 +git submodule --quiet foreach --recursive \ + 'tarfile='${tmp}/${sm_file_prefix}'-$name.'${ext}' && \ + git archive --prefix '${name}-${version}'/$displaypath/ -o ${tarfile} $sha1 && \ + tar -Af '"${tmp}/${file}"' ${tarfile}' # Publish the archive. mv "${tmp}/${file}" "${dir}/${file}" From 92911f02513d7799234879d611360375c57765d0 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 18 Mar 2025 21:13:46 -0600 Subject: [PATCH 9/9] Add recursive submodule support Signed-off-by: Jeff Olivier --- utils/rpms/archive.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/rpms/archive.sh b/utils/rpms/archive.sh index ef7f497a1ef..662c1bf6c94 100755 --- a/utils/rpms/archive.sh +++ b/utils/rpms/archive.sh @@ -6,9 +6,8 @@ # submodules. # # - Must run at the root of the Git repository. -# - Must have all submodules initialized and updated. +# - Must have all submodules recursively initialized and updated. # - Whitespaces are prohibited in any of the parameters. -# - Submodules within submodules are not supported. # # For example, #