diff --git a/.github/run_tests.sh b/.github/run_tests.sh new file mode 100755 index 0000000000..cfb36457a9 --- /dev/null +++ b/.github/run_tests.sh @@ -0,0 +1,240 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "$GITHUB_STEP_SUMMARY" ]]; then + echo "run_tests.sh is for use by CI (selected tests, timeout)." + echo "Users should go to opt/cachelib/tests and run make." + echo +fi + +# Optional (e.g., flaky) tests. Issues a warning instead of an error. +OPTIONAL=() +OPTIONAL+=("allocator-test-AllocationClassTest") # Ubuntu 18 (segfault) +OPTIONAL+=("allocator-test-AllocatorResizeTypeTest") # Rocky 8.6 +OPTIONAL+=("allocator-test-AllocatorTypeTest") # CentOS 8.5, Rocky 8.6 +OPTIONAL+=("allocator-test-BlockCacheTest") # Rocky 8.6 +OPTIONAL+=("allocator-test-MemoryAllocatorTest") # Ubuntu 18 (segfault) +OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 +OPTIONAL+=("allocator-test-NavySetupTest") +OPTIONAL+=("allocator-test-NvmCacheTests") # Rocky 8.6 +OPTIONAL+=("allocator-test-RefCountTest") # Ubuntu 18 +OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18 +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 +OPTIONAL+=("common-test-UtilTests") +OPTIONAL+=("datatype-test-MapTest") # Ubuntu 20 +# CentOS 8.1, Rocky 9, Ubuntu 18 +OPTIONAL+=("navy-test-BlockCacheTest") +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Ubuntu 18, Ubuntu 20, Ubuntu 22 +OPTIONAL+=("navy-test-DriverTest") +# CentOS 8.5, Rocky 9, Ubuntu 20 +OPTIONAL+=("navy-test-MockJobSchedulerTest") +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18, Ubuntu 20, Ubuntu 22 +# Large pages need to be enabled +OPTIONAL+=("shm-test-test_page_size") + +# Skip long-running benchmarks. +TO_SKIP=() +# TO_SKIP+=("allocator-test-AllocatorTypeTest") # 12 mins. +TO_SKIP+=("benchmark-test-CompactCacheBench") # 26 mins. +TO_SKIP+=("benchmark-test-MutexBench") # 60 mins. + +TEST_TIMEOUT=30m +BENCHMARK_TIMEOUT=20m +PARALLELISM=10 + +print_test_log() { + logfile=$1 + # Print last failed test + EXP_LOG=$(grep -Pazo \ + "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ + | sed 's/\x0/---------------\n/g') + # And contents of last test before core dumps + if grep -q -R "core dumped" $logfile; then + EXP_LOG+=$'\n' + EXP_LOG+=$(tac $logfile | sed '/\[ RUN \]/q' | tac) + fi + echo "::group::Logs:$logfile" + echo "$EXP_LOG" + echo + echo "::endgroup::" + echo + + echo "#### $logfile" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + echo "$EXP_LOG" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + echo >> $MD_OUT +} + +OPTIONAL_LIST=$(printf -- '%s\n' ${OPTIONAL[@]}) +TO_SKIP_LIST=$(printf -- '%s\n' ${TO_SKIP[@]}) + +MD_OUT=${GITHUB_STEP_SUMMARY:-$PWD/summary.md} +if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then + echo "Markdown summary will be saved in $MD_OUT. Truncating it." + echo + echo "Time started: $(date)" > $MD_OUT +fi + +echo "See Summary page of job for a table of test results and log excerpts." +echo + +dir=$(dirname "$0") +cd "$dir/.." || die "failed to change-dir into $dir/.." +test -d cachelib || die "failed to change-dir to expected root directory" + +PREFIX="$PWD/opt/cachelib" +LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH + +echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + +cd opt/cachelib/tests || die "failed to change-dir into opt/cachelib/tests" + +TESTS_TO_RUN=$(find * -type f -not -name "*bench*" -executable \ + | grep -vF "$TO_SKIP_LIST" \ + | awk ' { print $1 ".log" } ') +N_TESTS=$(echo $TESTS_TO_RUN | wc -w) + +if [[ $(< /proc/sys/vm/nr_hugepages) == "0" ]]; then + # GitHub's runners have 7GB of RAM (as of 2023) + echo + echo "Trying to allocate a 1GB huge page pool for shm-test-test_page_size" + sudo sysctl -w vm.nr_hugepages=512 +fi + +echo +echo "::group::Running tests for CI (total: $N_TESTS, max: $TEST_TIMEOUT)" +timeout --preserve-status $TEST_TIMEOUT make -j $PARALLELISM -s $TESTS_TO_RUN +echo "::endgroup::" +echo "Successful tests: $(find -name '*.ok' | wc -l)" +echo "Failed tests: $(find -name '*.fail' | wc -l)" +echo + +BENCHMARKS_TO_RUN=$(find * -type f -name "*bench*" -executable \ + | grep -vF "$TO_SKIP_LIST" \ + | awk ' { print $1 ".log" } ') +N_BENCHMARKS=$(echo $BENCHMARKS_TO_RUN | wc -w) + +echo "::group::Running benchmarks for CI (total: $N_BENCHMARKS, max: $BENCHMARK_TIMEOUT)" +timeout --preserve-status $BENCHMARK_TIMEOUT make -j $PARALLELISM -s $BENCHMARKS_TO_RUN +echo "::endgroup::" +echo "Successful benchmarks: $(find -name '*bench*.ok' | wc -l)" +echo "Failed benchmarks: $(find -name '*bench*.fail' | wc -l)" + +TESTS_PASSED=$(find * -name '*.log.ok' | sed 's/\.log\.ok$//') +TESTS_FAILED=$(find * -name '*.log.fail' | sed 's/\.log\.fail$//') +TESTS_TIMEOUT=$(find * -type f -executable \ + | grep -vF "$TESTS_PASSED" \ + | grep -vF "$TESTS_FAILED" \ + | grep -vF "$TO_SKIP_LIST") +TESTS_IGNORED=$(echo "$TESTS_FAILED" | grep -F "$OPTIONAL_LIST") +FAILURES_UNIGNORED=$(echo "$TESTS_FAILED" | grep -vF "$OPTIONAL_LIST") + +N_TIMEOUT=$(echo $TESTS_TIMEOUT | wc -w) +N_PASSED=$(echo $TESTS_PASSED | wc -w) +N_FAILED=$(echo $TESTS_FAILED | wc -w) +N_IGNORED=$(echo $TESTS_IGNORED | wc -w) +N_FAILURES_UNIGNORED=$(echo $FAILURES_UNIGNORED | wc -w) +N_SKIPPED=$(echo $TO_SKIP_LIST | wc -w) + +echo "## Test summary" >> $MD_OUT +echo "|Workflow| Passed | Failed | Ignored | Timeout | Skipped" >> $MD_OUT +echo "|--------|--------|--------|---------|---------|---------|" >> $MD_OUT +echo "| $GITHUB_JOB | $N_PASSED | $N_FAILED | $N_IGNORED | $N_TIMEOUT | $N_SKIPPED |" >> $MD_OUT + +STATUS=0 + +if [[ $N_FAILED -ne 0 ]]; then + + echo + echo "::group::Failures at a glance" + grep "core dumped" *.log || true + grep "FAILED.*ms" *.log || true + echo "::endgroup::" + + echo >> $MD_OUT + echo "## Failures at a glance" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + grep "core dumped" *.log >> $MD_OUT || true + grep "FAILED.*ms" *.log >> $MD_OUT || true + echo "\`\`\`" >> $MD_OUT + + if [ $N_FAILURES_UNIGNORED -eq 0 ]; then + echo "Only ignored tests failed." + else + STATUS=1 + echo + echo "== Failing tests ==" + echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed." + + echo "$FAILURES_UNIGNORED" + + echo >> $MD_OUT + echo "## Failing tests" >> $MD_OUT + echo "$FAILURES_UNIGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT + + + for failedtest in $FAILURES_UNIGNORED; do + echo "::error ::$failedtest failed. See job summary or log for details." + print_test_log "$failedtest.log" + done + fi + + if [[ $N_IGNORED -ne 0 ]]; then + echo + echo "::group::Ignored test failures " + echo "$TESTS_IGNORED" + echo "::endgroup::" + echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored." + + echo >> $MD_OUT + echo "## Ignored test failures" >> $MD_OUT + echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT + + for failedtest in $TESTS_IGNORED; do + echo "::warning ::$failedtest failed & ignored. See job summary or log for details." + print_test_log "$failedtest.log" + done + fi +else + echo + echo "All tests passed." +fi + +echo +echo "::group::Skipped tests" +echo "$TO_SKIP_LIST" +echo "::endgroup::" + +echo >> $MD_OUT +echo "## Skipped tests" >> $MD_OUT +echo "$TO_SKIP_LIST" | awk ' { print "1. " $1 } ' >> $MD_OUT + +if [[ $N_TIMEOUT -ne 0 ]]; then + echo + echo "::error ::$N_TIMEOUT tests exceeded time limit." \ + " Consider adding them to TO_SKIP or increasing TEST_TIMEOUT/BENCHMARK_TIMEOUT." + echo "::group::Timed out tests" + echo "$TESTS_TIMEOUT" + echo "::endgroup::" + + echo "## Tests timed out" >> $MD_OUT + echo "$TESTS_TIMEOUT" | awk ' { print "1. " $1 } ' >> $MD_OUT +fi + +# Comment out if you do not want (unignored) failing tests to fail the build +exit $STATUS diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index 3983e0c78b..6295194b3d 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -20,7 +20,7 @@ on: jobs: build-cachelib-centos8-1-1911: #if: "!contains(github.event.head_commit.author.name, 'svcscm')" - name: "CentOS/8.1.1911 - Build CacheLib with all dependencies" + name: "CentOS/8.1.1911 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "centos:8.1.1911" @@ -73,11 +73,21 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -92,16 +102,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -112,3 +128,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 4e6c2d12e1..ab848c28b7 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -19,7 +19,7 @@ on: - cron: '0 9 * * *' jobs: build-cachelib-centos8-latest: - name: "CentOS/8.5 - Build CacheLib with all dependencies" + name: "CentOS/8.5 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "centos:latest" @@ -72,11 +72,21 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -91,16 +101,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -111,3 +127,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 7f0ab29a6c..e001deb096 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -19,7 +19,7 @@ on: - cron: '0 13 * * *' jobs: build-cachelib-debian-10: - name: "Debian-10 (Buster) - Build CacheLib with all dependencies" + name: "Debian-10 (Buster) Build & Test" runs-on: ubuntu-latest # Docker container image name container: "debian:buster-slim" @@ -51,7 +51,7 @@ jobs: echo === g++ -v === g++ - || true - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Add Git safe directory" # Workaround for Docker image bug (GitHub issue #199). run: git config --system --add safe.directory $GITHUB_WORKSPACE @@ -59,6 +59,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -73,16 +83,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -93,3 +109,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index f8c0424400..f616217f91 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -19,7 +19,7 @@ on: - cron: '0 19 * * *' jobs: build-cachelib-fedora-36: - name: "Fedora 36 - Build CacheLib with all dependencies" + name: "Fedora 36 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "fedora:36" @@ -54,13 +54,23 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "git: ignore unsafe-directories" run: git config --global --add safe.directory '*' - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -75,16 +85,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -95,3 +111,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index c8af12327d..8c01c8e591 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -19,7 +19,7 @@ on: - cron: '0 15 * * 2,4,6' jobs: build-cachelib-centos8-latest: - name: "RockyLinux-8.6 - Build CacheLib with all dependencies" + name: "RockyLinux-8.6 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "rockylinux:8.6" @@ -56,11 +56,21 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -75,16 +85,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -95,3 +111,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index e26eac6ff1..5707be387b 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -19,7 +19,7 @@ on: - cron: '0 17 * * *' jobs: build-cachelib-rockylinux-9: - name: "RockyLinux-9.0 - Build CacheLib with all dependencies" + name: "RockyLinux-9.0 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "rockylinux:9.0" @@ -54,11 +54,21 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -76,16 +86,22 @@ jobs: # see: https://aur.archlinux.org/packages/folly#comment-862543 sed -i '$aset(LIBDWARF_INCLUDE_DIRS "/usr/include/libdwarf-0")' cachelib/external/folly/CMake/FindLibDwarf.cmake ./contrib/build-package.sh -S -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -96,3 +112,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index ad068278a4..05263a800b 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -24,7 +24,7 @@ on: - cron: '0 5 * * 2,4,6' jobs: build-cachelib-ubuntu-18: - name: "Ubuntu-18 - Build CacheLib with all dependencies" + name: "Ubuntu-18 Build & Test" runs-on: ubuntu-latest container: "ubuntu:18.04" steps: @@ -72,11 +72,22 @@ jobs: - name: "git: ignore unsafe-directories" run: git config --global --add safe.directory '*' - name: "checkout sources" + # v1 needed for Ubuntu 18 due to old Git version uses: actions/checkout@v1 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -91,16 +102,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -111,3 +128,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index a8380fdb96..593070c2df 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -20,7 +20,7 @@ on: - cron: '0 5 * * 1,3,5' jobs: build-cachelib-ubuntu-20: - name: "Ubuntu-20 - Build CacheLib with all dependencies" + name: "Ubuntu-20 Build & Test" runs-on: ubuntu-20.04 steps: - name: "Show Head Commit" @@ -52,11 +52,21 @@ jobs: echo === g++ -v === g++ -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -71,16 +81,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -91,3 +107,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index 4db194431d..d7cf74c1db 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -20,7 +20,7 @@ on: - cron: '0 7 * * *' jobs: build-cachelib-ubuntu-22: - name: "Ubuntu-22 - Build CacheLib with all dependencies" + name: "Ubuntu-22 Build & Test" runs-on: ubuntu-22.04 steps: - name: "Show Head Commit" @@ -52,11 +52,21 @@ jobs: echo === g++ -v === g++ -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -71,16 +81,22 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - name: "Run tests" + run: ./.github/run_tests.sh + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs @@ -91,3 +107,10 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 4b4897b610..26d942d182 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -11,7 +11,7 @@ jobs: - 'cachelib' - 'examples' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run clang-format style check for C/C++ programs. uses: jidicula/clang-format-action@v4.6.2 with: diff --git a/cachelib/cmake/Makefile.tests.in b/cachelib/cmake/Makefile.tests.in index 4be43a2890..b0320d380c 100644 --- a/cachelib/cmake/Makefile.tests.in +++ b/cachelib/cmake/Makefile.tests.in @@ -20,6 +20,8 @@ MARKERS_FAIL=$(LOGS:%=%.fail) OUTPUT=cachelib-test-summary.log .PHONY: all clean +# Avoid deleting test logs if interrupted (e.g., timeout) +.PRECIOUS: ($LOGS) $(OUTPUT) all: $(OUTPUT) @@ -86,6 +88,7 @@ $(LOGS): %.log: % else \ touch "$@.fail" ; \ fi + @echo Completed $^ clean: @rm -f $(LOGS) $(MARKERS_OK) $(MARKERS_FAIL) $(OUTPUT)