libfuzzer.yaml

name: Libfuzzer
"on":
  schedule:
    # run daily 1:00 on main branch
    - cron: '0 1 * * *'
  push:
    branches:
      - main
      - prerelease_test
      - trigger/libfuzzer
  pull_request:
    paths: .github/workflows/libfuzzer.yaml

jobs:
  fuzz:
    strategy:
      fail-fast: false
      matrix:
        case: [ { algo: gorilla, type: float8 }, { algo: deltadelta, type: int8 } ]

    name: Fuzz decompression ${{ matrix.case.algo }} ${{ matrix.case.type }}
    runs-on: ubuntu-22.04
    env:
      PG_SRC_DIR: pgbuild
      PG_INSTALL_DIR: postgresql

    steps:
    - name: Install Linux Dependencies
      run: |
        # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable.
        # If needed, install them before opening the core dump.
        sudo apt-get update
        sudo apt-get install clang lld llvm flex bison lcov systemd-coredump gdb libipc-run-perl \
          libtest-most-perl tree

    - name: Checkout TimescaleDB
      uses: actions/checkout@v3

    - name: Read configuration
      id: config
      run: python -B .github/gh_config_reader.py

    # We are going to rebuild Postgres daily, so that it doesn't suddenly break
    # ages after the original problem.
    - name: Get date for build caching
      id: get-date
      run: |
        echo "date=$(date +"%d")" >> $GITHUB_OUTPUT

    # we cache the build directory instead of the install directory here
    # because extension installation will write files to install directory
    # leading to a tainted cache
    - name: Cache PostgreSQL
      id: cache-postgresql
      uses: actions/cache@v3
      with:
        path: ~/${{ env.PG_SRC_DIR }}
        key: "postgresql-libfuzzer-${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}"

    - name: Build PostgreSQL
      if: steps.cache-postgresql.outputs.cache-hit != 'true'
      run: |
       wget -q -O postgresql.tar.bz2 \
         https://ftp.postgresql.org/pub/source/v${{ steps.config.outputs.PG15_LATEST }}/postgresql-${{ steps.config.outputs.PG15_LATEST }}.tar.bz2
        mkdir -p ~/$PG_SRC_DIR
        tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1
        cd ~/$PG_SRC_DIR
        CC=clang ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \
          --without-readline --without-zlib --without-libxml --enable-cassert \
          --enable-debug CC=clang \
          CFLAGS="-DTS_COMPRESSION_FUZZING=1 -fuse-ld=lld -ggdb3 -Og -fno-omit-frame-pointer"
        make -j$(nproc)

    - name: Install PostgreSQL
      run: |
        make -C ~/$PG_SRC_DIR install
        make -C ~/$PG_SRC_DIR/contrib/postgres_fdw install

    - name: Upload config.log
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: config.log for PostgreSQL
        path: ~/${{ env.PG_SRC_DIR }}/config.log

    - name: Build TimescaleDB
      run: |
        set -e

        export LIBFUZZER_PATH=$(dirname "$(find $(llvm-config --libdir) -name libclang_rt.fuzzer_no_main-x86_64.a | head -1)")

        cmake -B build -S . -DASSERTIONS=ON -DLINTER=OFF -DCMAKE_VERBOSE_MAKEFILE=1 \
            -DWARNINGS_AS_ERRORS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang \
            -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link -lstdc++ -L$LIBFUZZER_PATH -l:libclang_rt.fuzzer_no_main-x86_64.a -static-libsan" \
            -DPG_PATH=$HOME/$PG_INSTALL_DIR

        make -C build -j$(nproc) install

    - name: initdb
      run: |
        # Have to do this before initializing the corpus, or initdb will complain.
        set -xeu

        export PGDATA=db
        export PGPORT=5432
        export PGDATABASE=postgres
        export PATH=$HOME/$PG_INSTALL_DIR/bin:$PATH
        initdb
        echo "shared_preload_libraries = 'timescaledb'" >> $PGDATA/postgresql.conf

    - name: Restore the cached fuzzing corpus
      id: restore-corpus-cache
      uses: actions/cache/restore@v3
      with:
        path: db/corpus
        # If the initial corpus changes, probably it was updated by hand with
        # some important examples, and it makes sense to start anew from it.
        key: "libfuzzer-corpus-2-${{ matrix.case.algo }}-${{ matrix.case.type }}-\
          ${{ hashFiles(format('tsl/test/fuzzing/compression/{0}-{1}', matrix.case.algo, matrix.case.type)) }}"

    - name: Initialize the fuzzing corpus
      # cache-hit is only true for exact key matches, and we use prefix matches.
      if: steps.restore-corpus-cache.outputs.cache-matched-key == ''
      run: |
        # Copy the intial corpus files from the repository. The github actions
        # cache doesn't follow symlinks.
        mkdir -p db/corpus
        find "tsl/test/fuzzing/compression/${{ matrix.case.algo }}-${{ matrix.case.type }}" -type f -exec cp -t db/corpus {} +

    - name: Run libfuzzer for compression
      run: |
        set -xeu

        export PGDATA=db
        export PGPORT=5432
        export PGDATABASE=postgres
        export PATH=$HOME/$PG_INSTALL_DIR/bin:$PATH
        pg_ctl -l postmaster.log start

        psql -c "create extension timescaledb;"

        # Create the fuzzing function
        export MODULE_NAME=$(basename $(find $HOME/$PG_INSTALL_DIR -name "timescaledb-tsl-*.so"))
        psql -a -c "create or replace function fuzz(algo cstring, type regtype, runs int) returns int as '"$MODULE_NAME"', 'ts_fuzz_compression' language c;"

        # Start more fuzzing processes in the background. We won't even monitor
        # their progress, because the server will panic if they find an error.
        for x in {2..$(nproc)}
        do
          psql -v ON_ERROR_STOP=1 -c "select fuzz('${{ matrix.case.algo }}', '${{ matrix.case.type }}', 100000000);" &
        done

        # Start the one fuzzing process that we will monitor, in foreground.
        # The LLVM fuzzing driver calls exit(), so we expect to lose the connection.
        ret=0
        psql -v ON_ERROR_STOP=1 -c "select fuzz('${{ matrix.case.algo }}', '${{ matrix.case.type }}', 100000000);" || ret=$?
        if ! [ $ret -eq 2 ]
        then
            >&2 echo "Unexpected psql exit code $ret"
            exit 1
        fi

        # Check that the server is still alive.
        psql -c "select 1"

    - name: Collect the logs
      if: always()
      id: collectlogs
      run: |
        find . -name postmaster.log -exec cat {} + > postgres.log
        # wait in case there are in-progress coredumps
        sleep 10
        if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi
        # print OOM killer information
        sudo journalctl --system -q --facility=kern --grep "Killed process" || true

    - name: Save PostgreSQL log
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: PostgreSQL log for ${{ matrix.case.algo }} ${{ matrix.case.type }}
        path: postgres.log

    - name: Save fuzzer-generated crash cases
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: Crash cases for ${{ matrix.case.algo }} ${{ matrix.case.type }}
        path: db/crash-*

    # We use separate restore/save actions, because the default action won't
    # save the updated folder after the cache hit. We also can't overwrite the
    # existing cache, so we add a unique suffix. The cache is matched by key
    # prefix, not exact key, and picks the newest matching item, so this works.
    - name: Save fuzzer corpus
      uses: actions/cache/save@v3
      with:
        path: db/corpus
        key: "${{ format('{0}-{1}-{2}',
            steps.restore-corpus-cache.outputs.cache-primary-key,
            github.run_id, github.run_attempt) }}"

    - name: Stack trace
      if: always() && steps.collectlogs.outputs.coredumps == 'true'
      run: |
        sudo coredumpctl gdb <<<"
          set verbose on
          set trace-commands on
          show debug-file-directory
          printf "'"'"query = '%s'\n\n"'"'", debug_query_string
          frame function ExceptionalCondition
          printf "'"'"condition = '%s'\n"'"'", conditionName
          up 1
          l
          info args
          info locals
          bt full
        " 2>&1 | tee stacktrace.log
        ./scripts/bundle_coredumps.sh
        grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||:
        exit 1 # Fail the job if we have core dumps.

    - name: Upload core dumps
      if: always() && steps.collectlogs.outputs.coredumps == 'true'
      uses: actions/upload-artifact@v3
      with:
        name: Coredumps for ${{ matrix.case.algo }} ${{ matrix.case.type }}
        path: coredumps