From 72ff656609822cef698cba4d8a1cf6687e4da8e9 Mon Sep 17 00:00:00 2001 From: Andrew Quijano Date: Mon, 1 Jul 2024 15:22:35 -0400 Subject: [PATCH 1/4] Delete travis and publish_deb --- .github/workflows/publish_deb.yml | 35 -------- .travis.yml | 133 ------------------------------ 2 files changed, 168 deletions(-) delete mode 100644 .github/workflows/publish_deb.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/publish_deb.yml b/.github/workflows/publish_deb.yml deleted file mode 100644 index 0c70aadf1d6..00000000000 --- a/.github/workflows/publish_deb.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Publish deb package - -on: - push: - tags: v* - -jobs: - build_deb: - runs-on: panda-arc - strategy: - matrix: - ubuntu_version: - - 20.04 - - 22.04 - - steps: - - name: Install git - run: sudo apt-get -qq update -y && sudo apt-get -qq install git -y - - - name: Check out - uses: actions/checkout@v4 - - - name: Build package - working-directory: panda/debian - run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }} - - - name: Upload wheel and debian packages to release - uses: softprops/action-gh-release@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - files: | - panda/debian/pandare-*.whl - panda/debian/pandare_*.deb diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 102858fde6f..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,133 +0,0 @@ -dist: bionic -language: c -compiler: gcc-8 -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-8 -env: - - CC=gcc-8 - - AR=gcc-ar-8 - - -cache: - # There is one cache per branch and compiler version. - timeout: 1200 - ccache: true - pip: true - directories: - - $HOME/avocado/data/cache - - -addons: - apt: - sources: - # up to ubuntu xenial, we can use the ppa name - - sourceline: 'ppa:phulin/panda' - # the ppa doesn't support later ubuntu releases -- specify url instead - #- sourceline: 'deb http://ppa.launchpad.net/phulin/panda/ubuntu xenial main' - packages: - # Build dependencies - - chrpath - - libaio-dev - - libattr1-dev - - libbrlapi-dev - - libcap-dev - - libcap-ng-dev - - libgcc-4.8-dev - - libgnutls-dev - - libgtk-3-dev - - libiscsi-dev - - liblttng-ust-dev - - libncurses5-dev - - libnfs-dev - - libnss3-dev - - libpixman-1-dev - - libpng12-dev - - librados-dev - - libsdl1.2-dev - - libseccomp-dev - - libspice-protocol-dev - - libspice-server-dev - - libssh-dev - - liburcu-dev - - libusb-1.0-0-dev - - sparse - - uuid-dev - # PANDA stuff - - bison - - flex - - linux-libc-dev - - libcapstone-dev - - libdwarf-dev - - libelf-dev - - libprotobuf-c0-dev - - libprotoc-dev - - libwireshark-dev - - libwiretap-dev - - llvm-3.3-dev - - clang-3.3 - - protobuf-c-compiler - - protobuf-compiler - - python-pip - - python-protobuf - - python-pycparser - # PYPANDA stuff - - python3 - - python3-pip - - python3-venv - - python3-setuptools - - genisoimage # To make isos for run_guest - - wget - # To build 32-bit targets for testing - - libc6-dev-i386 - - gcc-multilib - - nasm - -# PANDA -sudo: required -dist: xenial -env: - global: - - PANDA_TEST="no" # These tests aren't too useful and timeout travis. They passed locally as of 2ce657c -git: - # we want to do this ourselves for some reason - submodules: false - -before_install: - - sudo pip install pycparser cffi colorama protobuf # pycparser is a core dependency, rest are for pypanda - - git submodule update --init dtc - - cd $(pyenv root) && git checkout master && git pull; cd - # Update pyenv - - echo 'core_%e.%p' | sudo tee /proc/sys/kernel/core_pattern # Store coredumps locally -before_script: - - ulimit -c unlimited -S # enable core dumps - # Setup python environment with pyenv (python2 and python3) - - pyenv install 2.7.9 || travis_terminate 1; # For core qemu - - pyenv install 3.6.6 || travis_terminate 1; # For pypanda - - pyenv shell 2.7.9 3.6.6 || travis_terminate 1; # Create a shell with both python2 and python3 - - pip3 install cffi colorama protobuf # PYPANDA dependencies - not auto installed by setup because we don't use pip to install pypanda - -script: - # Run install script - - sudo panda/scripts/install_ubuntu.sh || travis_terminate 1; # Build panda - # Install pypanda - - cd panda/python/core - - python3 setup.py install || travis_terminate 1 - # Run pypanda tests - # - cd ../tests - #- make || travis_terminate 1 - #- pip3 install -r requirements.txt || travis_terminate 1 # Install test python dependencies (capstone, pyelftools) - # - python3 multi_proc_cbs.py - # - python3 taint_reg.py - # - python3 taint_ram.py - # # Run record_then_replay on multiple architectures - # - python3 record_then_replay.py i386 - # - python3 record_then_replay.py x86_64 - # - python3 record_then_replay.py arm - # - python3 record_then_replay.py ppc - # # Test hooking framework - # - python3 hooking.py - # # Regression tests - # - python3 sleep_in_cb.py From 0aae9ffaa69906b3ab5722deadac7e0ed7ad9725 Mon Sep 17 00:00:00 2001 From: Andrew Quijano Date: Mon, 1 Jul 2024 15:33:56 -0400 Subject: [PATCH 2/4] Update install_ubuntu.sh to download libosi and capstonev5, remove dead variables from setup.py in pypanda, and optimize setup.sh to only use one panda container --- .github/workflows/parallel_tests.yml | 4 ++++ panda/debian/setup.sh | 17 +++++++++----- panda/python/core/setup.py | 3 --- panda/scripts/install_ubuntu.sh | 34 ++++++++++++---------------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/parallel_tests.yml b/.github/workflows/parallel_tests.yml index 55594fe78d7..805415a6470 100644 --- a/.github/workflows/parallel_tests.yml +++ b/.github/workflows/parallel_tests.yml @@ -3,6 +3,10 @@ name: Parallel Tests # then run CI tests using that container in parallel # For forked repos that can't use our panda-arc test suite, just build and run make check on: + # Allow to repo owner to manually run this workflow for external PRs once code is vetted + workflow_dispatch: + + # Run automatically for internal PRs and pushes pull_request: branches: - dev diff --git a/panda/debian/setup.sh b/panda/debian/setup.sh index d1327b8b3e1..0a1312f7cf7 100755 --- a/panda/debian/setup.sh +++ b/panda/debian/setup.sh @@ -42,11 +42,17 @@ if [[ ! -f "../dependencies/ubuntu_${version}_base.txt" ]]; then exit 1 fi -# First build main panda container for the target ubuntu version -DOCKER_BUILDKIT=1 docker build --target panda -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. -# Also build the installer, since that's where the whl file is built -DOCKER_BUILDKIT=1 docker build --target installer -t panda_installer --build-arg BASE_IMAGE="ubuntu:${version}" ../.. + +# Build the installer to generate the wheel file +DOCKER_BUILDKIT=1 docker build --target installer -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. + +# Copy wheel file out of container to host +# this also preserves wheel name, which is important as pip install WILL fail if you arbitarily change the generated wheel file name +docker run --rm -v $(pwd):/out panda bash -c "cp /panda/panda/python/core/dist/*.whl /out" + +# Finish building main panda container for the target ubuntu version +DOCKER_BUILDKIT=1 docker build --target panda -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. # Now build the packager container from that docker build -t packager . @@ -55,5 +61,4 @@ docker build -t packager . docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out" mv pandare.deb pandare_${version}.deb -# Copy whl file out of container to host, this also preserves wheel name, which is important as pip install WILL fail if you arbitarily change the generated wheel file name -docker run --rm -v $(pwd):/out panda_installer bash -c "cp /panda/panda/python/core/dist/*.whl /out" + diff --git a/panda/python/core/setup.py b/panda/python/core/setup.py index 5179019f773..dfa3d62d71e 100644 --- a/panda/python/core/setup.py +++ b/panda/python/core/setup.py @@ -17,8 +17,6 @@ root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."]) # panda-git/ root dir -pypi_build = False # Set to true if trying to minimize size for pypi package upload. Note this disables some architectures - lib_dir = os.path.join("pandare", "data") @@ -27,7 +25,6 @@ def copy_objs(): Run to copy objects into a (local and temporary) python module before installing to the system. Shouldn't be run if you're just installing in develop mode ''' - build_root = os.path.join(root_dir, "build") if os.path.isdir(lib_dir): shutil.rmtree(lib_dir) diff --git a/panda/scripts/install_ubuntu.sh b/panda/scripts/install_ubuntu.sh index 164051ed8fc..73b853e7281 100755 --- a/panda/scripts/install_ubuntu.sh +++ b/panda/scripts/install_ubuntu.sh @@ -25,13 +25,14 @@ lsb_release --help &>/dev/null || $SUDO apt-get update -qq && $SUDO apt-get -qq git --help &>/dev/null || $SUDO apt-get -qq update && $SUDO apt-get -qq install -y --no-install-recommends git # some globals +LIBOSI_VERSION="0.1.7" +UBUNTU_VERSION=$(lsb_release -r | awk '{print $2}') PANDA_GIT="https://github.com/panda-re/panda.git" -LIBDWARF_GIT="git://git.code.sf.net/p/libdwarf/code" # system information #vendor=$(lsb_release --id | awk -F':[\t ]+' '{print $2}') #codename=$(lsb_release --codename | awk -F':[\t ]+' '{print $2}') -version=$(lsb_release -r| awk '{print $2}' | awk -F'.' '{print $1}') +version=$(lsb_release -r | awk '{print $2}' | awk -F'.' '{print $1}') progress() { echo @@ -105,31 +106,24 @@ if [ "$version" -eq 18 ]; then rm z3-4.8.7-x64-ubuntu-16.04.zip fi -# Because libcapstone for Ubuntu 18 or 20 is really old, we download and install the v4.0.2 release if it's not present -if [[ !$(ldconfig -p | grep -q libcapstone.so.4) ]]; then - echo "Installing libcapstone v4" +# Install libcapstone v5 release if it's not present +if [[ !$(ldconfig -p | grep -q libcapstone.so.5) ]]; then + echo "Installing libcapstone v5" pushd /tmp && \ - curl -o /tmp/cap.tgz -L https://github.com/aquynh/capstone/archive/4.0.2.tar.gz && \ - tar xvf cap.tgz && cd capstone-4.0.2/ && MAKE_JOBS=$(nproc) ./make.sh && $SUDO make install && cd /tmp && \ - rm -rf /tmp/capstone-4.0.2 + git clone https://github.com/capstone-engine/capstone/ -b v5 && \ + cd capstone/ && MAKE_JOBS=$(nproc) ./make.sh && $SUDO make install && cd /tmp && \ + rm -rf /tmp/capstone $SUDO ldconfig popd fi # if the windows introspection library is not installed, clone and install if [[ !$(dpkg -l | grep -q libosi) ]]; then - libosi_name=libosi-$(date +"%Y%m%d") - libosi_branch=master - libosi_repo=https://github.com/panda-re/libosi - - echo "Installing libosi" - pushd . - git clone -b $libosi_branch $libosi_repo $libosi_name && cd $_ - mkdir build && cd $_ - cmake -GNinja .. && \ - ninja && ninja package && \ - $SUDO dpkg -i libosi*.deb - popd && rm -rf $libosi_name + pushd /tmp + curl -LJO https://github.com/panda-re/libosi/releases/download/v${LIBOSI_VERSION}/libosi_${UBUNTU_VERSION}.deb + $SUDO dpkg -i /tmp/libosi_${UBUNTU_VERSION}.deb + rm -rf /tmp/libosi_${UBUNTU_VERSION}.deb + popd fi # PyPANDA needs CFFI from pip (the version in apt is too old) From d9c5f340edd14d7884399134c4fd88b746c7be8c Mon Sep 17 00:00:00 2001 From: Andrew Quijano Date: Mon, 1 Jul 2024 15:40:09 -0400 Subject: [PATCH 3/4] Update documentation pointing out new debian packages and PyPanda on Pip is now in sync with GitHub releases --- README.md | 8 +++++--- panda/debian/setup.sh | 4 ---- panda/docs/build_ubuntu.md | 7 ++----- panda/python/core/README.md | 7 ------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 437fbca7e41..bfa0caf93e6 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,11 @@ $ docker run --rm pandadev panda-system-i386 --help ``` ### Quickstart: Python pip -The Python interface to PANDA (also known as *pypanda*) can be installed from [PIP](https://pypi.org/project/pandare/) by running `pip3 install pandare`. This will install everything you need for python-based PANDA analyses, but not stand-alone PANDA binaries. This package is not automatically updated so it may fall behind the master branch of PANDA. The distributed binaries are only tested on 64-bit Ubuntu 18.04 and other architectures/versions are unlikely to work. You can also install pypanda by building PANDA and then running `python3 setup.py install` from the directory `panda/panda/python/core`. +The Python interface to PANDA (also known as *pypanda*) can be installed from [PIP](https://pypi.org/project/pandare/) by running `pip3 install pandare`. This will install everything you need for python-based PANDA analyses, but not stand-alone PANDA binaries. The distributed binaries are only tested on 64-bit Ubuntu 18.04 and other architectures/versions are unlikely to work. You can also install pypanda by building PANDA and then running `python3 setup.py install` from the directory `panda/panda/python/core`. ### Debian, Ubuntu +The fastest way to install PANDA would be through installing [the debian packages](https://github.com/pandare/panda/releases). +There is a debian package for both Ubuntu 20.04 and Ubuntu 22.04, and its corresponding PyPanda package. Because PANDA has a few dependencies, we've encoded the build instructions into the [install\_ubuntu.sh](panda/scripts/install\_ubuntu.sh). The script should work on the latest Debian stable/Ubuntu LTS versions. @@ -73,8 +75,8 @@ commands into whatever package manager your distribution uses. Note that if you want to use our LLVM features (mainly the dynamic taint system), you will need to install LLVM 11 from OS packages or compiled from source. On Ubuntu this should happen automatically via `install_ubuntu.sh`. -Additionally, it is **strongly** recommended that you only build PANDA as 64bit -binary. Creating a 32bit build should be possible, but best avoided. +Additionally, it is **strongly** recommended that you only build PANDA as 64-bit binary. +Creating a 32-bit build should be possible, but best avoided. See the limitations section for details. ### Arch Linux diff --git a/panda/debian/setup.sh b/panda/debian/setup.sh index 0a1312f7cf7..b28dc85c7e2 100755 --- a/panda/debian/setup.sh +++ b/panda/debian/setup.sh @@ -42,8 +42,6 @@ if [[ ! -f "../dependencies/ubuntu_${version}_base.txt" ]]; then exit 1 fi - - # Build the installer to generate the wheel file DOCKER_BUILDKIT=1 docker build --target installer -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. @@ -60,5 +58,3 @@ docker build -t packager . # Copy deb file out of container to host docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out" mv pandare.deb pandare_${version}.deb - - diff --git a/panda/docs/build_ubuntu.md b/panda/docs/build_ubuntu.md index e3949d03181..bc2feb6828c 100644 --- a/panda/docs/build_ubuntu.md +++ b/panda/docs/build_ubuntu.md @@ -1,6 +1,6 @@ # Building on Ubuntu -Panda's build and runtime dependencies are listed in `panda/panda/dependencies/` for ubuntu 18, 19 and 20. +Panda's build and runtime dependencies are listed in `panda/panda/dependencies/` for ubuntu 18, 20 and 22. If you install all the packages listed there, clone panda, and run build.sh you should be able to build panda: @@ -19,10 +19,7 @@ mkdir -p build && cd build If you would like to use **PyPANDA** you'll need to also install it and its dependencies: ```sh -# Pypanda dependencies -pip3 install pycparser "protobuf==3.0.0" "https://foss.heptapod.net/pypy/cffi/-/archive/branch/default/cffi-branch-default.zip" colorama - -# Install pypanda +# Install pypanda, see requirements.txt for the installed python dependancies cd panda/panda/python/core python3 setup.py install ``` diff --git a/panda/python/core/README.md b/panda/python/core/README.md index 7b44ee00084..ac4195fd874 100644 --- a/panda/python/core/README.md +++ b/panda/python/core/README.md @@ -25,10 +25,3 @@ from pandare import Panda panda = Panda(generic='i386') ... ``` - - -This is a beta release - Last updated Dec 2021. ---- -* Although PANDA is fairly stable, this interface is new and subject to change significantly prior to version 1.0. - -* This package is manually generated and may fall behind the code on GitHub. From 3c913ffdc73e573132977b28dda15b44a4acbb01 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Tue, 30 Jul 2024 13:25:17 -0400 Subject: [PATCH 4/4] Bump Actions versions to the latest version Part of a patch written by Andrew Quijano --- .github/workflows/local_tests.yml | 5 ++++- .github/workflows/parallel_tests.yml | 4 ++-- .github/workflows/publish_docker.yml | 26 ++++++++++++++------------ .github/workflows/stale.yml | 2 +- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/local_tests.yml b/.github/workflows/local_tests.yml index 1da5e317390..df5695c965c 100644 --- a/.github/workflows/local_tests.yml +++ b/.github/workflows/local_tests.yml @@ -5,11 +5,14 @@ name: Local +on: + workflow_dispatch: + jobs: local_build_container: runs-on: panda-arc steps: - - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory + - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Build docker container from project root run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local:${{ github.sha }} . diff --git a/.github/workflows/parallel_tests.yml b/.github/workflows/parallel_tests.yml index 805415a6470..d58d93e7948 100644 --- a/.github/workflows/parallel_tests.yml +++ b/.github/workflows/parallel_tests.yml @@ -29,7 +29,7 @@ jobs: - name: Install ssl run: apt-get -qq install -y libssl-dev - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install Python dev headers @@ -159,7 +159,7 @@ jobs: runs-on: panda-arc steps: - - uses: actions/checkout@v1 # Clones code into to /home/runner/work/panda + - uses: actions/checkout@v4 # Clones code into to /home/runner/work/panda - name: Build docker container from project root run: cd $GITHUB_WORKSPACE && docker build -t panda_local . diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 4787f019c12..8867c720020 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -1,6 +1,7 @@ name: Build and Publish Docker Container and Pypanda Docs # Only for main panda-re repo, not forks on: + workflow_run: push: branches: - dev @@ -14,7 +15,6 @@ jobs: if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/dev' runs-on: panda-arc outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} v-version: ${{ steps.version.outputs.v-version }} steps: - name: Install git @@ -60,9 +60,9 @@ jobs: panda/debian/pandare_*.deb - name: Store the PyPanda distribution packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: python-package-distributions + name: pypanda path: panda/debian/pandare-*.whl if-no-files-found: error @@ -72,6 +72,7 @@ jobs: with: username: pandare password: ${{secrets.pandare_dockerhub}} + #- name: 'Login to GHCR Registry' # if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }} @@ -151,9 +152,9 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python-package-distributions + name: pypanda path: dist/ - name: Publish distribution 📦 to PyPI @@ -176,13 +177,14 @@ jobs: - name: Build Bionic container # Push both dev and regular container - run: DOCKER_BUILDKIT=1 docker build --progress=plain --target=panda -t pandare/panda_stable:${GITHUB_SHA} $GITHUB_WORKSPACE; - docker tag pandare/panda_stable:${GITHUB_SHA} pandare/panda_stable:latest - docker push pandare/panda_stable:${GITHUB_SHA}; - docker push pandare/panda_stable; - #DOCKER_BUILDKIT=1 docker build --progress=plain --target=developer -t pandare/pandadev:${GITHUB_SHA} $GITHUB_WORKSPACE; - #docker tag pandare/panadev:${GITHUB_SHA} pandare/pandadev:latest - #docker push pandare/pandadev; + uses: docker/build-push-action@v5 + with: + push: true + context: ${{ github.workspace }} + tags: | + pandare/panda_stable:${{ github.sha }} + pandare/panda_stable:latest + target: panda - name: Checkout docs and reset run: rm -rf "${GITHUB_WORKSPACE}/auto_pydoc"; diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d2425d5a088..dd3378ae001 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v3 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-close: 30