Skip to content

Commit

Permalink
Merge pull request #40 from lucasw/debianize
Browse files Browse the repository at this point in the history
Updates and debian experiments
  • Loading branch information
lucasw authored Mar 30, 2024
2 parents 0fdc1a0 + 968035f commit 5efcb23
Show file tree
Hide file tree
Showing 31 changed files with 1,356 additions and 113 deletions.
136 changes: 121 additions & 15 deletions .github/workflows/ubuntu_22_04.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ jobs:
fail-fast: false
matrix:
build_type: [install, devel]
build_all: [minimal, full]

runs-on: ubuntu-22.04
# env:
Expand Down Expand Up @@ -50,6 +51,7 @@ jobs:
sudo apt-get install -o Debug::pkgProblemResolver=true -yqq libunwind-dev
dpkg -l | grep libunwind
sudo apt-get install -o Debug::pkgProblemResolver=true -yqq libceres-dev
sudo apt-get install -yqq devscripts dh-make
- name: apt library installs 2
run: |
Expand All @@ -63,9 +65,16 @@ jobs:
sudo apt-get install -yqq libspnav-dev liburdfdom-dev
sudo apt-get install -yqq libyaml-cpp-dev
sudo apt-get install -yqq cython3
sudo apt-get install -yqq freeglut3-dev
sudo apt-get install -yqq libapriltag-dev
sudo apt-get install -yqq libcgal-dev
sudo apt-get install -yqq libfmt-dev
sudo apt-get install -yqq libgsl-dev
sudo apt-get install -yqq libhdf5-dev
sudo apt-get install -yqq libturbojpeg0-dev
sudo apt-get install -yqq libzmq3-dev
sudo apt-get install -yqq ocl-icd-opencl-dev opencl-headers
sudo apt-get install -yqq libopenvdb-dev
- name: apt sdl installs
run: |
Expand Down Expand Up @@ -108,6 +117,7 @@ jobs:
run: |
echo "PATH=$PATH:$DEST/bin" >> $GITHUB_ENV
echo "PYTHONPATH=$DEST/lib/python$PYTHON_MAJOR_VERSION.$PYTHON_MINOR_VERSION/site-packages/" >> $GITHUB_ENV
echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$DEST/lib/cmake" >> $GITHUB_ENV
- name: check setup
run: |
Expand Down Expand Up @@ -165,41 +175,103 @@ jobs:
sed -i 's/git@github.com:/https:\/\/github.com\//' base_repos.yaml
ls -l
vcs import --shallow < base_repos.yaml
# show which versions were imported
vcs export --exact
# ignore repos that aren't yet building in 22.04
../../other/src/ros_from_src/ubuntu_2204/ignore.sh
# the newer openvdb that doesn't have tbb error that the apt install version has
- name: install build openvdb
if: ${{ matrix.build_all == 'full' }}
run: |
mkdir -p other/build/openvdb
cd other/build/openvdb
cmake ../../../${{ matrix.build_type }}_base_catkin_ws/src/other/vdb/openvdb/ -DCMAKE_INSTALL_PREFIX=$DEST
make -j3 # it's a slow build
make install
# TODO(lucasw) later have an entirely separately layer for the larger slow building packages
- name: ignore large slow packages
if: ${{ matrix.build_all == 'minimal' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws/src
touch other/PlotJuggler/CATKIN_IGNORE
touch other/plotjuggler_msgs/CATKIN_IGNORE
touch other/plotjuggler-ros-plugins/CATKIN_IGNORE
touch other/jsk_common/CATKIN_IGNORE
touch other/jsk_common_msgs/CATKIN_IGNORE
touch other/jsk_recognition/CATKIN_IGNORE
touch other/jsk_roseus/CATKIN_IGNORE
touch other/jsk_visualization/CATKIN_IGNORE
touch other/fiducials/CATKIN_IGNORE
touch other/fuse/CATKIN_IGNORE
touch other/lvr2/CATKIN_IGNORE
touch other/mesh_tools/CATKIN_IGNORE
touch other/rtabmap/CATKIN_IGNORE
touch other/rtabmap_ros/CATKIN_IGNORE
touch ros/grid_map/CATKIN_IGNORE
touch other/anybotics/CATKIN_IGNORE
touch other/octomap_mapping/CATKIN_IGNORE
touch other/octomap_ros/CATKIN_IGNORE
touch other/vdb/CATKIN_IGNORE
- name: devel setup
if: ${{ matrix.build_type == 'devel' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release -Wno-deprecated
catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release -Wno-deprecated -Wno-dev -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=$DEST/include -DCMAKE_LIBRARY_PATH=$DEST/lib
# TODO(lucasw) lvr2 not building in devel, but could build it
# outside of catkin entirely and install to DEST
touch src/other/lvr2/CATKIN_IGNORE
touch src/other/mesh_tools/CATKIN_IGNORE
- name: install setup
if: ${{ matrix.build_type == 'install' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
catkin config --install --cmake-args -DCMAKE_BUILD_TYPE=Release -Wno-deprecated
catkin config --install --cmake-args -DCMAKE_BUILD_TYPE=Release -Wno-deprecated -Wno-dev -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=$DEST/include -DCMAKE_LIBRARY_PATH=$DEST/lib
# back to devel build
- name: build fiducials
# individual packages or sets of packages built separately
# this first one sources last because nothing has been built yet
- name: build qt_gui_core
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
catkin build --no-status fiducials
catkin build --no-status qt_gui_core
source ${{ matrix.build_type }}/setup.bash
- name: build grid_map
- name: build vdb mapping
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin build --no-status grid_map*
catkin build --no-status vdb_mapping* --verbose
- name: build qt_gui_core
- name: build fiducials
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin build --no-status qt_gui_core
catkin build --no-status fiducials
- name: build grid_map
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin build --no-status grid_map*
- name: build rtabmap_ros
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
Expand All @@ -212,7 +284,8 @@ jobs:
catkin build --no-status joint_trajectory_controller
- name: test joint_trajectory_controller
if: ${{ matrix.build_type == 'devel' }}
# TODO(lucasw) this test is slow, maybe don't do it at all
if: ${{ (matrix.build_type == 'devel' && matrix.build_all == 'full') }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
Expand All @@ -221,13 +294,14 @@ jobs:
# catkin test --no-status joint_trajectory_controller
- name: test install joint_trajectory_controller
if: ${{ matrix.build_type == 'install' }}
if: ${{ (matrix.build_type == 'devel' && matrix.build_all == 'full') }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin test --no-status joint_trajectory_controller
- name: build plotjuggler
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
Expand All @@ -243,13 +317,22 @@ jobs:
catkin build --no-status rviz
source ${{ matrix.build_type }}/setup.bash
- name: build rviz_map_plugin
if: ${{ matrix.build_all == 'full' && matrix.build_type == 'install' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin build --no-status rviz_map_plugin
- name: build jsk
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
catkin build --no-status jsk*
- name: build fuse
if: ${{ matrix.build_all == 'full' }}
run: |
cd ${{ matrix.build_type }}_base_catkin_ws
source ${{ matrix.build_type }}/setup.bash
Expand All @@ -264,12 +347,35 @@ jobs:
- name: tar up install_catkin_ws
if: ${{ matrix.build_type == 'install' }}
run: |
tar cvzf install_catkin_ws_2204.tgz ${{ matrix.build_type }}_base_catkin_ws/install
tar cvzf install_catkin_ws_2204.tgz install_base_catkin_ws/install
tar cvzf install_dest_2204.tgz $DEST
ls -l
- name: update install_catkin_ws_2204.tgz
- name: upload install_catkin_ws_2204.tgz
if: ${{ matrix.build_type == 'install' }}
uses: actions/upload-artifact@v4
with:
name: install_dest_2204_${{ matrix.build_all }}
path: install_dest_2204.tgz

- name: upload install_catkin_ws_2204.tgz
if: ${{ matrix.build_type == 'install' }}
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: install_catkin_ws_2204
name: install_catkin_ws_2204__${{ matrix.build_all }}
path: install_catkin_ws_2204.tgz

- name: make a deb from the entirety of install_catkin_ws/install
if: ${{ matrix.build_type == 'install' }}
run: |
cd other/src/ros_from_src/debian
ln -s ../../../../install_base_catkin_ws/install
dpkg-buildpackage -A -uc
- name: upload .deb
if: ${{ matrix.build_type == 'install' }}
uses: actions/upload-artifact@v4
with:
name: install_catkin_ws_2204_${{ matrix.build_all }}.deb
# TODO(lucasw) extract name from debian files, VERSION?
path: other/src/ros_from_src/rosone_0.0.6-1_all.deb
5 changes: 3 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ RUN apt-get install -y apt-utils
# TODO(lucasw) used to do these on separate layers but github actions has a layer limit?
# it was failing later with 'ERROR: failed to solve: failed to prepare...max depth exceeded'
RUN apt-get install -y build-essential bzip2 libbz2-dev cmake git libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libboost-thread-dev libfmt-dev libgpgme-dev libgtest-dev liblog4cxx-dev liblz4-dev lz4 libpoco-dev libtinyxml2-dev mawk coreutils python-is-python3 python3 python3-dev python3-empy python3-setuptools python3-yaml
RUN apt-get update && apt-get install -yqq libgsl-dev wget libspnav-dev liburdfdom-dev libyaml-cpp-dev cython3 freeglut3-dev libapriltag-dev libcgal-dev libhdf5-dev libturbojpeg0-dev libzmq3-dev ocl-icd-opencl-dev opencl-headers

ENV SRC=/src
RUN mkdir $SRC -p
Expand Down Expand Up @@ -178,7 +179,7 @@ RUN rosdep update || true
WORKDIR $WS/..
RUN source $DEST/setup.bash
RUN catkin init
RUN source $DEST/setup.bash && catkin config
RUN source $DEST/setup.bash && catkin config --install --cmake-args -DCMAKE_BUILD_TYPE=Release -Wno-deprecated
# rospack list won't work by itself
RUN source $DEST/setup.bash && rospack list

Expand All @@ -191,7 +192,7 @@ RUN echo $CMAKE_PREFIX_PATH
RUN echo $ROS_PACKAGE_PATH
RUN catkin build
# rospack list won't work by itself
RUN source devel/setup.bash && rospack list
RUN source install/setup.bash && rospack list

RUN apt-get install python3-netifaces

Expand Down
114 changes: 71 additions & 43 deletions debian/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# install all ros packages from debian
# install all ros packages from debian, then install custom debian built from ros_from_src
# cd ros_from_src # be one directory up from debian/
# docker build debian -t ros_from_src_ubuntu_2204
ARG IMAGE=ubuntu:22.04
FROM ${IMAGE}
ARG IMAGE
Expand All @@ -9,48 +11,74 @@ ENV DEBIAN_FRONTEND="noninteractive"
# be able to source files
RUN rm /bin/sh && ln -s /bin/bash /bin/sh

RUN apt-get update
RUN apt-get update -y
RUN apt-get upgrade -y
RUN apt-get install -y apt-utils

# apt installs
RUN apt-get update
RUN apt-get install -y build-essential
RUN apt-get install -y bzip2 libbz2-dev
RUN apt-get install -y cmake
RUN apt-get install -y coreutils
RUN apt-get install -y git
RUN apt-get install -y libboost-dev
RUN apt-get install -y libboost-filesystem-dev
RUN apt-get install -y libboost-program-options-dev
RUN apt-get install -y libboost-regex-dev
RUN apt-get install -y libboost-thread-dev
RUN apt-get install -y libgpgme-dev
RUN apt-get install -y libgtest-dev
RUN apt-get install -y liblog4cxx-dev
RUN apt-get install -y liblz4-dev lz4
RUN apt-get install -y libpoco-dev
RUN apt-get install -y libtinyxml2-dev
RUN apt-get install -y mawk

RUN apt-get update
RUN apt-get install -y python-is-python3
RUN apt-get install -y python3
RUN apt-get install -y python3-dateutil
RUN apt-get install -y python3-defusedxml
RUN apt-get install -y python3-dev
RUN apt-get install -y python3-distro
RUN apt-get install -y python3-docutils
RUN apt-get install -y python3-empy
RUN apt-get install -y python3-netifaces
RUN apt-get install -y python3-pyparsing
RUN apt-get install -y python3-setuptools
RUN apt-get install -y python3-yaml

RUN apt-get update
# ros debian installs
# https://wiki.debian.org/DebianScience/Robotics/ROS/Packages
# RUN apt-get install -y ros-robot-state-publisher
RUN apt-get install -y rviz
RUN apt-get install -y ros-robot
RUN apt-get install -y ros-robot-dev
RUN apt-get install -y python3-vcstools
RUN apt-get update && apt-get install -y git
RUN apt-get install -y ros-*
RUN apt-get install -y catkin-lint cython3 libapriltag-dev libceres-dev libfmt-dev libfrei0r-ocaml-dev libgmock-dev libgoogle-glog-dev libgst-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libimage-view-dev liborocos-bfl-dev libpcl-ros-dev libqt5svg5-dev libqt5websockets5-dev libqt5x11extras5-dev libqwt-qt5-dev libsdl-image1.2-dev libspnav-dev liburdfdom-dev libuvc-dev libv4l-dev libyaml-cpp-dev python-is-python3 python3-tf2-geometry-msgs python3-venv vim curl jq --fix-missing
# not depended on by anything in the rosone deb but useful to already have for building other packages from src
RUN apt-get install -y google-mock libdiagnostic-updater-dev libmuparser-dev libopenvdb-dev python3-geopy python3-open3d python3-pip python3-scipy
# RUN apt-get install -y libgeographic-dev

################################################################################
# catkin and related setup
# TODO(lucasw) there needs to be a deb made for just these packages, they aren't in the deb installed below
ENV DEST=/opt/ros/base
RUN mkdir $DEST -p
ENV SRC=/opt/src
RUN mkdir $SRC -p

WORKDIR $SRC
RUN python --version | awk '{print $2}' | cut -d'.' -f1
RUN python --version | awk '{print $2}' | cut -d'.' -f2
# TODO(lucasw) these aren't working
# RUN export PYTHON_MAJOR_VERSION=`python --version | awk '{print $2}' | cut -d'.' -f1`
# RUN export PYTHON_MINOR_VERSION=`python --version | awk '{print $2}' | cut -d'.' -f2`
# RUN PYTHON_MINOR_VERSION=`python --version | awk '{print $2}' | cut -d'.' -f2`
ARG PYTHON_MAJOR_VERSION=3
ARG PYTHON_MINOR_VERSION=10
ENV OPT_PYTHONPATH=$DEST/lib/python$PYTHON_MAJOR_VERSION.$PYTHON_MINOR_VERSION/site-packages/
RUN echo $PYTHONPATH
ENV PYTHONPATH=$OPT_PYTHONPATH
RUN echo $PYTHONPATH
ENV PATH=$DEST/bin:$PATH
################################################################################

# this deb isn't working right, env vars aren't set properly and rosrun doesn't work
# for packages inside it
# ARG DEB=rosone_0.0.6-1_all.deb
# RUN echo ${DEB}
# COPY rosone_0.0.6-1_all.deb .
# COPY ${DEB} .
# RUN dpkg -i ${DEB}
# TODO(lucasw) this setup isn't doing much, have to set this manually
# ENV ROS_PACKAGE_PATH=/opt/ros/one/share:/usr/share

WORKDIR /opt/ros
RUN apt-get install -yqq zip

COPY install_dest_2204_full.zip .
RUN unzip install_dest_2204_full.zip
RUN tar xvf install_dest_2204.tgz
# TODO(lucasw) zip up the install in a way that exclude the full path
RUN mv home/runner/other/install/* $DEST
RUN rm -rf home
RUN rm install_dest_2204*

RUN which catkin && catkin --help

COPY install_catkin_ws_2204__full.zip .
RUN unzip install_catkin_ws_2204__full.zip
RUN tar xvf install_catkin_ws_2204.tgz
RUN mv install_base_catkin_ws/install one
RUN rmdir install_base_catkin_ws
RUN rm install_catkin_ws*

RUN du -sh /opt/ros
RUN find /opt/ros | head -n 30
RUN ls -l /usr/lib/x86_64-linux-gnu/libpython*
RUN source /opt/ros/one/setup.bash && echo $ROS_PACKAGE_PATH && echo $PATH
RUN source /opt/ros/one/setup.bash && rospack find pcl_ros
Loading

0 comments on commit 5efcb23

Please sign in to comment.