Python 3 transition

aruco_pose/src/genmap.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (C) 2018 Copter Express Technologies
 #

aruco_pose/test/largemap.py

@@ -1,26 +1,19 @@
-#!/usr/bin/env python
-import sys
-import unittest
-import json
 import rospy
-import rostest
+import pytest
 
 from sensor_msgs.msg import Image
 from visualization_msgs.msg import MarkerArray as VisMarkerArray
 
+@pytest.fixture
+def node():
+    return rospy.init_node('test_aruco_largemap', anonymous=True)
 
-class TestArucoPose(unittest.TestCase):
-    def setUp(self):
-        rospy.init_node('test_aruco_largemap', anonymous=True)
+def test_large_map_image(node):
+    img = rospy.wait_for_message('aruco_map/image', Image, timeout=5)
+    assert img.width == 2000
+    assert img.height == 2000
+    assert img.encoding in ('mono8', 'rgb8')
 
-    def test_map_image(self):
-        img = rospy.wait_for_message('aruco_map/image', Image, timeout=5)
-        self.assertEqual(img.width, 2000)
-        self.assertEqual(img.height, 2000)
-        self.assertIn(img.encoding, ('mono8', 'rgb8'))
-
-    def test_map_visualization(self):
-        vis = rospy.wait_for_message('aruco_map/visualization', VisMarkerArray, timeout=5)
-
-
-rostest.rosrun('aruco_pose', 'test_aruco_largemap', TestArucoPose, sys.argv)
+def test_large_map_visualization(node):
+    vis = rospy.wait_for_message('aruco_map/visualization', VisMarkerArray, timeout=5)
+    assert len(vis.markers) == 11

aruco_pose/test/largemap.test

@@ -9,5 +9,6 @@
         <param name="map" value="$(find aruco_pose)/test/largemap.txt"/>
     </node>
 
-    <test test-name="test_aruco_pose_largemap" pkg="aruco_pose" type="largemap.py"/>
+    <param name="test_module" value="$(find aruco_pose)/test/largemap.py"/>
+    <test test-name="test_node_pass" pkg="ros_pytest" type="ros_pytest_runner"/>
 </launch>

builder/assets/clever/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from distutils.core import setup
 

builder/assets/python3.yaml

@@ -0,0 +1,9 @@
+python3-wxgtk:
+  debian:
+    buster: [python3-wxgtk4.0]
+python3-serial:
+  debian:
+    buster: [python3-serial]
+python3-requests:
+  debian:
+    buster: [python3-requests]

builder/image-build.sh

@@ -115,6 +115,7 @@ ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/image-network.
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/clover.service' '/lib/systemd/system/'
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/roscore.service' '/lib/systemd/system/'
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/melodic-rosdep-clover.yaml' '/etc/ros/rosdep/'
+${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/python3.yaml' '/etc/ros/rosdep/'
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/ros_python_paths' '/etc/sudoers.d/'
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/pigpiod.service' '/lib/systemd/system/'
 ${BUILDER_DIR}/image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/launch.nanorc' '/usr/share/nano/'

builder/image-ros.sh

@@ -68,7 +68,8 @@ my_travis_retry() {
 # TODO: 'kinetic-rosdep-clover.yaml' should add only if we use our repo?
 echo_stamp "Init rosdep"
 my_travis_retry rosdep init
-echo "yaml file:///etc/ros/rosdep/melodic-rosdep-clover.yaml" >> /etc/ros/rosdep/sources.list.d/20-default.list
+echo "yaml file:///etc/ros/rosdep/melodic-rosdep-clover.yaml" > /etc/ros/rosdep/sources.list.d/30-clover.list
+echo "yaml file:///etc/ros/rosdep/python3.yaml" > /etc/ros/rosdep/sources.list.d/40-python3.list
 my_travis_retry rosdep update
 
 echo_stamp "Populate rosdep for ROS user"
@@ -87,6 +88,7 @@ resolve_rosdep() {
 }
 
 export ROS_IP='127.0.0.1' # needed for running tests
+export ROS_PYTHON_VERSION=3
 
 echo_stamp "Reconfiguring Clover repository for simplier unshallowing"
 cd /home/pi/catkin_ws/src/clover
@@ -95,8 +97,8 @@ git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
 echo_stamp "Build and install Clover"
 cd /home/pi/catkin_ws
 resolve_rosdep $(pwd)
-my_travis_retry pip install wheel
-my_travis_retry pip install -r /home/pi/catkin_ws/src/clover/clover/requirements.txt
+my_travis_retry pip3 install wheel
+my_travis_retry pip3 install -r /home/pi/catkin_ws/src/clover/clover/requirements.txt
 source /opt/ros/melodic/setup.bash
 catkin_make -j2 -DCMAKE_BUILD_TYPE=Release
 
@@ -133,7 +135,7 @@ echo_stamp "Install GeographicLib datasets (needed for mavros)" \
 
 # FIXME: Buster comes with tornado==5.1.1 but we need tornado==4.2.1 for rosbridge_suite
 # (note that Python 3 will still have a more recent version)
-pip install tornado==4.2.1
+pip3 install tornado==4.2.1
 
 echo_stamp "Running tests"
 cd /home/pi/catkin_ws

builder/image-software.sh

@@ -67,7 +67,7 @@ apt-get update \
 
 echo "deb http://packages.ros.org/ros/ubuntu buster main" > /etc/apt/sources.list.d/ros-latest.list
 echo "deb http://deb.coex.tech/opencv3 buster main" > /etc/apt/sources.list.d/opencv3.list
-echo "deb http://deb.coex.tech/rpi-ros-melodic buster main" > /etc/apt/sources.list.d/rpi-ros-melodic.list
+echo "deb http://deb.coex.tech/melodic-py3 buster main" > /etc/apt/sources.list.d/rpi-ros-melodic.list
 echo "deb http://deb.coex.tech/clover buster main" > /etc/apt/sources.list.d/clover.list
 
 echo_stamp "Update apt cache"
@@ -95,20 +95,21 @@ libjpeg8 \
 tcpdump \
 ltrace \
 libpoco-dev \
-python-rosdep \
-python-rosinstall-generator \
-python-wstool \
-python-rosinstall \
+python3-rosdep \
+python3-rosinstall-generator \
+python3-wstool \
+python3-rosinstall \
 build-essential \
 libffi-dev \
 monkey \
 pigpio python-pigpio python3-pigpio \
 i2c-tools \
-espeak espeak-data python-espeak \
+espeak espeak-data python3-espeak \
 ntpdate \
 python-dev \
 python3-dev \
-python-systemd \
+python3-venv \
+python3-systemd \
 mjpg-streamer \
 python3-opencv \
 && echo_stamp "Everything was installed!" "SUCCESS" \
@@ -132,13 +133,14 @@ pip3 --version
 
 echo_stamp "Install and enable Butterfly (web terminal)"
 echo_stamp "Workaround for tornado >= 6.0 breaking butterfly"
-my_travis_retry pip3 install tornado==5.1.1
+my_travis_retry pip3 install tornado==4.2.1
 my_travis_retry pip3 install butterfly
 my_travis_retry pip3 install butterfly[systemd]
 systemctl enable butterfly.socket
 
 echo_stamp "Install ws281x library"
-my_travis_retry pip install --prefer-binary rpi_ws281x
+my_travis_retry pip2 install --prefer-binary rpi_ws281x
+my_travis_retry pip3 install --prefer-binary rpi_ws281x
 
 echo_stamp "Setup Monkey"
 mv /etc/monkey/sites/default /etc/monkey/sites/default.orig
@@ -154,7 +156,7 @@ rm -rf node-v10.15.0-linux-armv6l/
 rm node-v10.15.0-linux-armv6l.tar.gz
 
 echo_stamp "Installing ptvsd"
-my_travis_retry pip install ptvsd
+my_travis_retry pip2 install ptvsd
 my_travis_retry pip3 install ptvsd
 
 echo_stamp "Add .vimrc"

builder/image-validate.sh

@@ -18,13 +18,15 @@ echo "Run image tests"
 
 export ROS_DISTRO='melodic'
 export ROS_IP='127.0.0.1'
+export ROS_PYTHON_VERSION=3
 source /opt/ros/melodic/setup.bash
 source /home/pi/catkin_ws/devel/setup.bash
 
 cd /home/pi/catkin_ws/src/clover/builder/test/
 ./tests.sh
 ./tests.py
-./tests_py3.py
+# Disable Python 2 tests for image - we're dropping support, right?
+# ./tests_py2.py
 [[ $(./tests_clever.py) == "Warning: clever package is renamed to clover" ]]  # test backwards compatibility
 
 echo "Move /etc/ld.so.preload back to its original position"

builder/test/tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # validate all required modules installed
 
@@ -17,6 +17,8 @@
 from clover.srv import GetTelemetry, Navigate, NavigateGlobal, SetPosition, SetVelocity, \
     SetAttitude, SetRates, SetLEDEffect
 
+get_telemetry = rospy.ServiceProxy('get_telemetry', GetTelemetry)
+
 import tf2_ros
 import tf2_geometry_msgs
 
@@ -27,4 +29,4 @@
 import pigpio
 from espeak import espeak
 
-print cv2.getBuildInformation()
+print(cv2.getBuildInformation())

builder/test/tests_clever.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # test backwards compatibility
 

builder/test/tests_py2.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+# Make sure our Python 2 software is installed
+
+import cv2
+
+print(cv2.getBuildInformation())

clover/package.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<package format="2">
+<package format="3">
   <name>clover</name>
   <version>0.0.1</version>
   <description>The Clover package</description>
@@ -37,8 +37,10 @@
   <depend>rosbridge_server</depend>
   <depend>web_video_server</depend>
   <depend>tf2_web_republisher</depend>
-  <depend>python-lxml</depend>
-  <exec_depend>python-pymavlink</exec_depend>
+  <depend condition="$ROS_PYTHON_VERSION == 2">python-lxml</depend>
+  <depend condition="$ROS_PYTHON_VERSION == 3">python3-lxml</depend>
+  <exec_depend condition="$ROS_PYTHON_VERSION == 2">python-pymavlink</exec_depend>
+  <exec_depend condition="$ROS_PYTHON_VERSION == 3">python-pymavlink</exec_depend>
   <!-- Use test_depend for packages you need only for testing: -->
   <!--   <test_depend>gtest</test_depend> -->
 

clover/requirements.txt

@@ -1,5 +1,5 @@
 flask==1.1.1
 docopt==0.6.2
-geopy==1.11.0
-smbus2==0.2.1
-VL53L1X==0.0.2
+geopy==1.20.0
+smbus2==0.3.0
+VL53L1X==0.0.4

clover/src/selfcheck.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding=utf-8
 
 # Copyright (C) 2018 Copter Express Technologies
@@ -138,7 +138,7 @@ def mavlink_exec(cmd, timeout=3.0):
         timeout=3,
         baudrate=0,
         count=len(cmd),
-        data=map(ord, cmd.ljust(70, '\0')))
+        data=[ord(c) for c in cmd.ljust(70, '\0')])
     msg.pack(link)
     ros_msg = mavlink.convert_to_rosmsg(msg)
     mavlink_pub.publish(ros_msg)
@@ -609,7 +609,7 @@ def check_rangefinder():
 
 @check('Boot duration')
 def check_boot_duration():
-    output = subprocess.check_output('systemd-analyze')
+    output = subprocess.check_output('systemd-analyze').decode()
     r = re.compile(r'([\d\.]+)s\s*$', flags=re.MULTILINE)
     duration = float(r.search(output).groups()[0])
     if duration > 15:
@@ -620,7 +620,7 @@ def check_boot_duration():
 def check_cpu_usage():
     WHITELIST = 'nodelet',
     CMD = "top -n 1 -b -i | tail -n +8 | awk '{ printf(\"%-8s\\t%-8s\\t%-8s\\n\", $1, $9, $12); }'"
-    output = subprocess.check_output(CMD, shell=True)
+    output = subprocess.check_output(CMD, shell=True).decode()
     processes = output.split('\n')
     for process in processes:
         if not process:
@@ -636,7 +636,7 @@ def check_cpu_usage():
 def check_clover_service():
     try:
         output = subprocess.check_output('systemctl show -p ActiveState --value clover.service'.split(),
-                                         stderr=subprocess.STDOUT)
+                                         stderr=subprocess.STDOUT).decode()
     except subprocess.CalledProcessError as e:
         failure('systemctl returned %s: %s', e.returncode, e.output)
         return
@@ -751,7 +751,7 @@ def check_rpi_health():
         # <parameter>=<value>
         # In case of `get_throttled`, <value> is a hexadecimal number
         # with some of the FLAGs OR'ed together
-        output = subprocess.check_output(['vcgencmd', 'get_throttled'])
+        output = subprocess.check_output(['vcgencmd', 'get_throttled']).decode()
     except OSError:
         failure('could not call vcgencmd binary; not a Raspberry Pi?')
         return

clover/test/basic.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import rospy
 import pytest
 from mavros_msgs.msg import State