From 071d273dbf664acf2821b6b52a72ddd6f25b4266 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 23 Feb 2021 19:39:37 +0100 Subject: [PATCH 01/68] Fix jobs flags handling (#655) * fixup! argument parser: use None instead of [] for empty jobs args (#575) * add test that asserts that job flags do not disappear --- catkin_tools/argument_parsing.py | 2 +- tests/system/verbs/catkin_config/test_config.py | 10 ++++++++++ tests/workspace_assertions.py | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/catkin_tools/argument_parsing.py b/catkin_tools/argument_parsing.py index 16d557e0..fd8a2185 100644 --- a/catkin_tools/argument_parsing.py +++ b/catkin_tools/argument_parsing.py @@ -254,7 +254,7 @@ def extract_jobs_flags(mflags): :rtype: list """ if not mflags: - return [] + return None # Each line matches a flag type, i.e. -j, -l, --jobs, --load-average # (?:^|\s) and (?=$|\s) make sure that the flag is surrounded by whitespace diff --git a/tests/system/verbs/catkin_config/test_config.py b/tests/system/verbs/catkin_config/test_config.py index 98886c50..517d60c1 100644 --- a/tests/system/verbs/catkin_config/test_config.py +++ b/tests/system/verbs/catkin_config/test_config.py @@ -6,6 +6,7 @@ from ....workspace_assertions import assert_workspace_initialized from ....workspace_assertions import assert_warning_message from ....workspace_assertions import assert_no_warnings +from ....workspace_assertions import assert_in_config @in_temporary_directory @@ -28,3 +29,12 @@ def test_config_non_bare(): out = assert_cmd_success(['catkin', 'config', '--install']) assert_workspace_initialized('.') assert_warning_message(out, 'Source space .+ does not yet exist') + + +@in_temporary_directory +def test_config_unchanged(): + cwd = os.getcwd() + os.mkdir(os.path.join(cwd, 'src')) + assert_cmd_success(['catkin', 'config', '--make-args', '-j6']) + assert_cmd_success(['catkin', 'config']) + assert_in_config('.', 'default', 'jobs_args', ['-j6']) diff --git a/tests/workspace_assertions.py b/tests/workspace_assertions.py index 2ae912fe..ca239b0b 100644 --- a/tests/workspace_assertions.py +++ b/tests/workspace_assertions.py @@ -1,6 +1,8 @@ from __future__ import print_function +import os import re +import yaml from .utils import assert_files_exist @@ -31,3 +33,10 @@ def assert_no_warnings(out_str): out_str_stripped = ' '.join(str(out_str).splitlines()) found = re.findall('WARNING:', out_str_stripped) assert len(found) == 0 + + +def assert_in_config(workspace, profile, key, value): + with open(os.path.join(workspace, '.catkin_tools', 'profiles', profile, 'config.yaml')) as f: + config = yaml.safe_load(f) + + assert config.get(key, None) == value From c4718f6ba27d7ed80cb42c2fb4839b0ec0e36295 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 23 Feb 2021 21:12:53 +0100 Subject: [PATCH 02/68] import cleanup (#651) --- catkin_tools/argument_parsing.py | 2 -- catkin_tools/commands/catkin.py | 10 ++-------- catkin_tools/common.py | 12 ++---------- catkin_tools/config.py | 6 ++---- catkin_tools/context.py | 2 -- catkin_tools/execution/controllers.py | 8 +------- catkin_tools/execution/executor.py | 10 ++-------- catkin_tools/execution/io.py | 2 -- catkin_tools/execution/job_server.py | 9 +++------ catkin_tools/execution/jobs.py | 2 -- catkin_tools/execution/stages.py | 6 +----- catkin_tools/jobs/catkin.py | 6 +----- catkin_tools/jobs/cmake.py | 7 ------- catkin_tools/jobs/commands/cmake.py | 2 -- catkin_tools/jobs/output.py | 2 -- catkin_tools/jobs/utils.py | 2 -- catkin_tools/metadata.py | 2 -- catkin_tools/resultspace.py | 14 ++------------ catkin_tools/terminal_color.py | 2 -- catkin_tools/utils.py | 2 -- catkin_tools/verbs/catkin_build/build.py | 8 +------- catkin_tools/verbs/catkin_build/cli.py | 2 -- catkin_tools/verbs/catkin_clean/clean.py | 7 +------ catkin_tools/verbs/catkin_clean/cli.py | 9 +-------- catkin_tools/verbs/catkin_config/cli.py | 2 -- catkin_tools/verbs/catkin_create/cli.py | 2 -- catkin_tools/verbs/catkin_env/cli.py | 2 -- catkin_tools/verbs/catkin_init/cli.py | 2 -- catkin_tools/verbs/catkin_list/cli.py | 2 -- catkin_tools/verbs/catkin_locate/cli.py | 2 -- catkin_tools/verbs/catkin_profile/cli.py | 2 -- tests/system/verbs/catkin_build/test_args.py | 2 -- tests/system/verbs/catkin_build/test_build.py | 2 -- .../system/verbs/catkin_build/test_bwlists.py | 2 -- .../system/verbs/catkin_build/test_context.py | 2 -- .../system/verbs/catkin_build/test_eclipse.py | 2 -- .../verbs/catkin_build/test_modify_ws.py | 2 -- .../verbs/catkin_build/test_unit_tests.py | 2 -- tests/unit/test_io.py | 9 ++------- tests/utils.py | 19 +++---------------- tests/workspace_assertions.py | 2 -- 41 files changed, 24 insertions(+), 168 deletions(-) diff --git a/catkin_tools/argument_parsing.py b/catkin_tools/argument_parsing.py index fd8a2185..1ae9e33d 100644 --- a/catkin_tools/argument_parsing.py +++ b/catkin_tools/argument_parsing.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse import os import re diff --git a/catkin_tools/commands/catkin.py b/catkin_tools/commands/catkin.py index 1f09f1a9..16c9123f 100644 --- a/catkin_tools/commands/catkin.py +++ b/catkin_tools/commands/catkin.py @@ -12,18 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse -from datetime import date import os import pkg_resources import sys - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote +from datetime import date +from shlex import quote as cmd_quote from catkin_tools.common import is_tty diff --git a/catkin_tools/common.py b/catkin_tools/common.py index 9e934257..d18e0dfe 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -12,23 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - +import asyncio import datetime import errno import os import re import sys from fnmatch import fnmatch - -import asyncio - from shlex import split as cmd_split - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote +from shlex import quote as cmd_quote from catkin_pkg.packages import find_packages diff --git a/catkin_tools/config.py b/catkin_tools/config.py index 8eaa20e3..31e8d1df 100644 --- a/catkin_tools/config.py +++ b/catkin_tools/config.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import yaml -import shlex +from shlex import split as cmd_split from .common import string_type @@ -104,7 +102,7 @@ def get_verb_aliases(path=catkin_config_path): parsed_value = None if isinstance(value, string_type): # Parse using shlex - parsed_value = shlex.split(value) + parsed_value = cmd_split(value) elif isinstance(value, list) or isinstance(value, type(None)): # Take plainly parsed_value = value diff --git a/catkin_tools/context.py b/catkin_tools/context.py index f1581da1..21bdc5b2 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -14,8 +14,6 @@ """This module implements a class for representing a catkin workspace context""" -from __future__ import print_function - import os import re import sys diff --git a/catkin_tools/execution/controllers.py b/catkin_tools/execution/controllers.py index 4506ab2b..79ea1ac1 100644 --- a/catkin_tools/execution/controllers.py +++ b/catkin_tools/execution/controllers.py @@ -12,18 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - # Python3 - from queue import Empty -except ImportError: - # Python2 - from Queue import Empty - import math import itertools import sys import threading import time +from queue import Empty from catkin_tools.common import disable_wide_log from catkin_tools.common import format_time_delta diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index bbceb3be..ce1db811 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -12,17 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import traceback - -from itertools import tee - import asyncio - +import traceback from concurrent.futures import ThreadPoolExecutor from concurrent.futures import FIRST_COMPLETED - +from itertools import tee from osrf_pycommon.process_utils import async_execute_process from osrf_pycommon.process_utils import get_loop diff --git a/catkin_tools/execution/io.py b/catkin_tools/execution/io.py index 0e725cb5..ad95997e 100644 --- a/catkin_tools/execution/io.py +++ b/catkin_tools/execution/io.py @@ -14,9 +14,7 @@ import os import shutil - from glob import glob - from osrf_pycommon.process_utils import AsyncSubprocessProtocol from catkin_tools.common import mkdir_p diff --git a/catkin_tools/execution/job_server.py b/catkin_tools/execution/job_server.py index 3185be90..03fd1bc9 100644 --- a/catkin_tools/execution/job_server.py +++ b/catkin_tools/execution/job_server.py @@ -12,18 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -from multiprocessing import cpu_count -from tempfile import mkstemp -from termios import FIONREAD - import array import fcntl import os import re import subprocess import time +from multiprocessing import cpu_count +from tempfile import mkstemp +from termios import FIONREAD from catkin_tools.common import log from catkin_tools.common import version_tuple diff --git a/catkin_tools/execution/jobs.py b/catkin_tools/execution/jobs.py index 1826926e..7bb5bc2b 100644 --- a/catkin_tools/execution/jobs.py +++ b/catkin_tools/execution/jobs.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from catkin_tools.terminal_color import ColorMapper mapper = ColorMapper() diff --git a/catkin_tools/execution/stages.py b/catkin_tools/execution/stages.py index 5d4e9dee..e8e43bd8 100644 --- a/catkin_tools/execution/stages.py +++ b/catkin_tools/execution/stages.py @@ -14,14 +14,10 @@ import os import traceback +from shlex import quote as cmd_quote from catkin_tools.common import string_type -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote - from .io import IOBufferLogger from .io import IOBufferProtocol diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index bc1d9f2f..b93ef780 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -14,11 +14,7 @@ import csv import os - -try: - from md5 import md5 -except ImportError: - from hashlib import md5 +from hashlib import md5 from catkin_tools.argument_parsing import handle_make_arguments diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 7638bca0..70c360df 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -45,13 +45,6 @@ mapper = ColorMapper() clr = mapper.clr -# FileNotFoundError from Python3 -try: - FileNotFoundError -except NameError: - class FileNotFoundError(OSError): - pass - def copy_install_manifest( logger, event_queue, diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index caff7311..56038129 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import re diff --git a/catkin_tools/jobs/output.py b/catkin_tools/jobs/output.py index 429c6a61..f97fc197 100644 --- a/catkin_tools/jobs/output.py +++ b/catkin_tools/jobs/output.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.terminal_color import ansi diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index d9cb8b36..fbd7756d 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import shutil diff --git a/catkin_tools/metadata.py b/catkin_tools/metadata.py index 15f1bfc9..9f17c88c 100644 --- a/catkin_tools/metadata.py +++ b/catkin_tools/metadata.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import pkg_resources import shutil diff --git a/catkin_tools/resultspace.py b/catkin_tools/resultspace.py index 0f0ad9c3..4c014791 100644 --- a/catkin_tools/resultspace.py +++ b/catkin_tools/resultspace.py @@ -12,22 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -try: - from md5 import md5 -except ImportError: - from hashlib import md5 import os import subprocess import sys - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote - +from hashlib import md5 from osrf_pycommon.process_utils import execute_process +from shlex import quote as cmd_quote from .common import parse_env_str from .common import string_type diff --git a/catkin_tools/terminal_color.py b/catkin_tools/terminal_color.py index 1851ee41..97ff6fd1 100644 --- a/catkin_tools/terminal_color.py +++ b/catkin_tools/terminal_color.py @@ -16,8 +16,6 @@ Module to enable color terminal output """ -from __future__ import print_function - import string import os diff --git a/catkin_tools/utils.py b/catkin_tools/utils.py index 91c6854c..e116fb81 100644 --- a/catkin_tools/utils.py +++ b/catkin_tools/utils.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index 02eb1633..99e4eff7 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -16,19 +16,13 @@ import os import pkg_resources +from queue import Queue import sys import time import traceback import yaml - import asyncio -try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue try: from catkin_pkg.packages import find_packages diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 2b0e1c09..a3ccead7 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse import logging import os diff --git a/catkin_tools/verbs/catkin_clean/clean.py b/catkin_tools/verbs/catkin_clean/clean.py index f4073cd8..4c42ba03 100644 --- a/catkin_tools/verbs/catkin_clean/clean.py +++ b/catkin_tools/verbs/catkin_clean/clean.py @@ -18,13 +18,8 @@ import sys import time import traceback +from queue import Queue -try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue try: from catkin_pkg.packages import find_packages diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index cbd41d6d..fbb73f29 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -12,13 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -try: - raw_input -except NameError: - raw_input = input - import os import shutil import sys @@ -59,7 +52,7 @@ def yes_no_loop(question): while True: - resp = str(raw_input(question + " [yN]: ")) + resp = str(input(question + " [yN]: ")) if resp.lower() in ['n', 'no'] or len(resp) == 0: return False elif resp.lower() in ['y', 'yes']: diff --git a/catkin_tools/verbs/catkin_config/cli.py b/catkin_tools/verbs/catkin_config/cli.py index fc431640..b49abf35 100644 --- a/catkin_tools/verbs/catkin_config/cli.py +++ b/catkin_tools/verbs/catkin_config/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_cmake_and_make_and_catkin_make_args diff --git a/catkin_tools/verbs/catkin_create/cli.py b/catkin_tools/verbs/catkin_create/cli.py index 483de8b7..f3f2b633 100644 --- a/catkin_tools/verbs/catkin_create/cli.py +++ b/catkin_tools/verbs/catkin_create/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_context_args diff --git a/catkin_tools/verbs/catkin_env/cli.py b/catkin_tools/verbs/catkin_env/cli.py index 08992ea5..7c709cd6 100644 --- a/catkin_tools/verbs/catkin_env/cli.py +++ b/catkin_tools/verbs/catkin_env/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import re import sys diff --git a/catkin_tools/verbs/catkin_init/cli.py b/catkin_tools/verbs/catkin_init/cli.py index 514ab714..4c601742 100644 --- a/catkin_tools/verbs/catkin_init/cli.py +++ b/catkin_tools/verbs/catkin_init/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_workspace_arg diff --git a/catkin_tools/verbs/catkin_list/cli.py b/catkin_tools/verbs/catkin_list/cli.py index dac82ace..d56563b6 100644 --- a/catkin_tools/verbs/catkin_list/cli.py +++ b/catkin_tools/verbs/catkin_list/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import sys from catkin_tools.argument_parsing import add_context_args diff --git a/catkin_tools/verbs/catkin_locate/cli.py b/catkin_tools/verbs/catkin_locate/cli.py index 7a676cc9..b8b2100f 100644 --- a/catkin_tools/verbs/catkin_locate/cli.py +++ b/catkin_tools/verbs/catkin_locate/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import sys diff --git a/catkin_tools/verbs/catkin_profile/cli.py b/catkin_tools/verbs/catkin_profile/cli.py index 79b14d6d..fa2c23b4 100644 --- a/catkin_tools/verbs/catkin_profile/cli.py +++ b/catkin_tools/verbs/catkin_profile/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from catkin_tools.context import Context from catkin_tools.metadata import get_active_profile diff --git a/tests/system/verbs/catkin_build/test_args.py b/tests/system/verbs/catkin_build/test_args.py index 27627283..74a15946 100644 --- a/tests/system/verbs/catkin_build/test_args.py +++ b/tests/system/verbs/catkin_build/test_args.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import shutil diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index ae3c7645..b2a5352d 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import re import shutil diff --git a/tests/system/verbs/catkin_build/test_bwlists.py b/tests/system/verbs/catkin_build/test_bwlists.py index ea4e276c..234c28e7 100644 --- a/tests/system/verbs/catkin_build/test_bwlists.py +++ b/tests/system/verbs/catkin_build/test_bwlists.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_context.py b/tests/system/verbs/catkin_build/test_context.py index 740d36ac..7701911f 100644 --- a/tests/system/verbs/catkin_build/test_context.py +++ b/tests/system/verbs/catkin_build/test_context.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_eclipse.py b/tests/system/verbs/catkin_build/test_eclipse.py index c0f6a3e4..f8ed1237 100644 --- a/tests/system/verbs/catkin_build/test_eclipse.py +++ b/tests/system/verbs/catkin_build/test_eclipse.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os from ....utils import in_temporary_directory diff --git a/tests/system/verbs/catkin_build/test_modify_ws.py b/tests/system/verbs/catkin_build/test_modify_ws.py index 6168972e..14097135 100644 --- a/tests/system/verbs/catkin_build/test_modify_ws.py +++ b/tests/system/verbs/catkin_build/test_modify_ws.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_unit_tests.py b/tests/system/verbs/catkin_build/test_unit_tests.py index 33e83784..3aaf15b8 100644 --- a/tests/system/verbs/catkin_build/test_unit_tests.py +++ b/tests/system/verbs/catkin_build/test_unit_tests.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import shutil diff --git a/tests/unit/test_io.py b/tests/unit/test_io.py index ff03da80..00a2df87 100644 --- a/tests/unit/test_io.py +++ b/tests/unit/test_io.py @@ -1,11 +1,6 @@ -try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue - import binascii +from queue import Queue + from catkin_tools.execution import io as io diff --git a/tests/utils.py b/tests/utils.py index ab9080f3..21781bc3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,28 +1,15 @@ -from __future__ import print_function - import functools import os import re import shutil +import subprocess import sys import tempfile - -import subprocess +from io import StringIO +from subprocess import TimeoutExpired from catkin_tools.commands.catkin import main as catkin_main -try: - # Python2 - from StringIO import StringIO -except ImportError: - # Python3 - from io import StringIO - -try: - from subprocess import TimeoutExpired -except ImportError: - class TimeoutExpired(object): - pass TESTS_DIR = os.path.dirname(__file__) MOCK_DIR = os.path.join(TESTS_DIR, 'mock_resources') diff --git a/tests/workspace_assertions.py b/tests/workspace_assertions.py index ca239b0b..3c190a91 100644 --- a/tests/workspace_assertions.py +++ b/tests/workspace_assertions.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import re import yaml From 2e78d2fca363da843de215283ce81e01e9501808 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Thu, 25 Feb 2021 16:40:59 +0100 Subject: [PATCH 03/68] Fix cleaning jobs flags with --no-make-args (#656) * fix cleaning of jobs args * add test --- catkin_tools/context.py | 4 ++++ .../system/verbs/catkin_config/test_config.py | 23 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 21bdc5b2..5dbfb57e 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -235,6 +235,10 @@ def get_metadata_recursive(profile): if workspace: key_origins[k] = profile + # When --no-make-args and no other jobs args are given, clean jobs args too + if 'make_args' in opts_vars and opts_vars['make_args'] == [] and opts_vars['jobs_args'] is None: + context_args['jobs_args'] = [] + context_args["key_origins"] = key_origins # Create the build context diff --git a/tests/system/verbs/catkin_config/test_config.py b/tests/system/verbs/catkin_config/test_config.py index 517d60c1..9811c31c 100644 --- a/tests/system/verbs/catkin_config/test_config.py +++ b/tests/system/verbs/catkin_config/test_config.py @@ -1,5 +1,6 @@ import os +from ...workspace_factory import workspace_factory from ....utils import in_temporary_directory from ....utils import assert_cmd_success @@ -33,8 +34,20 @@ def test_config_non_bare(): @in_temporary_directory def test_config_unchanged(): - cwd = os.getcwd() - os.mkdir(os.path.join(cwd, 'src')) - assert_cmd_success(['catkin', 'config', '--make-args', '-j6']) - assert_cmd_success(['catkin', 'config']) - assert_in_config('.', 'default', 'jobs_args', ['-j6']) + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', '-j6']) + assert_cmd_success(['catkin', 'config']) + assert_in_config('.', 'default', 'jobs_args', ['-j6']) + + +def test_config_no_args_flags(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', '-j6', 'test']) + assert_cmd_success(['catkin', 'config', '--cmake-args', '-DCMAKE_BUILD_TYPE=Release']) + assert_cmd_success(['catkin', 'config', '--catkin-make-args', 'run_tests']) + assert_cmd_success(['catkin', 'config', '--no-make-args', '--no-cmake-args', '--no-catkin-make-args']) + assert_in_config('.', 'default', 'jobs_args', []) + assert_in_config('.', 'default', 'make_args', []) + assert_in_config('.', 'default', 'cmake_args', []) From 3f08efd65739fe360c4fc94cbd5e8bd5f6896c89 Mon Sep 17 00:00:00 2001 From: pseyfert Date: Thu, 25 Feb 2021 22:41:26 +0100 Subject: [PATCH 04/68] escape whitespaces in `catkin config` printout (#657) * escape whitespaces in `catkin config` printout In the `catkin config` printout, the cmake-args line changes from ``` Additional CMake Args: -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-g -fno-omit-frame-pointer -O3 -DNDEBUG -DENABLE_TIMING=0 ``` to ``` Additional CMake Args: -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-g\ -fno-omit-frame-pointer\ -O3\ -DNDEBUG -DENABLE_TIMING=0 ``` thereby the printout can be copy and pasted to `catkin config --cmake-args` without changing the configuration. In the above example the last element `-DENABLE_TIMING` is a cmake option and not a gcc preprocessor definition. Before this patch, this is indistinguishable from the printout. * review suggestion * edge cases * add comment --- catkin_tools/context.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 5dbfb57e..9338ee59 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -579,13 +579,25 @@ def existence_str(path, used=True): if self.__install: install_layout = 'merged' if not self.__isolate_install else 'isolated' + def quote(argument): + # Distinguish in the printout if space separates two arguments or if we + # print an argument with a space. + # e.g. -DCMAKE_C_FLAGS="-g -O3" -DCMAKE_C_COMPILER=clang + if ' ' in argument: + if "=" in argument: + key, value = argument.split("=", 1) + if ' ' not in key: + return key + '="' + value + '"' + return '"' + argument + '"' + return argument + subs = { 'profile': self.profile, 'extend_mode': extend_mode, 'extend': extend_value, 'install_layout': install_layout, 'cmake_prefix_path': (self.cmake_prefix_path or ['Empty']), - 'cmake_args': ' '.join(self.__cmake_args or ['None']), + 'cmake_args': ' '.join([quote(a) for a in self.__cmake_args or ['None']]), 'make_args': ' '.join(self.__make_args + self.__jobs_args or ['None']), 'catkin_make_args': ', '.join(self.__catkin_make_args or ['None']), 'source_missing': existence_str(self.source_space_abs), From 471f223bf5392b3cb1717bf1638d1df94c7da88b Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 26 Feb 2021 11:36:16 +0100 Subject: [PATCH 05/68] Fixes for extending profiles (#658) * fix catkin config --copy * make extending profiles possible from the command line --- catkin_tools/context.py | 2 +- catkin_tools/verbs/catkin_profile/cli.py | 8 +++++++ .../verbs/catkin_profile/test_profile.py | 23 ++++++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 9338ee59..866aca3c 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -264,7 +264,7 @@ def save_in_file(file, key, value): files[file] = {key: value} for key, val in data.items(): - if key in context.key_origins: + if context.extends is not None and key in context.key_origins: save_in_file(context.key_origins[key], key, val) else: save_in_file(context.profile, key, val) diff --git a/catkin_tools/verbs/catkin_profile/cli.py b/catkin_tools/verbs/catkin_profile/cli.py index fa2c23b4..63dac974 100644 --- a/catkin_tools/verbs/catkin_profile/cli.py +++ b/catkin_tools/verbs/catkin_profile/cli.py @@ -62,6 +62,8 @@ def prepare_arguments(parser): help="Copy the settings from an existing profile. (default: None)") add('--copy-active', action='store_true', default=False, help="Copy the settings from the active profile.") + add('--extend', metavar='PARENT_PROFILE', type=str, + help="Extend another profile") add = parser_rename.add_argument add('current_name', type=str, @@ -146,6 +148,12 @@ def main(opts): 'based on profile @{cf}%s@|' % (opts.name, opts.copy))) else: print(clr('[profile] @{rf}A profile with this name does not exist: %s@|' % opts.copy)) + elif opts.extend: + if opts.extend in profiles: + new_ctx = Context(workspace=ctx.workspace, profile=opts.name, extends=opts.extend) + Context.save(new_ctx) + print(clr('[profile] Created a new profile named @{cf}%s@| ' + 'extending profile @{cf}%s@|' % (opts.name, opts.extend))) else: new_ctx = Context(workspace=ctx.workspace, profile=opts.name) Context.save(new_ctx) diff --git a/tests/system/verbs/catkin_profile/test_profile.py b/tests/system/verbs/catkin_profile/test_profile.py index ea08c93e..f18c9b31 100644 --- a/tests/system/verbs/catkin_profile/test_profile.py +++ b/tests/system/verbs/catkin_profile/test_profile.py @@ -1,11 +1,10 @@ import os +from ...workspace_factory import workspace_factory from ....utils import in_temporary_directory from ....utils import assert_cmd_success -from ....workspace_assertions import assert_workspace_initialized -from ....workspace_assertions import assert_warning_message -from ....workspace_assertions import assert_no_warnings +from ....workspace_assertions import assert_in_config @in_temporary_directory @@ -22,3 +21,21 @@ def test_profile_set(): assert_cmd_success(['catkin', 'init']) assert_cmd_success(['catkin', 'build']) assert_cmd_success(['catkin', 'profile', 'set', 'default']) + + +def test_profile_copy(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', 'test']) + assert_cmd_success(['catkin', 'profile', 'add', '--copy', 'default', 'mycopy']) + assert_in_config('.', 'mycopy', 'make_args', ['test']) + + +def test_profile_extend(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', 'test']) + assert_cmd_success(['catkin', 'profile', 'add', '--extend', 'default', 'myextend']) + assert_cmd_success(['catkin', 'config', '--profile', 'myextend', '--blacklist', 'mypackage']) + assert_in_config('.', 'default', 'make_args', ['test']) + assert_in_config('.', 'myextend', 'blacklist', ['mypackage']) From 78a039a274b3b8c5ffa6d03832eee946afc00975 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 26 Feb 2021 13:34:31 +0100 Subject: [PATCH 06/68] remove hack (#659) --- catkin_tools/verbs/catkin_build/cli.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index a3ccead7..d8e43447 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -62,32 +62,6 @@ from .build import determine_packages_to_be_built from .build import verify_start_with_option -# -# Begin Hack -# - -# TODO(wjwwood): remove this, once it is no longer needed. -# argparse may not support mutually exclusive groups inside other groups, see: -# http://bugs.python.org/issue10680 - -# Backup the original constructor -backup__ArgumentGroup___init__ = argparse._ArgumentGroup.__init__ - - -# Make a new constructor with the fix -def fixed__ArgumentGroup___init__(self, container, title=None, description=None, **kwargs): - backup__ArgumentGroup___init__(self, container, title, description, **kwargs) - # Make sure this line is run, maybe redundant on versions which already have it - self._mutually_exclusive_groups = container._mutually_exclusive_groups - - -# Monkey patch in the fixed constructor -argparse._ArgumentGroup.__init__ = fixed__ArgumentGroup___init__ - -# -# End Hack -# - def prepare_arguments(parser): parser.description = """\ From ea32f1c29d20610ffdd8bfb96af31df824ff5bce Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 2 Mar 2021 07:30:39 +0100 Subject: [PATCH 07/68] Fix handling of invalid package.xml files (#660) --- catkin_tools/verbs/catkin_build/build.py | 12 +++++---- catkin_tools/verbs/catkin_build/cli.py | 7 +++--- catkin_tools/verbs/catkin_clean/cli.py | 5 ++-- catkin_tools/verbs/catkin_list/cli.py | 13 ++++------ catkin_tools/verbs/catkin_locate/cli.py | 32 +++++++++++++----------- 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index 99e4eff7..bdfb43b8 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -23,8 +23,9 @@ import yaml import asyncio - try: + from catkin_pkg.package import parse_package + from catkin_pkg.package import InvalidPackage from catkin_pkg.packages import find_packages from catkin_pkg.topological_order import topological_order_packages except ImportError as e: @@ -33,8 +34,6 @@ '"catkin_pkg", and that it is up to date and on the PYTHONPATH.' % e ) -from catkin_pkg.package import parse_package - from catkin_tools.common import FakeLock, expand_glob_package from catkin_tools.common import format_time_delta from catkin_tools.common import get_cached_recursive_build_depends_in_workspace @@ -52,7 +51,6 @@ from .color import clr - BUILDSPACE_MARKER_FILE = '.catkin_tools.yaml' BUILDSPACE_IGNORE_FILE = 'CATKIN_IGNORE' DEVELSPACE_MARKER_FILE = '.catkin_tools.yaml' @@ -294,7 +292,11 @@ def build_isolated_workspace( # Get all the packages in the context source space # Suppress warnings since this is a utility function - workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + try: + workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Get packages which have not been built yet built_packages, unbuilt_pkgs = get_built_unbuilt_packages(context, workspace_packages) diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index d8e43447..41bf59b9 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -229,15 +229,16 @@ def main(opts): # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) - # Suppress warnings since this won't necessaraly find all packages + # Suppress warnings since this won't necessarily find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) - except (InvalidPackage, RuntimeError): - this_package = None + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Handle context-based package building if opts.build_this: diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index fbb73f29..2a15b2aa 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -313,8 +313,9 @@ def clean_profile(opts, profile): search_start_path=getcwd(), ws_path=ws_path, warnings=[]) - except (InvalidPackage, RuntimeError): - this_package = None + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Handle context-based package cleaning if opts.clean_this: diff --git a/catkin_tools/verbs/catkin_list/cli.py b/catkin_tools/verbs/catkin_list/cli.py index d56563b6..61a0ea31 100644 --- a/catkin_tools/verbs/catkin_list/cli.py +++ b/catkin_tools/verbs/catkin_list/cli.py @@ -75,8 +75,7 @@ def main(opts): ctx = Context.load(opts.workspace, opts.profile, load_env=False) if not ctx: - print(clr("@{rf}ERROR: Could not determine workspace.@|"), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Could not determine workspace.@|"), file=sys.stderr) if opts.directory: folders = opts.directory @@ -92,9 +91,8 @@ def main(opts): packages = find_packages(folder, warnings=warnings) ordered_packages = topological_order_packages(packages) if ordered_packages and ordered_packages[-1][0] is None: - print(clr("@{rf}ERROR: Circular dependency within packages:@| " - + ordered_packages[-1][1]), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Circular dependency within packages:@| " + + ordered_packages[-1][1]), file=sys.stderr) packages_by_name = {pkg.name: (pth, pkg) for pth, pkg in ordered_packages} if opts.depends_on or opts.rdepends_on: @@ -152,9 +150,8 @@ def main(opts): for dep in run_deps: print(clr(' @{pf}-@| %s' % dep.name)) except InvalidPackage as ex: - message = '\n'.join(ex.args) - print(clr("@{rf}Error:@| The directory %s contains an invalid package." - " See below for details:\n\n%s" % (folder, message))) + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Print out warnings if not opts.quiet: diff --git a/catkin_tools/verbs/catkin_locate/cli.py b/catkin_tools/verbs/catkin_locate/cli.py index b8b2100f..88fd60db 100644 --- a/catkin_tools/verbs/catkin_locate/cli.py +++ b/catkin_tools/verbs/catkin_locate/cli.py @@ -15,6 +15,8 @@ import os import sys +from catkin_pkg.package import InvalidPackage + from catkin_tools.common import find_enclosing_package from catkin_tools.common import getcwd @@ -131,14 +133,17 @@ def main(opts): package = None if opts.package or opts.this: if opts.this: - package = find_enclosing_package( - search_start_path=getcwd(), - ws_path=ctx.workspace, - warnings=[]) - if package is None: - print(clr("@{rf}ERROR: Passed '--this' but could not determine enclosing package. " - "Is '%s' in a package in '%s' workspace?@|" % (getcwd(), ctx.workspace)), file=sys.stderr) - sys.exit(2) + try: + package = find_enclosing_package( + search_start_path=getcwd(), + ws_path=ctx.workspace, + warnings=[]) + if package is None: + sys.exit(clr("@{rf}ERROR: Passed '--this' but could not determine enclosing package. " + "Is '%s' in a package in '%s' workspace?@|" % (getcwd(), ctx.workspace))) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) else: package = opts.package # Get the path to the given package @@ -154,12 +159,10 @@ def main(opts): if catkin_package: path = os.path.join(path, catkin_package[0]) else: - print(clr("@{rf}ERROR: Could not locate a package named '%s' in path '%s'@|" % - (package, path)), file=sys.stderr) - sys.exit(2) + sys.exit(clr("@{rf}ERROR: Could not locate a package named '%s' in path '%s'@|" % + (package, path))) except RuntimeError as e: - print(clr('@{rf}ERROR: %s@|' % str(e)), file=sys.stderr) - sys.exit(1) + sys.exit(clr('@{rf}ERROR: %s@|' % str(e))) if not opts.space and package is None: # Get the path to the workspace root @@ -167,8 +170,7 @@ def main(opts): # Check if the path exists if opts.existing_only and not os.path.exists(path): - print(clr("@{rf}ERROR: Requested path '%s' does not exist.@|" % path), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Requested path '%s' does not exist.@|" % path)) # Make the path relative if desired if opts.relative: From cca4c41c3c9be336347af0735a48b752ded9bdb3 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 2 Mar 2021 07:31:07 +0100 Subject: [PATCH 08/68] Use PYTHONASYNCIODEBUG instead of TROLLIUSDEBUG (#661) --- catkin_tools/verbs/catkin_build/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 41bf59b9..1d4490d8 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -214,7 +214,7 @@ def main(opts): # Check for develdebug mode if opts.develdebug is not None: - os.environ['TROLLIUSDEBUG'] = opts.develdebug.lower() + os.environ['PYTHONASYNCIODEBUG'] = opts.develdebug.lower() logging.basicConfig(level=opts.develdebug.upper()) # Set color options From 5e5734849f237deab303bfc545ef8f5c576b6fad Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 2 Mar 2021 11:47:27 +0100 Subject: [PATCH 09/68] Evaluate conditional dependencies in package.xml (#662) * evaluate conditional dependencies * add tests * cleanup in common.py --- catkin_tools/common.py | 60 +++++++------------ catkin_tools/config.py | 6 +- catkin_tools/execution/stages.py | 4 +- catkin_tools/resultspace.py | 3 +- catkin_tools/verbs/catkin_list/cli.py | 4 +- .../depend_condition/CMakeLists.txt | 2 + .../catkin_pkgs/depend_condition/package.xml | 11 ++++ tests/system/verbs/catkin_build/test_build.py | 16 +++++ tests/unit/test_common.py | 19 ++++++ 9 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt create mode 100644 tests/system/resources/catkin_pkgs/depend_condition/package.xml diff --git a/catkin_tools/common.py b/catkin_tools/common.py index d18e0dfe..b2909291 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -19,6 +19,7 @@ import re import sys from fnmatch import fnmatch +from itertools import chain from shlex import split as cmd_split from shlex import quote as cmd_quote @@ -29,16 +30,6 @@ color_mapper = ColorMapper() clr = color_mapper.clr -try: - string_type = basestring -except NameError: - string_type = str - -try: - unicode_type = unicode -except NameError: - unicode_type = str - class FakeLock(asyncio.locks.Lock): @@ -47,9 +38,8 @@ class FakeLock(asyncio.locks.Lock): def locked(self): return False - @asyncio.coroutine - def acquire(self): - return(True) + async def acquire(self): + return True def release(self): pass @@ -177,18 +167,15 @@ def get_cached_recursive_build_depends_in_workspace(package, workspace_packages) def get_recursive_depends_in_workspace( packages, ordered_packages, - root_include_function, include_function, exclude_function): """Computes the recursive dependencies of a package in a workspace based on include and exclude functions of each package's dependencies. - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param packages: package for which the recursive depends should be calculated + :type packages: list(:py:class:`catkin_pkg.package.Package`) :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) - :param root_include_function: a function which take a package and returns a list of root packages to include - :type root_include_function: callable :param include_function: a function which take a package and returns a list of packages to include :type include_function: callable :param exclude_function: a function which take a package and returns a list of packages to exclude @@ -207,9 +194,12 @@ def get_recursive_depends_in_workspace( } # Initialize working sets - pkgs_to_check = set([ - pkg.name for pkg in sum([root_include_function(p) for p in packages], []) - ]) + pkgs_to_check = set( + pkg.name + # Only include the packages where the condition has evaluated to true + for pkg in chain(*(filter(lambda pkg: pkg.evaluated_condition, include_function(p)) for p in packages)) + ) + checked_pkgs = set() recursive_deps = set() @@ -221,16 +211,16 @@ def get_recursive_depends_in_workspace( continue # Add this package's dependencies which should be checked _, pkg = workspace_packages_by_name[pkg_name] - pkgs_to_check.update([ + pkgs_to_check.update( d.name - for d in include_function(pkg) + for d in filter(lambda pkg: pkg.evaluated_condition, include_function(pkg)) if d.name not in checked_pkgs - ]) + ) # Add this package's dependencies which shouldn't be checked - checked_pkgs.update([ + checked_pkgs.update( d.name for d in exclude_function(pkg) - ]) + ) # Add the package itself in case we have a circular dependency checked_pkgs.add(pkg.name) # Add this package to the list of recursive dependencies for this package @@ -262,11 +252,6 @@ def get_recursive_build_depends_in_workspace(package, ordered_packages): return get_recursive_depends_in_workspace( [package], ordered_packages, - root_include_function=lambda p: ( - p.build_depends + - p.buildtool_depends + - p.test_depends + - p.run_depends), include_function=lambda p: ( p.build_depends + p.buildtool_depends + @@ -281,7 +266,7 @@ def get_recursive_run_depends_in_workspace(packages, ordered_packages): but excluding packages which are build depended on by another package in the list :param packages: packages for which the recursive depends should be calculated - :type packages: list of :py:class:`catkin_pkg.package.Package` + :type packages: list(:py:class:`catkin_pkg.package.Package`) :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -293,7 +278,6 @@ def get_recursive_run_depends_in_workspace(packages, ordered_packages): return get_recursive_depends_in_workspace( packages, ordered_packages, - root_include_function=lambda p: p.run_depends, include_function=lambda p: p.run_depends, exclude_function=lambda p: p.buildtool_depends + p.build_depends ) @@ -303,8 +287,8 @@ def get_recursive_build_dependents_in_workspace(package_name, ordered_packages): """Calculates the recursive build dependents of a package which are also in the ordered_packages - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param package_name: name of the package for which the recursive depends should be calculated + :type package_name: str :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -332,8 +316,8 @@ def get_recursive_run_dependents_in_workspace(package_name, ordered_packages): """Calculates the recursive run dependents of a package which are also in the ordered_packages - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param package_name: name of the package for which the recursive depends should be calculated + :type package_name: str :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -374,7 +358,7 @@ def log(*args, **kwargs): except UnicodeEncodeError: # Strip unicode characters from string args sanitized_args = [unicode_sanitizer.sub('?', a) - if type(a) in [str, unicode_type] + if isinstance(a, str) else a for a in args] print(*sanitized_args, **kwargs) diff --git a/catkin_tools/config.py b/catkin_tools/config.py index 31e8d1df..549f63f2 100644 --- a/catkin_tools/config.py +++ b/catkin_tools/config.py @@ -16,8 +16,6 @@ import yaml from shlex import split as cmd_split -from .common import string_type - catkin_config_path = os.path.join(os.path.expanduser('~'), '.config', 'catkin') builtin_verb_aliases_content = """\ @@ -96,11 +94,11 @@ def get_verb_aliases(path=catkin_config_path): raise RuntimeError("Invalid alias file ('{0}'), expected a dict but got a {1}" .format(full_path, type(yaml_dict))) for key, value in yaml_dict.items(): - if not isinstance(key, string_type): + if not isinstance(key, str): raise RuntimeError("Invalid alias in file ('{0}'), expected a string but got '{1}' of type {2}" .format(full_path, key, type(key))) parsed_value = None - if isinstance(value, string_type): + if isinstance(value, str): # Parse using shlex parsed_value = cmd_split(value) elif isinstance(value, list) or isinstance(value, type(None)): diff --git a/catkin_tools/execution/stages.py b/catkin_tools/execution/stages.py index e8e43bd8..641dd50c 100644 --- a/catkin_tools/execution/stages.py +++ b/catkin_tools/execution/stages.py @@ -16,8 +16,6 @@ import traceback from shlex import quote as cmd_quote -from catkin_tools.common import string_type - from .io import IOBufferLogger from .io import IOBufferProtocol @@ -83,7 +81,7 @@ def __init__( :param logger_factory: The factory to use to construct a logger (default: IOBufferProtocol.factory) """ - if not type(cmd) in [list, tuple] or not all([isinstance(s, string_type) for s in cmd]): + if not type(cmd) in [list, tuple] or not all([isinstance(s, str) for s in cmd]): raise ValueError('Command stage must be a list of strings: {}'.format(cmd)) super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource) diff --git a/catkin_tools/resultspace.py b/catkin_tools/resultspace.py index 4c014791..8efedf0f 100644 --- a/catkin_tools/resultspace.py +++ b/catkin_tools/resultspace.py @@ -20,7 +20,6 @@ from shlex import quote as cmd_quote from .common import parse_env_str -from .common import string_type DEFAULT_SHELL = '/bin/bash' @@ -139,7 +138,7 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c for ret in execute_process(command, cwd=os.getcwd(), env=base_env, emulate_tty=False, shell=True): if type(ret) is bytes: ret = ret.decode() - if isinstance(ret, string_type): + if isinstance(ret, str): lines += ret else: p = subprocess.Popen(command, cwd=os.getcwd(), env=base_env, shell=True, stdout=subprocess.PIPE) diff --git a/catkin_tools/verbs/catkin_list/cli.py b/catkin_tools/verbs/catkin_list/cli.py index 61a0ea31..12c2af36 100644 --- a/catkin_tools/verbs/catkin_list/cli.py +++ b/catkin_tools/verbs/catkin_list/cli.py @@ -137,8 +137,8 @@ def main(opts): build_deps = [p for dp, p in get_recursive_build_depends_in_workspace(pkg, ordered_packages)] run_deps = [p for dp, p in get_recursive_run_depends_in_workspace([pkg], ordered_packages)] else: - build_deps = pkg.build_depends - run_deps = pkg.run_depends + build_deps = [dep for dep in pkg.build_depends if dep.evaluated_condition] + run_deps = [dep for dep in pkg.run_depends if dep.evaluated_condition] if opts.deps or opts.rdeps: if len(build_deps) > 0: diff --git a/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt b/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt new file mode 100644 index 00000000..a96ca4af --- /dev/null +++ b/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt @@ -0,0 +1,2 @@ +cmake_minimum_required(VERSION 2.8.12) +project(build_type_condition) diff --git a/tests/system/resources/catkin_pkgs/depend_condition/package.xml b/tests/system/resources/catkin_pkgs/depend_condition/package.xml new file mode 100644 index 00000000..279bbc93 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/depend_condition/package.xml @@ -0,0 +1,11 @@ + + depend_condition + Package with conditional depend element. + 0.1.0 + BSD + todo + + catkin + ros1_pkg + ros2_pkg + diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index b2a5352d..f6186ead 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -378,3 +378,19 @@ def test_pkg_with_conditional_build_type(): # So we have to infer this skipping by checking the build directory. msg = "Package with ROS 2 conditional build_type was skipped." assert os.path.exists(os.path.join('build', 'build_type_condition')), msg + + +def test_pkg_with_conditional_depend(): + """Test building a package with a condition attribute in the depend tag""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('ros1_pkg') + wf.create_package('ros2_pkg') + wf.build() + shutil.copytree( + os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'depend_condition'), + os.path.join('src/depend_condition')) + assert catkin_success(BUILD + ['depend_condition'], env={'ROS_VERSION': '1'}) + assert os.path.exists(os.path.join('build', 'depend_condition')) + assert os.path.exists(os.path.join('build', 'ros1_pkg')) + assert not os.path.exists(os.path.join('build', 'ros2_pkg')) diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py index 03a12aac..a277ff2f 100644 --- a/tests/unit/test_common.py +++ b/tests/unit/test_common.py @@ -13,6 +13,8 @@ def __init__(self, name): self.run_depends = [] self.exec_depends = [] self.build_export_depends = [] + self.evaluated_condition = True + def test_get_recursive_build_depends_in_workspace_with_test_depend(): pkg1 = MockPackage('pkg1') @@ -28,6 +30,23 @@ def test_get_recursive_build_depends_in_workspace_with_test_depend(): assert r == ordered_packages[1:], r +def test_get_recursive_build_depends_in_workspace_with_condition(): + pkg = MockPackage('pkg') + cond_false_pkg = MockPackage('cond_false_pkg') + cond_false_pkg.evaluated_condition = False + cond_true_pkg = MockPackage('cond_true_pkg') + pkg.build_depends = [cond_true_pkg, cond_false_pkg] + + ordered_packages = [ + ('/path/to/pkg', pkg), + ('/path/to/cond_false_pkg', cond_false_pkg), + ('/path/to/cond_true_pkg', cond_true_pkg), + ] + + r = common.get_recursive_build_depends_in_workspace(pkg, ordered_packages) + assert r == ordered_packages[2:], r + + def test_get_recursive_build_depends_in_workspace_circular_run_depend(): pkg1 = MockPackage('pkg1') pkg2 = MockPackage('pkg2') From 1cfdd7d75f8a7eddf988fff1060b09368fa819b4 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Thu, 11 Mar 2021 17:31:15 +0100 Subject: [PATCH 10/68] Use standard python function to determine terminal width (#653) --- catkin_tools/common.py | 57 ++---------------------------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/catkin_tools/common.py b/catkin_tools/common.py index b2909291..9cfe55dc 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -17,6 +17,7 @@ import errno import os import re +import shutil import sys from fnmatch import fnmatch from itertools import chain @@ -372,63 +373,9 @@ def log(*args, **kwargs): unicode_error_printed = True -__warn_terminal_width_once_has_printed = False -__default_terminal_width = 80 - - -def __warn_terminal_width_once(): - global __warn_terminal_width_once_has_printed - if __warn_terminal_width_once_has_printed: - return - __warn_terminal_width_once_has_printed = True - print('NOTICE: Could not determine the width of the terminal. ' - 'A default width of {} will be used. ' - 'This warning will only be printed once.'.format(__default_terminal_width), - file=sys.stderr) - - -def terminal_width_windows(): - """Returns the estimated width of the terminal on Windows""" - from ctypes import windll, create_string_buffer - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) - - # return default size if actual size can't be determined - if not res: - __warn_terminal_width_once() - return __default_terminal_width - - import struct - (bufx, bufy, curx, cury, wattr, left, top, right, bottom, maxx, maxy)\ - = struct.unpack("hhhhHhhhhhh", csbi.raw) - width = right - left + 1 - - return width - - -def terminal_width_linux(): - """Returns the estimated width of the terminal on linux""" - from fcntl import ioctl - from termios import TIOCGWINSZ - import struct - try: - with open(os.ctermid(), "rb") as f: - height, width = struct.unpack("hh", ioctl(f.fileno(), TIOCGWINSZ, "1234")) - except (IOError, OSError, struct.error): - # return default size if actual size can't be determined - __warn_terminal_width_once() - return __default_terminal_width - return width - - def terminal_width(): """Returns the estimated width of the terminal""" - try: - return terminal_width_windows() if os.name == 'nt' else terminal_width_linux() - except ValueError: - # Failed to get the width, use the default 80 - return __default_terminal_width + return shutil.get_terminal_size().columns _ansi_escape = re.compile(r'\x1b[^m]*m') From 95d1a620e480796d8331c08ce5d3a704cd46c4f2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 17 Mar 2021 11:24:40 +0100 Subject: [PATCH 11/68] Fix Blank lines in build output (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix line breaks in CMakeIOBufferProtocol.colorize_cmake * replace all \r with \n * rstrip event data and append with \n * rstrip and append \n to interleaved lines * Fix line ending appending Co-authored-by: Mathias Lüdtke --- catkin_tools/execution/controllers.py | 16 +++++++++------- catkin_tools/jobs/commands/cmake.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/catkin_tools/execution/controllers.py b/catkin_tools/execution/controllers.py index 79ea1ac1..624f77bd 100644 --- a/catkin_tools/execution/controllers.py +++ b/catkin_tools/execution/controllers.py @@ -689,7 +689,8 @@ def run(self): if self.show_buffered_stdout: if len(event.data['interleaved']) > 0: lines = [ - line for line in event.data['interleaved'].splitlines(True) + line + '\n' + for line in event.data['interleaved'].splitlines() if (self.show_compact_io is False or len(line.strip()) > 0) ] else: @@ -699,7 +700,8 @@ def run(self): elif self.show_buffered_stderr: if len(event.data['stderr']) > 0: lines = [ - line for line in event.data['stderr'].splitlines(True) + line + '\n' + for line in event.data['stderr'].splitlines() if (self.show_compact_io is False or len(line.strip()) > 0) ] lines_target = sys.stderr @@ -719,7 +721,7 @@ def run(self): if header_title: wide_log(header_title, file=header_title_file) if len(lines) > 0: - wide_log(''.join(lines), end='\r', file=lines_target) + wide_log(''.join(lines), end='\n', file=lines_target) if footer_border: wide_log(footer_border, file=footer_border_file) if footer_title: @@ -727,11 +729,11 @@ def run(self): elif 'STDERR' == eid: if self.show_live_stderr and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\r', file=sys.stderr) + wide_log(self.format_interleaved_lines(event.data), end='\n', file=sys.stderr) elif 'STDOUT' == eid: if self.show_live_stdout and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\r') + wide_log(self.format_interleaved_lines(event.data), end='\n') elif 'MESSAGE' == eid: wide_log(event.data['msg']) @@ -757,7 +759,7 @@ def format_interleaved_lines(self, data): else: prefix = '' - template = '\r{}\r{}'.format(' ' * terminal_width(), prefix) + template = '\n{}\n{}'.format(' ' * terminal_width(), prefix) suffix = clr('@|') - return ''.join(template + line + suffix for line in data['data'].splitlines(True)) + return ''.join(template + line + '\n' + suffix for line in data['data'].splitlines()) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index 56038129..12bd40da 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -148,7 +148,7 @@ def colorize_cmake(self, line): cline = cline.format(*match.groups()) break - return cline + '\r\n' + return cline + '\n' class CMakeMakeIOBufferProtocol(IOBufferProtocol): From fe1bec69416f45a310aa43fe1308505db328c798 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 19 Mar 2021 15:14:51 +0100 Subject: [PATCH 12/68] Fix shell completion install locations (#652) --- ...tkin_tools-completion.bash => catkin.bash} | 0 setup.py | 53 +++++-------------- stdeb.cfg | 1 + 3 files changed, 15 insertions(+), 39 deletions(-) rename completion/{catkin_tools-completion.bash => catkin.bash} (100%) diff --git a/completion/catkin_tools-completion.bash b/completion/catkin.bash similarity index 100% rename from completion/catkin_tools-completion.bash rename to completion/catkin.bash diff --git a/setup.py b/setup.py index 6d0bcdee..98eeb78c 100644 --- a/setup.py +++ b/setup.py @@ -34,38 +34,33 @@ for x in osx_notification_resources] -def _resolve_prefix(prefix, type): +def _resolve_prefix(type): osx_system_prefix = '/System/Library/Frameworks/Python.framework/Versions' - if type == 'man': - if prefix == '/usr': - return '/usr/share' + if type == 'bash_comp': if sys.prefix.startswith(osx_system_prefix): - return '/usr/share' - elif type == 'bash_comp': - if prefix == '/usr': - return '/' - if sys.prefix.startswith(osx_system_prefix): - return '/' + return '/usr' elif type == 'zsh_comp': if sys.prefix.startswith(osx_system_prefix): return '/usr/local' else: raise ValueError('not supported type') - return prefix + return '' -def get_data_files(prefix): +def get_data_files(): data_files = [] # Bash completion - bash_comp_dest = os.path.join(_resolve_prefix(prefix, 'bash_comp'), - 'etc/bash_completion.d') - data_files.append((bash_comp_dest, - ['completion/catkin_tools-completion.bash'])) + bash_comp_dest = os.path.join(_resolve_prefix('bash_comp'), + 'share/bash-completion/completions') + data_files.append((bash_comp_dest, ['completion/catkin.bash'])) # Zsh completion - zsh_comp_dest = os.path.join(_resolve_prefix(prefix, 'zsh_comp'), - 'share/zsh/site-functions') + if 'DEB_BUILD' in os.environ: + dirname = 'share/zsh/vendor-completions' + else: + dirname = 'share/zsh/site-functions' + zsh_comp_dest = os.path.join(_resolve_prefix('zsh_comp'), dirname) data_files.append((zsh_comp_dest, ['completion/_catkin'])) return data_files @@ -81,26 +76,6 @@ def run(self): log.info("changing permissions of %s to %o" % (file, mode)) os.chmod(file, mode) - # Provide information about bash completion after default install. - if (sys.platform.startswith("linux") and - self.install_data == "/usr/local"): - log.info(""" ----------------------------------------------------------------- -To enable tab completion, add the following to your '~/.bashrc': - - source {0} - ----------------------------------------------------------------- -""".format(os.path.join(self.install_data, - 'etc/bash_completion.d', - 'catkin_tools-completion.bash'))) - -from distutils.core import setup -from distutils.dist import Distribution -dist = Distribution() -dist.parse_config_files() -dist.parse_command_line() -prefix = dist.get_option_dict('install').get('prefix',("default", sys.prefix))[1] setup( name='catkin_tools', @@ -117,7 +92,7 @@ def run(self): 'docs/examples', ] + osx_notification_resources }, - data_files=get_data_files(prefix), + data_files=get_data_files(), install_requires=install_requires, author='William Woodall', author_email='william@osrfoundation.org', diff --git a/stdeb.cfg b/stdeb.cfg index 23de0f22..17ef2fce 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -5,3 +5,4 @@ Conflicts: python3-catkin-tools Conflicts3: python-catkin-tools Suite: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster X-Python3-Version: >= 3.5 +Setup-Env-Vars: DEB_BUILD=1 From 5a133fa8217060763ed834892961c117ee880b71 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 19 Mar 2021 16:46:33 +0100 Subject: [PATCH 13/68] Remove superfluous argument to sys.exit --- catkin_tools/verbs/catkin_list/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catkin_tools/verbs/catkin_list/cli.py b/catkin_tools/verbs/catkin_list/cli.py index 12c2af36..b20e5301 100644 --- a/catkin_tools/verbs/catkin_list/cli.py +++ b/catkin_tools/verbs/catkin_list/cli.py @@ -75,7 +75,7 @@ def main(opts): ctx = Context.load(opts.workspace, opts.profile, load_env=False) if not ctx: - sys.exit(clr("@{rf}ERROR: Could not determine workspace.@|"), file=sys.stderr) + sys.exit(clr("@{rf}ERROR: Could not determine workspace.@|")) if opts.directory: folders = opts.directory @@ -92,7 +92,7 @@ def main(opts): ordered_packages = topological_order_packages(packages) if ordered_packages and ordered_packages[-1][0] is None: sys.exit(clr("@{rf}ERROR: Circular dependency within packages:@| " - + ordered_packages[-1][1]), file=sys.stderr) + + ordered_packages[-1][1])) packages_by_name = {pkg.name: (pth, pkg) for pth, pkg in ordered_packages} if opts.depends_on or opts.rdepends_on: From 50cc55d341d94b7d4ab3c54baa040a4b5f0711d5 Mon Sep 17 00:00:00 2001 From: mobangjack Date: Tue, 23 Mar 2021 17:04:34 +0800 Subject: [PATCH 14/68] Use loadavg over the last 1 minute (#668) Co-authored-by: jack.mo --- catkin_tools/execution/job_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catkin_tools/execution/job_server.py b/catkin_tools/execution/job_server.py index 03fd1bc9..c3ebefc6 100644 --- a/catkin_tools/execution/job_server.py +++ b/catkin_tools/execution/job_server.py @@ -219,7 +219,7 @@ def _check_load(cls): if cls._max_load is not None: try: load = os.getloadavg() - if load[1] < cls._max_load: + if load[0] < cls._max_load: cls._load_ok = True else: cls._load_ok = False From 5860b7ff86191b722feb2dd666224ce9330e00f2 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 23 Mar 2021 11:23:42 +0100 Subject: [PATCH 15/68] Support building from a symlinked workspace (#669) --- catkin_tools/context.py | 4 ++-- tests/system/verbs/catkin_build/test_build.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 866aca3c..878b996d 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -90,7 +90,7 @@ def space_setter(self, value): if self.__locked: raise RuntimeError("Setting of context members is not allowed while locked.") setattr(self, '__%s_space' % space, value) - setattr(self, '__%s_space_abs' % space, os.path.join(self.__workspace, value)) + setattr(self, '__%s_space_abs' % space, os.path.realpath(os.path.join(self.__workspace, value))) def space_exists(self): """ @@ -651,7 +651,7 @@ def workspace(self, value): # Validate Workspace if not os.path.exists(value): raise ValueError("Workspace path '{0}' does not exist.".format(value)) - self.__workspace = os.path.abspath(value) + self.__workspace = os.path.realpath(os.path.abspath(value)) @property def extend_path(self): diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index f6186ead..67cffe57 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -4,7 +4,7 @@ from ...workspace_factory import workspace_factory -from ....utils import in_temporary_directory +from ....utils import in_temporary_directory, temporary_directory from ....utils import assert_cmd_success from ....utils import assert_cmd_failure from ....utils import assert_files_exist @@ -394,3 +394,15 @@ def test_pkg_with_conditional_depend(): assert os.path.exists(os.path.join('build', 'depend_condition')) assert os.path.exists(os.path.join('build', 'ros1_pkg')) assert not os.path.exists(os.path.join('build', 'ros2_pkg')) + + +def test_symlinked_workspace(): + """Test building from a symlinked workspace""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('pkg') + wf.build() + assert catkin_success(BUILD) + with temporary_directory() as t: + os.symlink(wf.workspace, os.path.join(t, 'ws')) + assert catkin_success(BUILD + ['-w', os.path.join(t, 'ws')]) From a3a9c97b9c25bc802440307f8199556b7498cbc6 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Wed, 24 Mar 2021 17:56:43 +0100 Subject: [PATCH 16/68] Fix `catkin create pkg` without license parameter (#671) --- catkin_tools/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 878b996d..40cbb5bc 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -388,7 +388,7 @@ def __init__( # Handle default authors/maintainers self.authors = authors or [] self.maintainers = maintainers or [] - self.licenses = licenses or 'TODO' + self.licenses = licenses or ['TODO'] # Handle build options self.devel_layout = devel_layout if devel_layout else 'linked' From fc555e7602c6fd4dd546716d746411a4de74a165 Mon Sep 17 00:00:00 2001 From: zig-for Date: Thu, 8 Apr 2021 10:42:34 -0700 Subject: [PATCH 17/68] Fix catkin clean --all-profiles when not at workspace root (#673) --- catkin_tools/verbs/catkin_clean/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index 2a15b2aa..a1efde62 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -379,7 +379,7 @@ def main(opts): # Check for all profiles option if opts.all_profiles: - profiles = get_profile_names(opts.workspace or os.getcwd()) + profiles = get_profile_names(opts.workspace or find_enclosing_workspace(getcwd())) else: profiles = [opts.profile] From 3d06964e7cb7a6c8ae4ba77c0fffd5a3854e7a05 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 19 Apr 2021 14:16:00 +0200 Subject: [PATCH 18/68] Fix placeholders for cmake warning and error coloring (#678) --- catkin_tools/jobs/commands/cmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index 12bd40da..f5cae3d9 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -70,8 +70,8 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, source_pat (r'CMake Error at (.+):(.+)', '@{rf}@!CMake Error@| at {}:{}', self.abspath), (r'CMake Warning at (.+):(.+)', '@{yf}@!CMake Warning@| at {}:{}', self.abspath), (r'CMake Warning (dev) at (.+):(.+)', '@{yf}@!CMake Warning (dev)@| at {}:{}', self.abspath), - (r'(?i)(warning.*)', '@(yf){}@|', None), - (r'(?i)ERROR:(.*)', '@!@(rf)ERROR:@|{}@|', None), + (r'(?i)(warning.*)', '@{yf}{}@|', None), + (r'(?i)ERROR:(.*)', '@!@{rf}ERROR:@|{}@|', None), (r'Call Stack \(most recent call first\):(.*)', '@{cf}Call Stack (most recent call first):@|{}', None), ] From 11da5e095b0e37568e2acd5827904d26f0b785fa Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Jul 2021 20:13:44 +0200 Subject: [PATCH 19/68] Fix stdeb for python 3 (#648) * fix stdeb for python 3 * do not use catkin-pkg-modules in setup.py * only require catkin-pkg when not building deb * 0.6.0 Signed-off-by: William Woodall * changelog for 0.7.0 * make ros_release_python skip python2 release * increase version to 0.7.1 Co-authored-by: William Woodall --- CHANGELOG.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 8 ++++++-- stdeb.cfg | 5 ++--- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c186b896..3940ca55 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,53 @@ Changelog for package catkin_tools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.7.1 (2021-07-17) +------------------ +* Fixes in the build system requiring a version increase + +0.7.0 (2021-03-01) +------------------ +* Fix placeholders for cmake warning and error coloring (#678) +* Fix catkin clean --all-profiles when not at workspace root (#673) +* Fix `catkin create pkg` without license parameter (#671) +* Support building from a symlinked workspace (#669) +* Use loadavg over the last 1 minute (#668) +* Fix shell completion install locations (#652) +* Fix blank lines in build output (#666) +* Use standard python function to determine terminal width (#653) +* Fix handling of invalid package.xml files (#660) +* Fixes for extending profiles (#658) +* escape whitespaces in `catkin config` printout (#657) +* updates to zsh completion (#609) +* Ignore catkin_tools_prebuild package in build space (#650) +* fix 'catkin locate' for symlinked pkgs inside workspace (#624) +* Report circular dependencies detected by topological_order_packages() (#617) +* Add `--this` option to `clean` verb (#623) +* In catkin build, preserve original job list topological ordering (#626) +* Fail build if jobs were abandoned (#644) +* Fix installation of new cmake files (#615) +* Abort with error message on circular dependency. (#641) +* Changed yield from lock to await for Python 3.9 compat (#635) +* Remove older py35+xenial config and add py39+focal (#637) +* Install python2 before travis runs on Focal. (#639) +* Bump cmake min ver to 2.8.12 (#634) +* Fix byte decoding for python 3 (Issue #625) (#627) +* Cleanup of jobs flag parsing (#610, #656, #655) +* Fix get_python_install_dir for Python 2/3 (#601) +* Minor cleanup: + - import cleanup (#651) + - remove hack (#659) + - Add missing space in devel layout error message + - fix TypeError on executing catkin env (#649) + - Put a space between 'workspace' and 'and' (#619) + - Remove redundant 'configuration' in mechanics.rst (#646) + - Use PYTHONASYNCIODEBUG instead of TROLLIUSDEBUG (#661) +* Contributors: Akash Patel, Guglielmo Gemignani, Ivor Wanders, Kevin Jaget, Lucas Walter, Mathias Lüdtke, Matthijs van der Burgh, Mike Purvis, Robert Haschke, Simon Schmeisser, Tim Rakowski, Timon Engelke, Vojtech Spurny, ckurtz22, mobangjack, pseyfert, xiaxi, zig-for + +0.6.0 (2020-06-03) +------------------ +* This release restores the 0.4.5 state due to an accident with the 0.5.0 release where we pushed it to Python2 users which it doesn't support. + 0.5.0 (2020-06-02) ------------------ * Revert "jobs: Fixing environment required to run catkin test targets on pre-indigo catkin" (`#600 `_) diff --git a/setup.py b/setup.py index 98eeb78c..f6128f59 100644 --- a/setup.py +++ b/setup.py @@ -11,12 +11,16 @@ # Setup installation dependencies install_requires = [ - 'catkin-pkg > 0.2.9', 'setuptools', 'PyYAML', 'osrf-pycommon > 0.1.1', ] +# When building the deb, do not require catkin_pkg +if 'DEB_BUILD' not in os.environ: + install_requires += ['catkin_pkg >= 0.3.0'] + + # Figure out the resources that need to be installed this_dir = os.path.abspath(os.path.dirname(__file__)) osx_resources_path = os.path.join( @@ -79,7 +83,7 @@ def run(self): setup( name='catkin_tools', - version='0.5.0', + version='0.7.1', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={ diff --git a/stdeb.cfg b/stdeb.cfg index 17ef2fce..8dc9259f 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,8 +1,7 @@ [DEFAULT] -Depends: python-argparse, python-setuptools, python-catkin-pkg (> 0.2.9), python-yaml -Depends3: python3-setuptools, python3-catkin-pkg (> 0.2.9), python3-yaml -Conflicts: python3-catkin-tools +Depends3: python3-setuptools, python3-catkin-pkg-modules, python3-yaml, python3-osrf-pycommon Conflicts3: python-catkin-tools Suite: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster X-Python3-Version: >= 3.5 Setup-Env-Vars: DEB_BUILD=1 +No-Python2: From f5a4a0ade82eb02891727b390c59cd1b6eef6f85 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Jul 2021 20:15:33 +0200 Subject: [PATCH 20/68] Regenerate setup files when the install space was cleaned (#682) --- catkin_tools/verbs/catkin_build/build.py | 2 ++ tests/system/verbs/catkin_build/test_build.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index bdfb43b8..acfbb96f 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -391,6 +391,8 @@ def build_isolated_workspace( # Generate prebuild and prebuild clean jobs, if necessary prebuild_jobs = {} setup_util_present = os.path.exists(os.path.join(context.devel_space_abs, '_setup_util.py')) + if context.install: + setup_util_present &= os.path.exists(os.path.join(context.install_space_abs, '_setup_util.py')) catkin_present = 'catkin' in (packages_to_be_built_names + packages_to_be_built_deps_names) catkin_built = 'catkin' in built_packages prebuild_built = 'catkin_tools_prebuild' in built_packages diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index 67cffe57..71fb95dd 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -406,3 +406,28 @@ def test_symlinked_workspace(): with temporary_directory() as t: os.symlink(wf.workspace, os.path.join(t, 'ws')) assert catkin_success(BUILD + ['-w', os.path.join(t, 'ws')]) + + +def test_generate_setup_util(): + """Test generation of setup utilities in a linked devel space""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('pkg') + wf.build() + # Test that the files are generated in a clean workspace + assert catkin_success(['config', '--install']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) + + # Test that the files are regenerated after clean + assert catkin_success(['clean', '--yes']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) + + # Test that the files are regenerated after cleaning the install space + assert catkin_success(['clean', '--yes', '--install']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) From 404ed5f465a48dd08393c9a3b85ad2018c6eba25 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 13:24:49 +0200 Subject: [PATCH 21/68] Switch from Travis CI to GitHub actions (#684) * Add GitHub actions configuration * Remove travis configuration * Update CI badge in README --- .github/workflows/workflow.yml | 51 ++++++++++++++++++++++++++++++++++ .travis.before_install.bash | 15 ---------- .travis.before_script.bash | 7 ----- .travis.yml | 51 ---------------------------------- README.md | 2 +- 5 files changed, 52 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/workflow.yml delete mode 100755 .travis.before_install.bash delete mode 100755 .travis.before_script.bash delete mode 100644 .travis.yml diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 00000000..10752e06 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,51 @@ +name: build + +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + versions: + - dist: ubuntu-18.04 + python: 3.6 + catkin: indigo-devel + - dist: ubuntu-18.04 + python: 3.7 + catkin: indigo-devel + - dist: ubuntu-18.04 + python: 3.8 + catkin: indigo-devel + - dist: ubuntu-20.04 + python: 3.9 + catkin: indigo-devel + + runs-on: ${{ matrix.versions.dist }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.versions.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.versions.python }} + - name: Install package and dependencies + run: | + python -m pip install --upgrade pip + pip install . + pip install --upgrade empy sphinx_rtd_theme sphinxcontrib-spelling nose coverage flake8 mock + - name: Set up catkin + run: | + git clone https://github.com/ros/catkin.git -b ${{ matrix.versions.catkin }} /tmp/catkin_source + mkdir /tmp/catkin_source/build + pushd /tmp/catkin_source/build + cmake .. && make + popd + - name: Test catkin_tools + run: | + source /tmp/catkin_source/build/devel/setup.bash + nosetests -s + - name: Build documentation + run: | + pushd docs + make html + sphinx-build -b spelling . build -t use_spelling + popd diff --git a/.travis.before_install.bash b/.travis.before_install.bash deleted file mode 100755 index e0c39d22..00000000 --- a/.travis.before_install.bash +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -if [ "$TRAVIS_OS_NAME" == "linux" ]; then - echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main universe restricted" > /etc/apt/sources.list - echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main universe restricted" >> /etc/apt/sources.list - sudo apt update - sudo apt install enchant -y - sudo apt install python2 -y || true -elif [ "$TRAVIS_OS_NAME" == "osx" ]; then - brew upgrade python - $PYTHON -m pip install virtualenv - $PYTHON -m virtualenv -p $PYTHON venv - brew install enchant - source venv/bin/activate -fi diff --git a/.travis.before_script.bash b/.travis.before_script.bash deleted file mode 100755 index 7f217730..00000000 --- a/.travis.before_script.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -if [ "$TRAVIS_OS_NAME" == "linux" ]; then - sudo apt-get install cmake build-essential -elif [ "$TRAVIS_OS_NAME" == "osx" ]; then - echo "No OS X-specific before_script steps." -fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9957e88a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Travis auto-virtualenv isn't supported on OS X -language: generic -matrix: - include: - - dist: bionic - language: python - python: "3.6" - os: linux - env: PYTHON=/usr/bin/python3.6 CATKIN_VERSION=indigo-devel - - dist: bionic - language: python - python: "3.7" - os: linux - env: PYTHON=/usr/bin/python3.7 CATKIN_VERSION=indigo-devel - - dist: bionic - language: python - python: "3.8" - os: linux - env: PYTHON=/usr/bin/python3.8 CATKIN_VERSION=indigo-devel - - dist: focal - language: python - python: "3.9" - os: linux - env: PYTHON=/usr/bin/python3.9 CATKIN_VERSION=indigo-devel -before_install: - # Install catkin_tools dependencies - - source .travis.before_install.bash - - pip install setuptools argparse catkin-pkg PyYAML psutil osrf_pycommon pyenchant sphinxcontrib-spelling -install: - # Install catkin_tools - - python setup.py install -before_script: - # Install catkin_tools test harness dependencies - - ./.travis.before_script.bash - - pip install empy sphinx_rtd_theme nose coverage flake8 mock --upgrade - - git clone https://github.com/ros/catkin.git /tmp/catkin_source -b $CATKIN_VERSION --depth 1 - - mkdir /tmp/catkin_source/build - - pushd /tmp/catkin_source/build - - cmake .. && make - - source devel/setup.bash - - popd -script: - # Run catkin_tools test harness - - python setup.py nosetests -s - # Build documentation - - pushd docs - - make html - - sphinx-build -b spelling . build -t use_spelling - - popd -notifications: - email: false diff --git a/README.md b/README.md index fc14a36f..f7137ac7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# catkin_tools [![Build Status](https://travis-ci.org/catkin/catkin_tools.svg?branch=master)](https://travis-ci.org/catkin/catkin_tools) +# catkin_tools [![Build Status](https://github.com/catkin/catkin_tools/actions/workflows/workflow.yml/badge.svg)](https://github.com/catkin/catkin_tools/actions/workflows/workflow.yml) Command line tools for working with [catkin](https://github.com/ros/catkin) From cffdf51cb410fb089b57e6b664c3d876706d7f59 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 13:34:12 +0200 Subject: [PATCH 22/68] Make it possible to clean individual packages with isolated layout (#683) * Make it possible to clean individual packages with isolated layout * update changelog --- CHANGELOG.rst | 5 +++++ catkin_tools/verbs/catkin_clean/cli.py | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3940ca55..eb4c2288 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog for package catkin_tools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Upcoming +-------- + +* Add possibility to clean individual package with isolated devel space (#683) + 0.7.1 (2021-07-17) ------------------ * Fixes in the build system requiring a version increase diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index a1efde62..9432aaea 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -275,7 +275,7 @@ def clean_profile(opts, profile): safe_rmtree(ctx.log_space_abs, ctx.workspace, opts.force) # Find orphaned packages - if ctx.link_devel and not any([opts.build, opts.devel]): + if ctx.link_devel or ctx.isolate_devel and not any([opts.build, opts.devel]): if opts.orphans: if os.path.exists(ctx.build_space_abs): log("[clean] Determining orphaned packages...") @@ -338,9 +338,9 @@ def clean_profile(opts, profile): return False elif opts.orphans or len(opts.packages) > 0 or opts.clean_this: - log("[clean] Error: Individual packages can only be cleaned from " - "workspaces with symbolically-linked develspaces (`catkin " - "config --link-devel`).") + log("[clean] Error: Individual packages cannot be cleaned from " + "workspaces with merged develspaces, use a symbolically-linked " + "or isolated develspace instead.") except: # noqa: E722 # Silencing E722 here since we immediately re-raise the exception. From a32d2b388f28c160ef68189f6ce026ea26956570 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 13:37:47 +0200 Subject: [PATCH 23/68] Avoid that catkin build --this can create a new workspace (#685) --- catkin_tools/verbs/catkin_build/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 1d4490d8..23420d18 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -260,7 +260,14 @@ def main(opts): sys.exit(clr("[build] @!@{rf}Error:@| With --no-deps, you must specify packages to build.")) # Load the context - ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + if opts.build_this or opts.start_with_this: + ctx = Context.load(opts.workspace, opts.profile, opts, append=True, strict=True) + else: + ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + + # Handle no workspace + if ctx is None: + sys.exit(clr("[build] @!@{rf}Error:@| The current folder is not part of a catkin workspace.")) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args( From 132d8abb5c232d73021cb765fadc828a707ea740 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 13:38:32 +0200 Subject: [PATCH 24/68] Update changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eb4c2288..a8f83b6a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ Upcoming -------- * Add possibility to clean individual package with isolated devel space (#683) +* Fix regeneration of setup file when the install space was cleaned (#682) +* Fix workspace generation with catkin build --this and --start-with-this (#685) 0.7.1 (2021-07-17) ------------------ From 7ed2ee2affc7ef494c9df817aee0a2712d80b312 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 16 Apr 2021 23:53:28 +0200 Subject: [PATCH 25/68] catkin test: initial commit --- catkin_tools/config.py | 2 - catkin_tools/jobs/test.py | 62 +++++++++++ catkin_tools/jobs/utils.py | 4 +- catkin_tools/verbs/catkin_test/__init__.py | 22 ++++ catkin_tools/verbs/catkin_test/cli.py | 44 ++++++++ catkin_tools/verbs/catkin_test/test.py | 118 +++++++++++++++++++++ setup.py | 1 + 7 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 catkin_tools/jobs/test.py create mode 100644 catkin_tools/verbs/catkin_test/__init__.py create mode 100644 catkin_tools/verbs/catkin_test/cli.py create mode 100644 catkin_tools/verbs/catkin_test/test.py diff --git a/catkin_tools/config.py b/catkin_tools/config.py index 549f63f2..7917a2e4 100644 --- a/catkin_tools/config.py +++ b/catkin_tools/config.py @@ -31,8 +31,6 @@ ls: list install: config --install p: create pkg -test: build --verbose --make-args test -- -run_tests: build --verbose --catkin-make-args run_tests -- """ diff --git a/catkin_tools/jobs/test.py b/catkin_tools/jobs/test.py new file mode 100644 index 00000000..24a45e55 --- /dev/null +++ b/catkin_tools/jobs/test.py @@ -0,0 +1,62 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from catkin_tools.execution.io import IOBufferProtocol +from catkin_tools.execution.jobs import Job +from catkin_tools.execution.stages import FunctionStage, CommandStage +from catkin_tools.jobs.commands.cmake import CMakeMakeIOBufferProtocol +from catkin_tools.jobs.commands.make import MAKE_EXEC +from catkin_tools.jobs.utils import loadenv + + +def create_test_job( + context, + package, + package_path, +): + """Generate a job that tests a package""" + + # Package build space path + build_space = context.package_build_space(package) + # Environment dictionary for the job, which will be built + # up by the executions in the loadenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Load environment for job + stages.append(FunctionStage( + 'loadenv', + loadenv, + locked_resource=None, + job_env=job_env, + package=package, + context=context, + verbose=False, + )) + + # Make command + stages.append(CommandStage( + 'make', + [MAKE_EXEC, 'run_tests'], + cwd=build_space, + logger_factory=IOBufferProtocol.factory + )) + + return Job( + jid=package.name, + deps=[], + env=job_env, + stages=stages, + ) \ No newline at end of file diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index fbd7756d..37ddc69b 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -101,7 +101,7 @@ def merge_envs(job_env, overlay_envs): job_env[key] = os.pathsep.join(reversed(new_values_list)) -def loadenv(logger, event_queue, job_env, package, context): +def loadenv(logger, event_queue, job_env, package, context, verbose=True): # Get the paths to the env loaders env_loader_paths = get_env_loaders(package, context) # If DESTDIR is set, set _CATKIN_SETUP_DIR as well @@ -110,7 +110,7 @@ def loadenv(logger, event_queue, job_env, package, context): envs = [] for env_loader_path in env_loader_paths: - if logger: + if logger and verbose: logger.out('Loading environment from: {}'.format(env_loader_path)) envs.append(get_resultspace_environment( os.path.split(env_loader_path)[0], diff --git a/catkin_tools/verbs/catkin_test/__init__.py b/catkin_tools/verbs/catkin_test/__init__.py new file mode 100644 index 00000000..9094837b --- /dev/null +++ b/catkin_tools/verbs/catkin_test/__init__.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cli import main +from .cli import prepare_arguments + +# This describes this command to the loader +description = dict( + verb='test', + description="Tests a catkin workspace.", + main=main, + prepare_arguments=prepare_arguments, +) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py new file mode 100644 index 00000000..88de2645 --- /dev/null +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from catkin_tools.argument_parsing import add_context_args +from catkin_tools.context import Context +from catkin_tools.verbs.catkin_test.test import test_workspace + + +def prepare_arguments(parser): + parser.description = """\ +Test one or more packages in a catkin workspace. +This invokes `make run_tests` or `make test` for either all or the specified +packages in a catkin workspace.\ +""" + + # Workspace / profile args + add_context_args(parser) + # Sub-commands + # What packages to test + pkg_group = parser.add_argument_group('Packages', 'Control which packages get tested.') + add = pkg_group.add_argument + add('packages', metavar='PKGNAME', nargs='*', + help='Workspace packages to test. If no packages are given, then all the packages are tested.') + + return parser + + +def main(opts): + ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + + return test_workspace( + ctx, + packages=opts.packages, + ) \ No newline at end of file diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py new file mode 100644 index 00000000..fa3ca0bc --- /dev/null +++ b/catkin_tools/verbs/catkin_test/test.py @@ -0,0 +1,118 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import traceback +from queue import Queue + +from catkin_pkg.package import InvalidPackage +from catkin_pkg.packages import find_packages +from catkin_pkg.topological_order import topological_order_packages + +from catkin_tools.common import clr, wide_log +from catkin_tools.execution import job_server +from catkin_tools.execution.controllers import ConsoleStatusController +from catkin_tools.execution.executor import run_until_complete, execute_jobs +from catkin_tools.jobs.test import create_test_job + + +def test_workspace( + context, + packages=None, +): + """Tests a catkin workspace + + :param context: context in which to test the catkin workspace + :type context: :py:class:`catkin_tools.context.Context` + :param packages: list of packages to test + :type packages: list + """ + # Get all the packages in the context source space + # Suppress warnings since this is a utility function + try: + workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) + + ordered_packages = topological_order_packages(workspace_packages) + n_jobs = 1 + + # Construct jobs + jobs = [] + for pkg_path, pkg in ordered_packages: + if pkg.name not in packages: + continue + + # Determine the job parameters + test_job_kwargs = dict( + context=context, + package=pkg, + package_path=pkg_path) + + jobs.append(create_test_job(**test_job_kwargs)) + + # Queue for communicating status + event_queue = Queue() + + # Initialize job server + job_server.initialize( + max_jobs=1, + max_load=None, + ) + + try: + # Spin up status output thread + status_thread = ConsoleStatusController( + 'test', + ['package', 'packages'], + jobs, + n_jobs, + [pkg.name for _, pkg in context.packages], + [p for p in context.whitelist], + [p for p in context.blacklist], + event_queue, + show_live_stdout=True, + show_live_stderr=True, + ) + + status_thread.start() + + locks = {} + + # Block while running N jobs asynchronously + try: + all_succeeded = run_until_complete(execute_jobs( + 'build', + jobs, + locks, + event_queue, + context.log_space_abs, + max_toplevel_jobs=n_jobs, + continue_without_deps=False)) + except Exception: + status_thread.keep_running = False + all_succeeded = False + status_thread.join(1.0) + wide_log(str(traceback.format_exc())) + + status_thread.join(1.0) + + if all_succeeded: + return 0 + else: + return 1 + + except KeyboardInterrupt: + wide_log("[test] Interrupted by user!") + event_queue.put(None) + + return 130 diff --git a/setup.py b/setup.py index f6128f59..68397817 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ def run(self): 'list = catkin_tools.verbs.catkin_list:description', 'locate = catkin_tools.verbs.catkin_locate:description', 'profile = catkin_tools.verbs.catkin_profile:description', + 'test = catkin_tools.verbs.catkin_test:description', ], 'catkin_tools.jobs': [ 'catkin = catkin_tools.jobs.catkin:description', From 53649cda6423d68887332d306a92022ed2e428c4 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 16 Apr 2021 23:54:00 +0200 Subject: [PATCH 26/68] catkin test: fix output of live stdout/stderr --- catkin_tools/execution/controllers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/catkin_tools/execution/controllers.py b/catkin_tools/execution/controllers.py index 624f77bd..895b2b81 100644 --- a/catkin_tools/execution/controllers.py +++ b/catkin_tools/execution/controllers.py @@ -729,11 +729,11 @@ def run(self): elif 'STDERR' == eid: if self.show_live_stderr and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\n', file=sys.stderr) + wide_log(self.format_interleaved_lines(event.data), file=sys.stderr) elif 'STDOUT' == eid: if self.show_live_stdout and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\n') + wide_log(self.format_interleaved_lines(event.data)) elif 'MESSAGE' == eid: wide_log(event.data['msg']) @@ -759,7 +759,8 @@ def format_interleaved_lines(self, data): else: prefix = '' - template = '\n{}\n{}'.format(' ' * terminal_width(), prefix) + # This is used to clear the status bar that is printed in the current line + clear_line = '\r{}\r'.format(' ' * terminal_width()) suffix = clr('@|') - - return ''.join(template + line + '\n' + suffix for line in data['data'].splitlines()) + lines = data['data'].splitlines() + return clear_line + '\n'.join(prefix + line + suffix for line in lines) From ae7d3111b1de29a7617f6de08d9ebc35fd551395 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 11:54:46 +0200 Subject: [PATCH 27/68] catkin test: split test jobs in cmake and catkin test job generators --- catkin_tools/jobs/catkin.py | 49 +++++++++++++++++++- catkin_tools/jobs/cmake.py | 45 ++++++++++++++++++- catkin_tools/jobs/test.py | 62 -------------------------- catkin_tools/verbs/catkin_test/test.py | 18 +++++++- 4 files changed, 108 insertions(+), 66 deletions(-) delete mode 100644 catkin_tools/jobs/test.py diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index b93ef780..ce62b8b8 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -577,11 +577,58 @@ def create_catkin_clean_job( stages=stages) +def create_catkin_test_job( + context, + package, + package_path, +): + """Generate a job that tests a package""" + + # Package source space path + pkg_dir = os.path.join(context.source_space_abs, package_path) + # Package build space path + build_space = context.package_build_space(package) + # Environment dictionary for the job, which will be built + # up by the executions in the loadenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Load environment for job + stages.append(FunctionStage( + 'loadenv', + loadenv, + locked_resource=None, + job_env=job_env, + package=package, + context=context, + verbose=False, + )) + + # Make command + stages.append(CommandStage( + 'make', + [MAKE_EXEC, 'run_tests'], + cwd=build_space, + #logger_factory=CMakeIOBufferProtocol.factory_factory(pkg_dir) + logger_factory=CMakeMakeIOBufferProtocol.factory, + )) + + return Job( + jid=package.name, + deps=[], + env=job_env, + stages=stages, + ) + + description = dict( build_type='catkin', description="Builds a catkin package.", create_build_job=create_catkin_build_job, - create_clean_job=create_catkin_clean_job + create_clean_job=create_catkin_clean_job, + create_test_job=create_catkin_test_job, ) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 70c360df..4892b8a6 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -405,11 +405,54 @@ def create_cmake_clean_job( stages=stages) +def create_cmake_test_job( + context, + package, + package_path, +): + """Generate a job to test a cmake package""" + # Package build space path + build_space = context.package_build_space(package) + # Environment dictionary for the job, which will be built + # up by the executions in the loadenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Load environment for job + stages.append(FunctionStage( + 'loadenv', + loadenv, + locked_resource=None, + job_env=job_env, + package=package, + context=context, + verbose=False, + )) + + # Make command + stages.append(CommandStage( + 'make', + [MAKE_EXEC, 'test'], + cwd=build_space, + logger_factory=CMakeMakeIOBufferProtocol.factory + )) + + return Job( + jid=package.name, + deps=[], + env=job_env, + stages=stages, + ) + + description = dict( build_type='cmake', description="Builds a plain CMake package.", create_build_job=create_cmake_build_job, - create_clean_job=create_cmake_clean_job + create_clean_job=create_cmake_clean_job, + create_test_job=create_cmake_test_job, ) diff --git a/catkin_tools/jobs/test.py b/catkin_tools/jobs/test.py deleted file mode 100644 index 24a45e55..00000000 --- a/catkin_tools/jobs/test.py +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from catkin_tools.execution.io import IOBufferProtocol -from catkin_tools.execution.jobs import Job -from catkin_tools.execution.stages import FunctionStage, CommandStage -from catkin_tools.jobs.commands.cmake import CMakeMakeIOBufferProtocol -from catkin_tools.jobs.commands.make import MAKE_EXEC -from catkin_tools.jobs.utils import loadenv - - -def create_test_job( - context, - package, - package_path, -): - """Generate a job that tests a package""" - - # Package build space path - build_space = context.package_build_space(package) - # Environment dictionary for the job, which will be built - # up by the executions in the loadenv stage. - job_env = dict(os.environ) - - # Create job stages - stages = [] - - # Load environment for job - stages.append(FunctionStage( - 'loadenv', - loadenv, - locked_resource=None, - job_env=job_env, - package=package, - context=context, - verbose=False, - )) - - # Make command - stages.append(CommandStage( - 'make', - [MAKE_EXEC, 'run_tests'], - cwd=build_space, - logger_factory=IOBufferProtocol.factory - )) - - return Job( - jid=package.name, - deps=[], - env=job_env, - stages=stages, - ) \ No newline at end of file diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index fa3ca0bc..05eb3b31 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -13,6 +13,7 @@ import traceback from queue import Queue +import pkg_resources from catkin_pkg.package import InvalidPackage from catkin_pkg.packages import find_packages from catkin_pkg.topological_order import topological_order_packages @@ -21,7 +22,6 @@ from catkin_tools.execution import job_server from catkin_tools.execution.controllers import ConsoleStatusController from catkin_tools.execution.executor import run_until_complete, execute_jobs -from catkin_tools.jobs.test import create_test_job def test_workspace( @@ -46,6 +46,16 @@ def test_workspace( ordered_packages = topological_order_packages(workspace_packages) n_jobs = 1 + # Get all build type plugins + test_job_creators = { + ep.name: ep.load()['create_test_job'] + for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs') + } + + # It's a problem if there aren't any build types available + if len(test_job_creators) == 0: + sys.exit('Error: No build types available. Please check your catkin_tools installation.') + # Construct jobs jobs = [] for pkg_path, pkg in ordered_packages: @@ -58,7 +68,11 @@ def test_workspace( package=pkg, package_path=pkg_path) - jobs.append(create_test_job(**test_job_kwargs)) + # Create the job based on the build type + build_type = pkg.get_build_type() + + if build_type in test_job_creators: + jobs.append(test_job_creators[build_type](**test_job_kwargs)) # Queue for communicating status event_queue = Queue() From fe10c71522d0a43557b7766ff8f8201f198b1bc4 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 12:49:47 +0200 Subject: [PATCH 28/68] catkin test: add CMakeMakeRunTestsIOBufferProtocol --- catkin_tools/jobs/catkin.py | 4 +-- catkin_tools/jobs/cmake.py | 3 +- catkin_tools/jobs/commands/cmake.py | 41 ++++++++++++++++++++++++++ catkin_tools/verbs/catkin_test/test.py | 2 ++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index ce62b8b8..01bc1753 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -27,6 +27,7 @@ from .commands.cmake import CMAKE_EXEC from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol +from .commands.cmake import CMakeMakeRunTestsIOBufferProtocol from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC @@ -611,8 +612,7 @@ def create_catkin_test_job( 'make', [MAKE_EXEC, 'run_tests'], cwd=build_space, - #logger_factory=CMakeIOBufferProtocol.factory_factory(pkg_dir) - logger_factory=CMakeMakeIOBufferProtocol.factory, + logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, )) return Job( diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 4892b8a6..073fd02b 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -27,6 +27,7 @@ from .commands.cmake import CMAKE_INSTALL_MANIFEST_FILENAME from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol +from .commands.cmake import CMakeMakeRunTestsIOBufferProtocol from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC @@ -436,7 +437,7 @@ def create_cmake_test_job( 'make', [MAKE_EXEC, 'test'], cwd=build_space, - logger_factory=CMakeMakeIOBufferProtocol.factory + logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, )) return Job( diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index f5cae3d9..f7d62fa0 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -172,6 +172,47 @@ def on_stdout_received(self, data): percent=str(progress_matches.groups()[0]))) +class CMakeMakeRunTestsIOBufferProtocol(IOBufferProtocol): + """An IOBufferProtocol which parses the output of `make run_tests`.""" + def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs): + super(CMakeMakeRunTestsIOBufferProtocol, self).__init__( + label, job_id, stage_label, event_queue, log_path, *args, **kwargs) + + # Line formatting filters + # Each is a 2-tuple: + # - regular expression + # - output formatting line + filters = [ + (r'^-- run_tests.py:', '@!@{kf}{}@|'), + (r'^Removing test result files from ', '@!@{kf}{}@|'), + (r'^- removing ', '@!@{kf}{}@|'), + ] + + self.filters = [(re.compile(p), r) for (p, r) in filters] + + def on_stdout_received(self, data): + # Parse CMake Make completion progress + progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data)) + if progress_matches: + self.event_queue.put(ExecutionEvent( + 'STAGE_PROGRESS', + job_id=self.job_id, + stage_label=self.stage_label, + percent=str(progress_matches.groups()[0]))) + else: + # Write to stdout + colored = self.colorize_run_tests(data) + super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(colored.encode()) + + def colorize_run_tests(self, line): + cline = sanitize(line.decode()).rstrip() + for p, r in self.filters: + if p.match(cline): + lines = [fmt(r).format(line) for line in cline.splitlines()] + cline = '\n'.join(lines) + return cline + '\n' + + def get_installed_files(path): """Get a set of files installed by a CMake package as specified by an install_manifest.txt in a given directory.""" diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 05eb3b31..848842d6 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -96,6 +96,8 @@ def test_workspace( event_queue, show_live_stdout=True, show_live_stderr=True, + show_buffered_stdout=False, + show_buffered_stderr=False, ) status_thread.start() From bc0d18656c48a43ffa433edaf19e34cfd99f3cf2 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 13:08:05 +0200 Subject: [PATCH 29/68] catkin test: add some command line arguments --- catkin_tools/verbs/catkin_test/cli.py | 74 +++++++++++++++++++++++++- catkin_tools/verbs/catkin_test/test.py | 20 ++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 88de2645..f2422f23 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -9,11 +9,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import sys +from catkin_pkg.package import InvalidPackage from catkin_tools.argument_parsing import add_context_args +from catkin_tools.common import is_tty +from catkin_tools.common import getcwd +from catkin_tools.common import find_enclosing_package + from catkin_tools.context import Context -from catkin_tools.verbs.catkin_test.test import test_workspace +from catkin_tools.context import clr + +from catkin_tools.metadata import find_enclosing_workspace + +from catkin_tools.resultspace import load_resultspace_environment + +from catkin_tools.terminal_color import set_color + +from .test import test_workspace def prepare_arguments(parser): @@ -31,14 +46,71 @@ def prepare_arguments(parser): add = pkg_group.add_argument add('packages', metavar='PKGNAME', nargs='*', help='Workspace packages to test. If no packages are given, then all the packages are tested.') + add('--this', dest='build_this', action='store_true', default=False, + help='Test the package containing the current working directory.') + + behavior_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') + add = behavior_group.add_argument + add('--verbose', '-v', action='store_true', default=False, + help='Print output from commands in ordered blocks once the command finishes.') + add('--no-status', action='store_true', default=False, + help='Suppresses status line, useful in situations where carriage return is not properly supported.') + add('--no-notify', action='store_true', default=False, + help='Suppresses system pop-up notification.') return parser def main(opts): + # Set color options + opts.force_color = os.environ.get('CATKIN_TOOLS_FORCE_COLOR', opts.force_color) + if (opts.force_color or is_tty(sys.stdout)) and not opts.no_color: + set_color(True) + else: + set_color(False) + + # Context-aware args + if opts.build_this: + # Determine the enclosing package + try: + ws_path = find_enclosing_workspace(getcwd()) + # Suppress warnings since this won't necessarily find all packages + # in the workspace (it stops when it finds one package), and + # relying on it for warnings could mislead people. + this_package = find_enclosing_package( + search_start_path=getcwd(), + ws_path=ws_path, + warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) + + # Handle context-based package building + if this_package: + opts.packages += [this_package] + else: + sys.exit( + "[build] Error: In order to use --this, the current directory must be part of a catkin package.") + + # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + # Load the environment of the workspace to extend + if ctx.extend_path is not None: + try: + load_resultspace_environment(ctx.extend_path) + except IOError as exc: + sys.exit(clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % + (ctx.extend_path, exc.message))) + + # Set VERBOSE environment variable + if opts.verbose and 'VERBOSE' not in os.environ: + os.environ['VERBOSE'] = '1' + return test_workspace( ctx, packages=opts.packages, + quiet=not opts.verbose, + no_status=opts.no_status, + no_notify=opts.no_notify, ) \ No newline at end of file diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 848842d6..2c77e4fe 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -11,6 +11,7 @@ # limitations under the License. import sys import traceback +import time from queue import Queue import pkg_resources @@ -27,6 +28,9 @@ def test_workspace( context, packages=None, + quiet=False, + no_status=False, + no_notify=False, ): """Tests a catkin workspace @@ -34,7 +38,15 @@ def test_workspace( :type context: :py:class:`catkin_tools.context.Context` :param packages: list of packages to test :type packages: list + :param quiet: suppresses verbose build or test information + :type quiet: bool + :param no_status: suppresses the bottom status line + :type no_status: bool + :param no_notify: suppresses system notifications + :type no_notify: bool """ + pre_start_time = time.time() + # Get all the packages in the context source space # Suppress warnings since this is a utility function try: @@ -94,10 +106,14 @@ def test_workspace( [p for p in context.whitelist], [p for p in context.blacklist], event_queue, - show_live_stdout=True, - show_live_stderr=True, + show_notifications=not no_notify, + show_active_status=not no_status, show_buffered_stdout=False, show_buffered_stderr=False, + show_live_stdout=True, + show_live_stderr=True, + show_stage_events=not quiet, + pre_start_time=pre_start_time, ) status_thread.start() From 88b62d89a900e897fc5dd6c0bafc0b5322a110f7 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 13:09:23 +0200 Subject: [PATCH 30/68] catkin test: add run_tests alias for backwards compatibility --- catkin_tools/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catkin_tools/config.py b/catkin_tools/config.py index 7917a2e4..ac923900 100644 --- a/catkin_tools/config.py +++ b/catkin_tools/config.py @@ -31,6 +31,7 @@ ls: list install: config --install p: create pkg +run_tests: test """ From 8a8cd2b2864c550dec603d36c83834f9d04e2376 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 13:29:33 +0200 Subject: [PATCH 31/68] catkin test: better output filtering --- catkin_tools/jobs/commands/cmake.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index f7d62fa0..46be4350 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -189,18 +189,24 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k ] self.filters = [(re.compile(p), r) for (p, r) in filters] + self.progress = '0' def on_stdout_received(self, data): # Parse CMake Make completion progress progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data)) + # CMake also has output 'Scanning dependencies of target...', filter them + scanning_dependencies_matches = re.search(r'Scanning dependencies of target ', self._decode(data)) if progress_matches: + self.progress = str(progress_matches.groups()[0]) self.event_queue.put(ExecutionEvent( 'STAGE_PROGRESS', job_id=self.job_id, stage_label=self.stage_label, - percent=str(progress_matches.groups()[0]))) - else: - # Write to stdout + percent=self.progress)) + elif scanning_dependencies_matches: + pass + elif self.progress == '100': + # Only when make is finished, write to stdout colored = self.colorize_run_tests(data) super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(colored.encode()) From 6d2d1b23d93218c1ee1b6651e2605da6cd01816b Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 13:49:29 +0200 Subject: [PATCH 32/68] catkin test: add result of catkin_test_results --- catkin_tools/execution/io.py | 20 ++++++++++++++++++++ catkin_tools/jobs/catkin.py | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/catkin_tools/execution/io.py b/catkin_tools/execution/io.py index ad95997e..a5161597 100644 --- a/catkin_tools/execution/io.py +++ b/catkin_tools/execution/io.py @@ -13,12 +13,16 @@ # limitations under the License. import os +import re import shutil from glob import glob + from osrf_pycommon.process_utils import AsyncSubprocessProtocol from catkin_tools.common import mkdir_p +from catkin_tools.terminal_color import fmt + from .events import ExecutionEvent MAX_LOGFILE_HISTORY = 10 @@ -279,3 +283,19 @@ def on_process_exited2(self, returncode): self.on_stdout_received(self.intermediate_stdout_buffer + b'\n') if len(self.intermediate_stderr_buffer) > 0: self.on_stderr_received(self.intermediate_stderr_buffer + b'\n') + + +class CatkinTestResultsIOBufferProtocol(IOBufferProtocol): + """An IOBufferProtocol which parses the output of catkin_test_results""" + def on_stdout_received(self, data): + lines = data.decode().splitlines() + clines = [] + for line in lines: + match = re.match(r'(.*): (\d+) tests, (\d+) errors, (\d+) failures, (\d+) skipped', line) + if match: + line = fmt('@!{}@|: {} tests, @{rf}{} errors@|, @{rf}{} failures@|, @{kf}{} skipped@|').format(*match.groups()) + clines.append(line) + + cdata = '\n'.join(clines) + '\n' + + super(CatkinTestResultsIOBufferProtocol, self).on_stdout_received(cdata.encode()) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index 01bc1753..f6f40c98 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -23,6 +23,7 @@ from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage from catkin_tools.execution.stages import FunctionStage +from catkin_tools.execution.io import CatkinTestResultsIOBufferProtocol from .commands.cmake import CMAKE_EXEC from .commands.cmake import CMakeIOBufferProtocol @@ -615,6 +616,14 @@ def create_catkin_test_job( logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, )) + # catkin_test_results + stages.append(CommandStage( + 'results', + ['catkin_test_results'], + cwd=build_space, + logger_factory=CatkinTestResultsIOBufferProtocol.factory, + )) + return Job( jid=package.name, deps=[], From ac9eebca4d48fa5b6b5fa25def71132f52627904 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 16:24:22 +0200 Subject: [PATCH 33/68] catkin test: update tests --- tests/system/verbs/catkin_build/test_unit_tests.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/system/verbs/catkin_build/test_unit_tests.py b/tests/system/verbs/catkin_build/test_unit_tests.py index 3aaf15b8..10105e4d 100644 --- a/tests/system/verbs/catkin_build/test_unit_tests.py +++ b/tests/system/verbs/catkin_build/test_unit_tests.py @@ -5,6 +5,7 @@ from ....utils import assert_cmd_success from ....utils import assert_cmd_failure from ....utils import catkin_success +from ....utils import catkin_failure from ....utils import redirected_stdio TEST_DIR = os.path.dirname(__file__) @@ -23,9 +24,7 @@ def test_build_pkg_unit_tests(): 'python_tests', '--make-args', 'run_tests']) assert_cmd_success(['catkin_test_results', 'build/python_tests']) - assert catkin_success( - ['run_tests', 'python_tests', '--no-deps', '--no-notify', '--no-status']) - assert_cmd_success(['catkin_test_results', 'build/python_tests']) + assert catkin_success(['test', 'python_tests', '--no-notify', '--no-status']) @in_temporary_directory @@ -41,6 +40,4 @@ def test_build_pkg_unit_tests_broken(): 'python_tests_err', '--make-args', 'run_tests']) assert_cmd_failure(['catkin_test_results', 'build/python_tests_err']) - assert catkin_success( - ['run_tests', 'python_tests_err', '--no-deps', '--no-notify', '--no-status']) - assert_cmd_failure(['catkin_test_results', 'build/python_tests_err']) + assert catkin_failure(['test', 'python_tests_err', '--no-notify', '--no-status']) From 0f2044d15f1bb0d820f1a3144961c5a990bcb759 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 16:48:05 +0200 Subject: [PATCH 34/68] catkin test: add more options for package selection --- catkin_tools/verbs/catkin_test/test.py | 50 +++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 2c77e4fe..e550c754 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -19,7 +19,7 @@ from catkin_pkg.packages import find_packages from catkin_pkg.topological_order import topological_order_packages -from catkin_tools.common import clr, wide_log +from catkin_tools.common import clr, wide_log, expand_glob_package from catkin_tools.execution import job_server from catkin_tools.execution.controllers import ConsoleStatusController from catkin_tools.execution.executor import run_until_complete, execute_jobs @@ -55,9 +55,6 @@ def test_workspace( sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." " See below for details:\n\n%s" % (ex.package_path, ex.msg))) - ordered_packages = topological_order_packages(workspace_packages) - n_jobs = 1 - # Get all build type plugins test_job_creators = { ep.name: ep.load()['create_test_job'] @@ -68,12 +65,49 @@ def test_workspace( if len(test_job_creators) == 0: sys.exit('Error: No build types available. Please check your catkin_tools installation.') + # Get list of packages to test + ordered_packages = topological_order_packages(workspace_packages) + workspace_packages = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) + packages_to_test = [] + if packages: + for package in packages: + if package not in workspace_packages: + # Try whether package is a pattern and matches + glob_packages = expand_glob_package(package, workspace_packages) + if len(glob_packages) > 0: + packages.extend(glob_packages) + else: + sys.exit("[test] Given packages '{}' is not in the workspace " + "and pattern does not match any package".format(package)) + for pkg_path, package in ordered_packages: + if package.name in packages: + packages_to_test.append((pkg_path, package)) + else: + # Only use whitelist when no other packages are specified + if len(context.whitelist) > 0: + # Expand glob patterns in whitelist + whitelist = [] + for whitelisted_package in context.whitelist: + whitelist.extend(expand_glob_package(whitelisted_package, workspace_packages)) + packages_to_test = [p for p in ordered_packages if (p[1].name in whitelist)] + else: + packages_to_test = ordered_packages + + # Filter packages on blacklist + if len(context.blacklist) > 0: + # Expand glob patterns in blacklist + blacklist = [] + for blacklisted_package in context.blacklist: + blacklist.extend(expand_glob_package(blacklisted_package, workspace_packages)) + # Apply blacklist to packages and dependencies + packages_to_test = [ + (path, pkg) for path, pkg in packages_to_test + if (pkg.name not in blacklist or pkg.name in packages)] + # Construct jobs + n_jobs = 1 jobs = [] - for pkg_path, pkg in ordered_packages: - if pkg.name not in packages: - continue - + for pkg_path, pkg in packages_to_test: # Determine the job parameters test_job_kwargs = dict( context=context, From 2984af44ff1195a74bac89aba67b71f7a72b3adf Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 16:54:28 +0200 Subject: [PATCH 35/68] catkin test: add --continue-on-failure --- catkin_tools/verbs/catkin_test/cli.py | 3 +++ catkin_tools/verbs/catkin_test/test.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index f2422f23..d7251edb 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -48,6 +48,8 @@ def prepare_arguments(parser): help='Workspace packages to test. If no packages are given, then all the packages are tested.') add('--this', dest='build_this', action='store_true', default=False, help='Test the package containing the current working directory.') + add('--continue-on-failure', '-c', action='store_true', default=False, + help='Continue testing packages even if the tests for other requested packages fail.') behavior_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') add = behavior_group.add_argument @@ -113,4 +115,5 @@ def main(opts): quiet=not opts.verbose, no_status=opts.no_status, no_notify=opts.no_notify, + continue_on_failure=opts.continue_on_failure, ) \ No newline at end of file diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index e550c754..b04590f6 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -31,6 +31,7 @@ def test_workspace( quiet=False, no_status=False, no_notify=False, + continue_on_failure=False, ): """Tests a catkin workspace @@ -44,6 +45,8 @@ def test_workspace( :type no_status: bool :param no_notify: suppresses system notifications :type no_notify: bool + :param continue_on_failure: do not stop testing other packages on error + :type continue_on_failure: bool """ pre_start_time = time.time() @@ -163,6 +166,7 @@ def test_workspace( event_queue, context.log_space_abs, max_toplevel_jobs=n_jobs, + continue_on_failure=continue_on_failure, continue_without_deps=False)) except Exception: status_thread.keep_running = False From 61d4e6024f335ee7f4e7bf1567877e1b6c02ca26 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 17:24:38 +0200 Subject: [PATCH 36/68] catkin test: parallel execution --- catkin_tools/verbs/catkin_test/cli.py | 16 ++++++++++++++-- catkin_tools/verbs/catkin_test/test.py | 7 +++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index d7251edb..6a7b6735 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -51,8 +51,13 @@ def prepare_arguments(parser): add('--continue-on-failure', '-c', action='store_true', default=False, help='Continue testing packages even if the tests for other requested packages fail.') - behavior_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') - add = behavior_group.add_argument + config_group = parser.add_argument_group('Config', 'Parameters for the underlying build system.') + add = config_group.add_argument + add('-p', '--parallel-packages', metavar='PACKAGE_JOBS', dest='parallel_jobs', default=None, type=int, + help='Maximum number of packages allowed to be built in parallel (default is cpu count)') + + interface_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') + add = interface_group.add_argument add('--verbose', '-v', action='store_true', default=False, help='Print output from commands in ordered blocks once the command finishes.') add('--no-status', action='store_true', default=False, @@ -105,6 +110,12 @@ def main(opts): sys.exit(clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) + # Get parallel toplevel jobs + try: + parallel_jobs = int(opts.parallel_jobs) + except TypeError: + parallel_jobs = None + # Set VERBOSE environment variable if opts.verbose and 'VERBOSE' not in os.environ: os.environ['VERBOSE'] = '1' @@ -112,6 +123,7 @@ def main(opts): return test_workspace( ctx, packages=opts.packages, + n_jobs=parallel_jobs, quiet=not opts.verbose, no_status=opts.no_status, no_notify=opts.no_notify, diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index b04590f6..6c18bba2 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -28,6 +28,7 @@ def test_workspace( context, packages=None, + n_jobs=None, quiet=False, no_status=False, no_notify=False, @@ -39,6 +40,8 @@ def test_workspace( :type context: :py:class:`catkin_tools.context.Context` :param packages: list of packages to test :type packages: list + :param n_jobs: number of parallel package test jobs + :type n_jobs: int :param quiet: suppresses verbose build or test information :type quiet: bool :param no_status: suppresses the bottom status line @@ -108,7 +111,6 @@ def test_workspace( if (pkg.name not in blacklist or pkg.name in packages)] # Construct jobs - n_jobs = 1 jobs = [] for pkg_path, pkg in packages_to_test: # Determine the job parameters @@ -128,8 +130,9 @@ def test_workspace( # Initialize job server job_server.initialize( - max_jobs=1, + max_jobs=n_jobs, max_load=None, + gnu_make_enabled=context.use_internal_make_jobserver, ) try: From 4431f5f2b3046d6afd2f4cca5c2dd19e8acd0615 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 17:28:19 +0200 Subject: [PATCH 37/68] catkin test: add --interleave-output --- catkin_tools/verbs/catkin_test/cli.py | 3 +++ catkin_tools/verbs/catkin_test/test.py | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 6a7b6735..500126e6 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -60,6 +60,8 @@ def prepare_arguments(parser): add = interface_group.add_argument add('--verbose', '-v', action='store_true', default=False, help='Print output from commands in ordered blocks once the command finishes.') + add('--interleave-output', '-i', action='store_true', default=False, + help='Prevents ordering of command output when multiple commands are running at the same time.') add('--no-status', action='store_true', default=False, help='Suppresses status line, useful in situations where carriage return is not properly supported.') add('--no-notify', action='store_true', default=False, @@ -125,6 +127,7 @@ def main(opts): packages=opts.packages, n_jobs=parallel_jobs, quiet=not opts.verbose, + interleave_output=opts.interleave_output, no_status=opts.no_status, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 6c18bba2..5f5aa380 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -30,6 +30,7 @@ def test_workspace( packages=None, n_jobs=None, quiet=False, + interleave_output=False, no_status=False, no_notify=False, continue_on_failure=False, @@ -44,6 +45,8 @@ def test_workspace( :type n_jobs: int :param quiet: suppresses verbose build or test information :type quiet: bool + :param interleave_output: prints the output of commands as they are received + :type interleave_output: bool :param no_status: suppresses the bottom status line :type no_status: bool :param no_notify: suppresses system notifications @@ -148,10 +151,10 @@ def test_workspace( event_queue, show_notifications=not no_notify, show_active_status=not no_status, - show_buffered_stdout=False, - show_buffered_stderr=False, - show_live_stdout=True, - show_live_stderr=True, + show_buffered_stdout=not interleave_output, + show_buffered_stderr=not interleave_output, + show_live_stdout=interleave_output, + show_live_stderr=interleave_output, show_stage_events=not quiet, pre_start_time=pre_start_time, ) From 14638bfba28d67d671f35cb0b489ac4ec2cbe190 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 17:38:39 +0200 Subject: [PATCH 38/68] catkin test: add --summarize --- catkin_tools/verbs/catkin_test/cli.py | 3 +++ catkin_tools/verbs/catkin_test/test.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 500126e6..7b3ac46b 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -64,6 +64,8 @@ def prepare_arguments(parser): help='Prevents ordering of command output when multiple commands are running at the same time.') add('--no-status', action='store_true', default=False, help='Suppresses status line, useful in situations where carriage return is not properly supported.') + add('--summarize', '--summary', '-s', action='store_true', default=None, + help='Adds a summary to the end of the log') add('--no-notify', action='store_true', default=False, help='Suppresses system pop-up notification.') @@ -131,4 +133,5 @@ def main(opts): no_status=opts.no_status, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, + summarize_build=opts.summarize, ) \ No newline at end of file diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 5f5aa380..a7359261 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -34,6 +34,7 @@ def test_workspace( no_status=False, no_notify=False, continue_on_failure=False, + summarize_build=False, ): """Tests a catkin workspace @@ -53,6 +54,8 @@ def test_workspace( :type no_notify: bool :param continue_on_failure: do not stop testing other packages on error :type continue_on_failure: bool + :param summarize_build: summarizes the build at the end + :type summarize_build: bool """ pre_start_time = time.time() @@ -145,7 +148,7 @@ def test_workspace( ['package', 'packages'], jobs, n_jobs, - [pkg.name for _, pkg in context.packages], + [pkg.name for path, pkg in packages_to_test], [p for p in context.whitelist], [p for p in context.blacklist], event_queue, @@ -155,6 +158,7 @@ def test_workspace( show_buffered_stderr=not interleave_output, show_live_stdout=interleave_output, show_live_stderr=interleave_output, + show_full_summary=summarize_build, show_stage_events=not quiet, pre_start_time=pre_start_time, ) From 056c8e411a777447e9dc0d090456f65c59ba6cbc Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 17:55:35 +0200 Subject: [PATCH 39/68] catkin test: PEP8 --- catkin_tools/execution/io.py | 3 ++- catkin_tools/jobs/catkin.py | 2 -- catkin_tools/verbs/catkin_test/cli.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/catkin_tools/execution/io.py b/catkin_tools/execution/io.py index a5161597..9d416989 100644 --- a/catkin_tools/execution/io.py +++ b/catkin_tools/execution/io.py @@ -293,7 +293,8 @@ def on_stdout_received(self, data): for line in lines: match = re.match(r'(.*): (\d+) tests, (\d+) errors, (\d+) failures, (\d+) skipped', line) if match: - line = fmt('@!{}@|: {} tests, @{rf}{} errors@|, @{rf}{} failures@|, @{kf}{} skipped@|').format(*match.groups()) + line = fmt('@!{}@|: {} tests, @{rf}{} errors@|, @{rf}{} failures@|, @{kf}{} skipped@|') + line = line.format(*match.groups()) clines.append(line) cdata = '\n'.join(clines) + '\n' diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index f6f40c98..cd5e4430 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -586,8 +586,6 @@ def create_catkin_test_job( ): """Generate a job that tests a package""" - # Package source space path - pkg_dir = os.path.join(context.source_space_abs, package_path) # Package build space path build_space = context.package_build_space(package) # Environment dictionary for the job, which will be built diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 7b3ac46b..8055f792 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -134,4 +134,4 @@ def main(opts): no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize, - ) \ No newline at end of file + ) From abbc436f12f547a917ebe798dd92a711044f1086 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 18:12:34 +0200 Subject: [PATCH 40/68] catkin test: show error message when unbuilt packages are requested --- catkin_tools/verbs/catkin_test/test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index a7359261..0c22c556 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -116,6 +116,19 @@ def test_workspace( (path, pkg) for path, pkg in packages_to_test if (pkg.name not in blacklist or pkg.name in packages)] + # Check if all packages to test are already built + built_packages = set([ + pkg.name for (path, pkg) in + find_packages(context.package_metadata_path(), warnings=[]).items()]) + + packages_to_test_names = set(pkg.name for path, pkg in packages_to_test) + if not built_packages.issuperset(packages_to_test_names): + wide_log(clr("@{rf}Error: Packages have to be built before they can be tested.@|")) + wide_log(clr("The following requested packages are not built yet:")) + for package_name in packages_to_test_names.difference(built_packages): + wide_log(' - ' + package_name) + sys.exit(1) + # Construct jobs jobs = [] for pkg_path, pkg in packages_to_test: From c19e43b2adfcd079cdb8bce8232074073215d834 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sat, 17 Apr 2021 18:23:11 +0200 Subject: [PATCH 41/68] prevent nosetests from executing functions starting with test_ in source --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 10752e06..2aceb242 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -42,7 +42,7 @@ jobs: - name: Test catkin_tools run: | source /tmp/catkin_source/build/devel/setup.bash - nosetests -s + nosetests -s tests - name: Build documentation run: | pushd docs From 388d5d3b5894684f38d5807f5a97da5aaa577b9c Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Apr 2021 12:24:19 +0200 Subject: [PATCH 42/68] add tests for catkin test --- .../cmake_pkgs/test_err_pkg/CMakeLists.txt | 12 ++++ .../cmake_pkgs/test_err_pkg/package.xml | 10 ++++ .../cmake_pkgs/test_err_pkg/test.cpp | 6 ++ .../cmake_pkgs/test_pkg/CMakeLists.txt | 12 ++++ .../resources/cmake_pkgs/test_pkg/package.xml | 10 ++++ .../resources/cmake_pkgs/test_pkg/test.cpp | 6 ++ .../verbs/catkin_build/test_unit_tests.py | 43 -------------- tests/system/verbs/catkin_test/__init__.py | 0 .../verbs/catkin_test/test_unit_tests.py | 59 +++++++++++++++++++ 9 files changed, 115 insertions(+), 43 deletions(-) create mode 100644 tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt create mode 100644 tests/system/resources/cmake_pkgs/test_err_pkg/package.xml create mode 100644 tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp create mode 100644 tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt create mode 100644 tests/system/resources/cmake_pkgs/test_pkg/package.xml create mode 100644 tests/system/resources/cmake_pkgs/test_pkg/test.cpp delete mode 100644 tests/system/verbs/catkin_build/test_unit_tests.py create mode 100644 tests/system/verbs/catkin_test/__init__.py create mode 100644 tests/system/verbs/catkin_test/test_unit_tests.py diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt b/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt new file mode 100644 index 00000000..b68158d5 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_err_pkg) + +include(CTest) +add_executable(MyTest test.cpp) +add_test(NAME MyTest COMMAND MyTest) +enable_testing() + +install(TARGETS MyTest + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib/static) diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml b/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml new file mode 100644 index 00000000..b4d9610a --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml @@ -0,0 +1,10 @@ + + test_err_pkg + This package has a failing ctest test + 0.1.0 + BSD + todo + + cmake + + diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp b/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp new file mode 100644 index 00000000..b830dcd3 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp @@ -0,0 +1,6 @@ +#include + +int main() { + std::cout << "Test failure" << std::endl; + return 1; +} diff --git a/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt b/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt new file mode 100644 index 00000000..b55a2b25 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_pkg) + +include(CTest) +add_executable(MyTest test.cpp) +add_test(NAME MyTest COMMAND MyTest) +enable_testing() + +install(TARGETS MyTest + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib/static) diff --git a/tests/system/resources/cmake_pkgs/test_pkg/package.xml b/tests/system/resources/cmake_pkgs/test_pkg/package.xml new file mode 100644 index 00000000..f9501c7f --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/package.xml @@ -0,0 +1,10 @@ + + test_pkg + This package has a passing ctest test + 0.1.0 + BSD + todo + + cmake + + diff --git a/tests/system/resources/cmake_pkgs/test_pkg/test.cpp b/tests/system/resources/cmake_pkgs/test_pkg/test.cpp new file mode 100644 index 00000000..1e5d1669 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/test.cpp @@ -0,0 +1,6 @@ +#include + +int main() { + std::cout << "Test success" << std::endl; + return 0; +} diff --git a/tests/system/verbs/catkin_build/test_unit_tests.py b/tests/system/verbs/catkin_build/test_unit_tests.py deleted file mode 100644 index 10105e4d..00000000 --- a/tests/system/verbs/catkin_build/test_unit_tests.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import shutil - -from ....utils import in_temporary_directory -from ....utils import assert_cmd_success -from ....utils import assert_cmd_failure -from ....utils import catkin_success -from ....utils import catkin_failure -from ....utils import redirected_stdio - -TEST_DIR = os.path.dirname(__file__) -RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') - - -@in_temporary_directory -def test_build_pkg_unit_tests(): - """Test running working unit tests""" - cwd = os.getcwd() - source_space = os.path.join(cwd, 'src') - shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests'), source_space) - with redirected_stdio() as (out, err): - assert catkin_success( - ['build', '--no-notify', '--no-status', '--verbose', '--no-deps', - 'python_tests', '--make-args', 'run_tests']) - assert_cmd_success(['catkin_test_results', 'build/python_tests']) - - assert catkin_success(['test', 'python_tests', '--no-notify', '--no-status']) - - -@in_temporary_directory -def test_build_pkg_unit_tests_broken(): - """Test running broken unit tests""" - cwd = os.getcwd() - source_space = os.path.join(cwd, 'src') - shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_err'), source_space) - - with redirected_stdio() as (out, err): - assert catkin_success( - ['build', '--no-notify', '--no-status', '--verbose', '--no-deps', - 'python_tests_err', '--make-args', 'run_tests']) - assert_cmd_failure(['catkin_test_results', 'build/python_tests_err']) - - assert catkin_failure(['test', 'python_tests_err', '--no-notify', '--no-status']) diff --git a/tests/system/verbs/catkin_test/__init__.py b/tests/system/verbs/catkin_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/system/verbs/catkin_test/test_unit_tests.py b/tests/system/verbs/catkin_test/test_unit_tests.py new file mode 100644 index 00000000..e75c564c --- /dev/null +++ b/tests/system/verbs/catkin_test/test_unit_tests.py @@ -0,0 +1,59 @@ +import os +import shutil + +from ....utils import in_temporary_directory +from ....utils import assert_cmd_success +from ....utils import assert_cmd_failure +from ....utils import catkin_success +from ....utils import catkin_failure +from ....utils import redirected_stdio + +TEST_DIR = os.path.dirname(__file__) +RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') + + +@in_temporary_directory +def test_catkin_success(): + """Test running working unit tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests'), source_space) + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests', '--no-notify', '--no-status']) + assert catkin_success(['test', 'python_tests', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_catkin_failure(): + """Test running broken unit tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_err'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests_err', '--no-notify', '--no-status']) + assert catkin_failure(['test', 'python_tests_err', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_cmake_success(): + """Test vanilla cmake package""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'test_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'test_pkg', '--no-notify', '--no-status']) + assert catkin_success(['test', 'test_pkg', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_cmake_failure(): + """Test failing vanilla cmake package""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'test_err_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'test_err_pkg', '--no-notify', '--no-status']) + assert catkin_failure(['test', 'test_err_pkg', '--no-notify', '--no-status']) From fe15d87fa2490920307a1845363498fc5295b94c Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Apr 2021 12:27:19 +0200 Subject: [PATCH 43/68] catkin test: show complete output of ctest for cmake packages --- catkin_tools/jobs/cmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 073fd02b..5ee82891 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -27,7 +27,6 @@ from .commands.cmake import CMAKE_INSTALL_MANIFEST_FILENAME from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol -from .commands.cmake import CMakeMakeRunTestsIOBufferProtocol from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC @@ -37,6 +36,7 @@ from .utils import require_command from .utils import rmfiles +from catkin_tools.execution.io import IOBufferProtocol from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage from catkin_tools.execution.stages import FunctionStage @@ -437,7 +437,7 @@ def create_cmake_test_job( 'make', [MAKE_EXEC, 'test'], cwd=build_space, - logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, + logger_factory=IOBufferProtocol.factory, )) return Job( From 520ddbbd0e4c4435326ce20959c9cff9a12c0354 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Apr 2021 13:22:42 +0200 Subject: [PATCH 44/68] catkin test: report success when no tests exist --- catkin_tools/execution/executor.py | 6 +++++- catkin_tools/execution/stages.py | 18 +++++++++++++++--- catkin_tools/jobs/cmake.py | 11 +++++++++++ .../verbs/catkin_test/test_unit_tests.py | 12 ++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index ce1db811..4f908188 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -148,7 +148,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): raise TypeError("Bad Job Stage: {}".format(stage)) # Set whether this stage succeeded - stage_succeeded = (retcode == 0) + stage_succeeded = (retcode in stage.success_retcodes) # Update success tracker from this stage all_stages_succeeded = all_stages_succeeded and stage_succeeded @@ -166,6 +166,10 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): repro=stage.get_reproduction_cmd(verb, job.jid), retcode=retcode)) + # Early termination of the whole job + if retcode == stage.early_termination_retcode: + break + # Close logger logger.close() finally: diff --git a/catkin_tools/execution/stages.py b/catkin_tools/execution/stages.py index 641dd50c..758c1593 100644 --- a/catkin_tools/execution/stages.py +++ b/catkin_tools/execution/stages.py @@ -34,12 +34,16 @@ def __init__( logger_factory=IOBufferProtocol.factory, occupy_job=True, locked_resource=None, + early_termination_retcode=None, + success_retcodes=(0,), repro=None): self.label = str(label) self.logger_factory = logger_factory self.occupy_job = occupy_job self.repro = repro self.locked_resource = locked_resource + self.early_termination_retcode = early_termination_retcode + self.success_retcodes = success_retcodes def get_reproduction_cmd(self, verb, jid): """Get a command line to reproduce this stage with the proper environment.""" @@ -63,7 +67,9 @@ def __init__( stderr_to_stdout=False, occupy_job=True, locked_resource=None, - logger_factory=IOBufferProtocol.factory): + logger_factory=IOBufferProtocol.factory, + early_termination_retcode=None, + success_retcodes=(0,)): """ :param label: The label for the stage :param command: A list of strings composing a system command @@ -79,11 +85,14 @@ def __init__( :param occupy_job: Whether this stage should wait for a worker from the job server (default: True) :param logger_factory: The factory to use to construct a logger (default: IOBufferProtocol.factory) + :param early_termination_retcode: A return code that ends the whole job successfully (default: None) + :param success_retcodes: A tuple of return codes that mean successful termination of the command (default: (0,)) """ if not type(cmd) in [list, tuple] or not all([isinstance(s, str) for s in cmd]): raise ValueError('Command stage must be a list of strings: {}'.format(cmd)) - super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource) + super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource, + early_termination_retcode, success_retcodes) # Store environment overrides self.env_overrides = env_overrides or {} @@ -154,11 +163,14 @@ def __init__( logger_factory=IOBufferLogger.factory, occupy_job=True, locked_resource=None, + early_termination_retcode=None, + success_retcodes=(0,), *args, **kwargs): if not callable(function): raise ValueError('Function stage must be callable.') - super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource) + super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource, + early_termination_retcode, success_retcodes) self.args = args self.kwargs = kwargs diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 5ee82891..ddf24596 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -432,6 +432,17 @@ def create_cmake_test_job( verbose=False, )) + # Check if the test target exists + # make -q test returns 2 if the test target does not exist, in that case we want to terminate this test job + # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored + stages.append(CommandStage( + 'findtest', + [MAKE_EXEC, '-q', 'test'], + cwd=build_space, + early_termination_retcode=2, + success_retcodes=(0, 1, 2), + )) + # Make command stages.append(CommandStage( 'make', diff --git a/tests/system/verbs/catkin_test/test_unit_tests.py b/tests/system/verbs/catkin_test/test_unit_tests.py index e75c564c..199cc059 100644 --- a/tests/system/verbs/catkin_test/test_unit_tests.py +++ b/tests/system/verbs/catkin_test/test_unit_tests.py @@ -57,3 +57,15 @@ def test_cmake_failure(): with redirected_stdio() as (out, err): assert catkin_success(['build', 'test_err_pkg', '--no-notify', '--no-status']) assert catkin_failure(['test', 'test_err_pkg', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_skip_missing_test(): + """Test to skip packages without tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'cmake_pkg', '--no-notify', '--no-status']) + assert catkin_success(['test', '--no-notify', '--no-status']) From 3645adffa6b6e4af588aacfedae93cb19ab960a8 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Apr 2021 15:49:39 +0200 Subject: [PATCH 45/68] catkin test: add option to manually specify test target --- catkin_tools/jobs/catkin.py | 14 +++++++++++++- catkin_tools/jobs/cmake.py | 7 ++++--- catkin_tools/verbs/catkin_test/cli.py | 15 +++++++++++++++ catkin_tools/verbs/catkin_test/test.py | 11 +++++++++++ .../python_tests_targets/CMakeLists.txt | 10 ++++++++++ .../catkin_pkgs/python_tests_targets/package.xml | 11 +++++++++++ .../catkin_pkgs/python_tests_targets/setup.py | 6 ++++++ .../catkin_pkgs/python_tests_targets/test_bad.py | 12 ++++++++++++ .../catkin_pkgs/python_tests_targets/test_good.py | 12 ++++++++++++ tests/system/verbs/catkin_test/test_unit_tests.py | 15 +++++++++++++++ 10 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt create mode 100644 tests/system/resources/catkin_pkgs/python_tests_targets/package.xml create mode 100644 tests/system/resources/catkin_pkgs/python_tests_targets/setup.py create mode 100644 tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py create mode 100644 tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index cd5e4430..636915cd 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -583,6 +583,7 @@ def create_catkin_test_job( context, package, package_path, + test_target, ): """Generate a job that tests a package""" @@ -606,10 +607,21 @@ def create_catkin_test_job( verbose=False, )) + # Check if the test target exists + # make -q target_name returns 2 if the target does not exist, in that case we want to terminate this test job + # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored + stages.append(CommandStage( + 'findtest', + [MAKE_EXEC, '-q', test_target], + cwd=build_space, + early_termination_retcode=2, + success_retcodes=(0, 1, 2), + )) + # Make command stages.append(CommandStage( 'make', - [MAKE_EXEC, 'run_tests'], + [MAKE_EXEC, test_target], cwd=build_space, logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, )) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index ddf24596..60ddfec4 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -410,6 +410,7 @@ def create_cmake_test_job( context, package, package_path, + test_target ): """Generate a job to test a cmake package""" # Package build space path @@ -433,11 +434,11 @@ def create_cmake_test_job( )) # Check if the test target exists - # make -q test returns 2 if the test target does not exist, in that case we want to terminate this test job + # make -q target_name returns 2 if the target does not exist, in that case we want to terminate this test job # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored stages.append(CommandStage( 'findtest', - [MAKE_EXEC, '-q', 'test'], + [MAKE_EXEC, '-q', test_target], cwd=build_space, early_termination_retcode=2, success_retcodes=(0, 1, 2), @@ -446,7 +447,7 @@ def create_cmake_test_job( # Make command stages.append(CommandStage( 'make', - [MAKE_EXEC, 'test'], + [MAKE_EXEC, test_target], cwd=build_space, logger_factory=IOBufferProtocol.factory, )) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 8055f792..576a97bd 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -55,6 +55,10 @@ def prepare_arguments(parser): add = config_group.add_argument add('-p', '--parallel-packages', metavar='PACKAGE_JOBS', dest='parallel_jobs', default=None, type=int, help='Maximum number of packages allowed to be built in parallel (default is cpu count)') + add('-t', '--test-target', metavar='TARGET', default=None, type=str, + help='Make target to run for tests (default is "run_tests" for catkin and "test" for cmake)') + add('--catkin-test-target', metavar='TARGET', default=None, type=str, + help='Make target to run for tests for catkin packages, overwrites --test-target (default is "run_tests")') interface_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') add = interface_group.add_argument @@ -124,6 +128,15 @@ def main(opts): if opts.verbose and 'VERBOSE' not in os.environ: os.environ['VERBOSE'] = '1' + # Get test targets + catkin_test_target = 'run_tests' + cmake_test_target = 'test' + if opts.test_target: + catkin_test_target = opts.test_target + cmake_test_target = opts.test_target + if opts.catkin_test_target: + catkin_test_target = opts.catkin_test_target + return test_workspace( ctx, packages=opts.packages, @@ -134,4 +147,6 @@ def main(opts): no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize, + catkin_test_target=catkin_test_target, + cmake_test_target=cmake_test_target, ) diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 0c22c556..9baec6b1 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -35,6 +35,8 @@ def test_workspace( no_notify=False, continue_on_failure=False, summarize_build=False, + catkin_test_target='run_tests', + cmake_test_target='test', ): """Tests a catkin workspace @@ -56,6 +58,10 @@ def test_workspace( :type continue_on_failure: bool :param summarize_build: summarizes the build at the end :type summarize_build: bool + :param catkin_test_target: make target for tests in catkin packages + :type catkin_test_target: str + :param cmake_test_target: make target for tests in cmake packages + :type cmake_test_target: str """ pre_start_time = time.time() @@ -141,6 +147,11 @@ def test_workspace( # Create the job based on the build type build_type = pkg.get_build_type() + if build_type == 'catkin': + test_job_kwargs['test_target'] = catkin_test_target + elif build_type == 'cmake': + test_job_kwargs['test_target'] = cmake_test_target + if build_type in test_job_creators: jobs.append(test_job_creators[build_type](**test_job_kwargs)) diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt b/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt new file mode 100644 index 00000000..4e349acd --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.12) +project(python_tests_targets) +find_package(catkin REQUIRED) + +catkin_package() + +if(CATKIN_ENABLE_TESTING) + catkin_add_nosetests(test_good.py) + catkin_add_nosetests(test_bad.py) +endif() diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml b/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml new file mode 100644 index 00000000..9b4b2e6b --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml @@ -0,0 +1,11 @@ + + python_tests_targets + 0.1.0 + BSD + todo + This package contains two python tests. + + catkin + unittest + + diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py b/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py new file mode 100644 index 00000000..fb1311ce --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +from distutils.core import setup +from catkin_pkg.python_setup import generate_distutils_setup + +d = generate_distutils_setup() +setup(**d) diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py b/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py new file mode 100644 index 00000000..f46637e8 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import unittest + + +class TestBad(unittest.TestCase): + + def test_zero(self): + self.assertEqual(0, 1) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py b/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py new file mode 100644 index 00000000..0ae625a2 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import unittest + + +class TestGood(unittest.TestCase): + + def test_zero(self): + self.assertEqual(0, 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/system/verbs/catkin_test/test_unit_tests.py b/tests/system/verbs/catkin_test/test_unit_tests.py index 199cc059..dd3fcd7a 100644 --- a/tests/system/verbs/catkin_test/test_unit_tests.py +++ b/tests/system/verbs/catkin_test/test_unit_tests.py @@ -69,3 +69,18 @@ def test_skip_missing_test(): with redirected_stdio() as (out, err): assert catkin_success(['build', 'cmake_pkg', '--no-notify', '--no-status']) assert catkin_success(['test', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_other_target(): + """Test with a manually specified target""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_targets'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests_targets', '--no-notify', '--no-status']) + assert catkin_success(['test', '--test-target', 'run_tests_python_tests_targets_nosetests_test_good.py', + '--no-notify', '--no-status']) + assert catkin_failure(['test', '--test-target', 'run_tests_python_tests_targets_nosetests_test_bad.py', + '--no-notify', '--no-status']) From 8f11788acf440e73717d36f03c7e03e68dc10715 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 19 Apr 2021 14:11:02 +0200 Subject: [PATCH 46/68] catkin test: add shell completion --- completion/_catkin | 25 +++++++++++++++++++++++++ completion/catkin.bash | 6 +++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/completion/_catkin b/completion/_catkin index aa35d240..146b6204 100644 --- a/completion/_catkin +++ b/completion/_catkin @@ -205,6 +205,7 @@ _catkin_verbs_complete() { 'list:List workspace components' 'locate:Locate workspace components' 'profile:Switch between configurations' + 'test:Test packages' ) _describe -t verbs 'catkin verb' verbs_array -V verbs && return 0 } @@ -468,6 +469,28 @@ _catkin_locate_complete() { } +_catkin_test_complete() { + local test_opts + + test_opts=( + {-h,--help}'[Show usage help]'\ + {-w,--workspace}'[The workspace to test]:workspace:_files'\ + '--profile[Which configuration profile to use]:profile:_catkin_get_profiles'\ + {-v,--verbose}'[Print all output from test commands]'\ + {-i,--interleave-output}"[Print output from test commands as it's generated]"\ + '--this[Test the current package]'\ + {-c,--continue-on-failure}"[Continue testing after failed tests]"\ + {-p,--parallel-packages}"[Number of packages to build in parallel]:number"\ + {-t,--test-target}"[make target to run for tests]:target"\ + '--catkin-test-target[make target to run for tests in catkin packages]:target'\ + '--no-status[Suppress status line]'\ + {-s,--summarize}"[Add a summary at the end]"\ + '*:package:_catkin_get_packages') + + _arguments -C $test_opts && return 0 +} + + local curcontext="$curcontext" state line ret ret=1 @@ -580,6 +603,8 @@ case "$state" in ;; (profile) _catkin_profile_complete && ret=0 ;; + (test) _catkin_test_complete && ret=0 + ;; esac ret=0 ;; diff --git a/completion/catkin.bash b/completion/catkin.bash index 86de5c72..2b592b06 100644 --- a/completion/catkin.bash +++ b/completion/catkin.bash @@ -55,7 +55,7 @@ _catkin() _init_completion || return # this handles default completion (variables, redirection) # complete to the following verbs - local catkin_verbs="build clean config create init list profile run_tests" + local catkin_verbs="build clean config create init list profile test" # filter for long options (from bash_completion) local OPTS_FILTER='s/.*\(--[-A-Za-z0-9]\{1,\}=\{0,1\}\).*/\1/p' @@ -133,9 +133,9 @@ _catkin() local catkin_list_opts=$(catkin list --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) COMPREPLY=($(compgen -W "${catkin_list_opts}" -- ${cur})) ;; - run_tests) + test) if [[ ${cur} == -* ]]; then - local catkin_run_tests_opts=$(catkin run_tests --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) + local catkin_run_tests_opts=$(catkin test --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) COMPREPLY=($(compgen -W "${catkin_run_tests_opts}" -- ${cur})) else COMPREPLY=($(compgen -W "$(_catkin_pkgs)" -- ${cur})) From bbb479927007d7bf62a3fd25d1dc414154927a1b Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 19 Apr 2021 14:38:27 +0200 Subject: [PATCH 47/68] catkin test: add documentation --- docs/cheat_sheet.rst | 14 +++++++++ docs/verbs/catkin_build.rst | 53 -------------------------------- docs/verbs/catkin_test.rst | 55 ++++++++++++++++++++++++++++++++++ docs/verbs/cli/catkin_test.txt | 55 ++++++++++++++++++++++++++++++++++ docs/verbs/cli/dump_cli | 1 + 5 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 docs/verbs/catkin_test.rst create mode 100644 docs/verbs/cli/catkin_test.txt diff --git a/docs/cheat_sheet.rst b/docs/cheat_sheet.rst index aef7e846..6d054015 100644 --- a/docs/cheat_sheet.rst +++ b/docs/cheat_sheet.rst @@ -75,6 +75,20 @@ Build all packages in a given directory: ... or in the current folder: - ``catkin build $(catkin list -u -d .)`` +Testing Packages +^^^^^^^^^^^^^^^^ + +Test all the packages: + - ``catkin test`` + +... one at a time, with live output: + - ``catkin build -p 1 -i`` + +Test a specific package: + - ``catkin test my_package`` + +... or a specific test target of a package + - ``catkin test -t my_target my_package`` Cleaning Build Products ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/verbs/catkin_build.rst b/docs/verbs/catkin_build.rst index 404cd995..30c6d0cd 100644 --- a/docs/verbs/catkin_build.rst +++ b/docs/verbs/catkin_build.rst @@ -232,59 +232,6 @@ This will skip all of the package's dependencies, build the given package, and t
-Building and Running Tests -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Running tests for a given package typically is done by invoking a special ``make`` target like ``test`` or ``run_tests``. -catkin packages all define the ``run_tests`` target which aggregates all types of tests and runs them together. -So in order to get tests to build and run for your packages you need to pass them this additional ``run_tests`` or ``test`` target as a command line option to ``make``. - -To run catkin tests for all catkin packages in the workspace, use the following: - -.. code-block:: bash - - $ catkin run_tests - -Or the longer version: - -.. code-block:: bash - - $ catkin build [...] --catkin-make-args run_tests - -To run a catkin test for a specific catkin package, from a directory within that package: - -.. code-block:: bash - - $ catkin run_tests --no-deps --this - -For non-catkin packages which define a ``test`` target, you can do this: - -.. code-block:: bash - - $ catkin build [...] --make-args test - -If you want to run tests for just one package, then you should build that package and this narrow down the build to just that package with the additional make argument: - -.. code-block:: bash - - $ # First build the package - $ catkin build package - ... - $ # Then run its tests - $ catkin build package --no-deps --catkin-make-args run_tests - $ # Or for non-catkin packages - $ catkin build package --no-deps --make-args test - -For catkin packages and the ``run_tests`` target, failing tests will not result in an non-zero exit code. -So if you want to check for failing tests, use the ``catkin_test_results`` command like this: - -.. code-block:: bash - - $ catkin_test_results build/ - -The result code will be non-zero unless all tests passed. - - Advanced Options ^^^^^^^^^^^^^^^^ diff --git a/docs/verbs/catkin_test.rst b/docs/verbs/catkin_test.rst new file mode 100644 index 00000000..14aab526 --- /dev/null +++ b/docs/verbs/catkin_test.rst @@ -0,0 +1,55 @@ +``catkin test`` -- Test Packages +================================== + +The ``test`` verb is used to test one or more packages in a catkin workspace. +Like most verbs, ``test`` is context-aware and can be executed from within any directory contained by an initialized workspace. +Specific workspaces can also be built from arbitrary working directories with the ``--workspace`` option. + +Basic Usage +^^^^^^^^^^^ + +Before running tests for packages in the workspace, they have to be built with ``catkin build``. +Then, to run the tests, use the following: + +.. code-block:: bash + + $ catkin test + +Under the hood, this invokes the ``make`` targets ``run_tests`` or ``test``, depending on the package. +catkin packages all define the ``run_tests`` target which aggregates all types of tests and runs them together. +For cmake packages that do not use catkin, the ``test`` target is invoked. +This target is usually populated by cmake when the ``enable_testing()`` command is used in the ``CMakeLists.txt``. +If it does not exist, a warning is printed. + +To run a catkin test for a specific catkin package, from a directory within that package: + +.. code-block:: bash + + $ catkin test --this + +Advanced Options +^^^^^^^^^^^^^^^^ + +To manually specify a different ``make`` target, use ``--test-target``: + +.. code-block:: bash + + $ catkin test --test-target gtest + +It is also possible to use ``--catkin-test-target`` to change the target only for catkin packages. + +Normally, the tests are run in parallel, similar to the build jobs of ``catkin build``. +To avoid building packages in parallel or to reduce the amount of parallel jobs, use ``-p``: + +.. code-block:: bash + + $ catkin test -p 1 + +Sometimes, it can be helpful to see the output of tests while they are still running. +This can be achieved using ``--interleave-output``. + +Full Command-Line Interface +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: cli/catkin_test.txt + :language: text diff --git a/docs/verbs/cli/catkin_test.txt b/docs/verbs/cli/catkin_test.txt new file mode 100644 index 00000000..df36c774 --- /dev/null +++ b/docs/verbs/cli/catkin_test.txt @@ -0,0 +1,55 @@ +usage: catkin test [-h] [--workspace WORKSPACE] [--profile PROFILE] [--this] + [--continue-on-failure] [-p PACKAGE_JOBS] [-t TARGET] + [--catkin-test-target TARGET] [--verbose] + [--interleave-output] [--no-status] [--summarize] + [--no-notify] + [PKGNAME ...] + +Test one or more packages in a catkin workspace. This invokes `make run_tests` +or `make test` for either all or the specified packages in a catkin workspace. + +optional arguments: + -h, --help show this help message and exit + --workspace WORKSPACE, -w WORKSPACE + The path to the catkin_tools workspace or a directory + contained within it (default: ".") + --profile PROFILE The name of a config profile to use (default: active + profile) + +Packages: + Control which packages get tested. + + PKGNAME Workspace packages to test. If no packages are given, + then all the packages are tested. + --this Test the package containing the current working + directory. + --continue-on-failure, -c + Continue testing packages even if the tests for other + requested packages fail. + +Config: + Parameters for the underlying build system. + + -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS + Maximum number of packages allowed to be built in + parallel (default is cpu count) + -t TARGET, --test-target TARGET + Make target to run for tests (default is "run_tests" + for catkin and "test" for cmake) + --catkin-test-target TARGET + Make target to run for tests for catkin packages, + overwrites --test-target (default is "run_tests") + +Interface: + The behavior of the command-line interface. + + --verbose, -v Print output from commands in ordered blocks once the + command finishes. + --interleave-output, -i + Prevents ordering of command output when multiple + commands are running at the same time. + --no-status Suppresses status line, useful in situations where + carriage return is not properly supported. + --summarize, --summary, -s + Adds a summary to the end of the log + --no-notify Suppresses system pop-up notification. diff --git a/docs/verbs/cli/dump_cli b/docs/verbs/cli/dump_cli index 1143b024..c69c828a 100755 --- a/docs/verbs/cli/dump_cli +++ b/docs/verbs/cli/dump_cli @@ -15,3 +15,4 @@ catkin profile set -h > catkin_profile_set.txt catkin profile add -h > catkin_profile_add.txt catkin profile rename -h > catkin_profile_rename.txt catkin profile remove -h > catkin_profile_remove.txt +catkin test -h > catkin_test.txt From c4b2c7badffd711a45b3b95dc2b103a313cfa772 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 19 Apr 2021 23:35:30 +0200 Subject: [PATCH 48/68] catkin test: fix --verbose --- catkin_tools/jobs/catkin.py | 3 ++- catkin_tools/jobs/cmake.py | 3 ++- catkin_tools/jobs/commands/cmake.py | 23 ++++++++++++++++++----- catkin_tools/verbs/catkin_test/test.py | 3 ++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index 636915cd..9844e622 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -584,6 +584,7 @@ def create_catkin_test_job( package, package_path, test_target, + verbose, ): """Generate a job that tests a package""" @@ -623,7 +624,7 @@ def create_catkin_test_job( 'make', [MAKE_EXEC, test_target], cwd=build_space, - logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory, + logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory_factory(verbose), )) # catkin_test_results diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 60ddfec4..cdead64f 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -410,7 +410,8 @@ def create_cmake_test_job( context, package, package_path, - test_target + test_target, + verbose, ): """Generate a job to test a cmake package""" # Package build space path diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index 46be4350..1b3a1d28 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -174,7 +174,7 @@ def on_stdout_received(self, data): class CMakeMakeRunTestsIOBufferProtocol(IOBufferProtocol): """An IOBufferProtocol which parses the output of `make run_tests`.""" - def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs): + def __init__(self, label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs): super(CMakeMakeRunTestsIOBufferProtocol, self).__init__( label, job_id, stage_label, event_queue, log_path, *args, **kwargs) @@ -190,6 +190,7 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k self.filters = [(re.compile(p), r) for (p, r) in filters] self.progress = '0' + self.verbose = verbose def on_stdout_received(self, data): # Parse CMake Make completion progress @@ -203,10 +204,11 @@ def on_stdout_received(self, data): job_id=self.job_id, stage_label=self.stage_label, percent=self.progress)) - elif scanning_dependencies_matches: - pass - elif self.progress == '100': - # Only when make is finished, write to stdout + super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(data) + elif self.progress == '100' or self.verbose: + # Only when make is finished or we are verbose, write to stdout + if scanning_dependencies_matches and not self.verbose: + return colored = self.colorize_run_tests(data) super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(colored.encode()) @@ -218,6 +220,17 @@ def colorize_run_tests(self, line): cline = '\n'.join(lines) return cline + '\n' + @classmethod + def factory_factory(cls, verbose): + """Factory factory for constructing protocols that know the verbosity.""" + def factory(label, job_id, stage_label, event_queue, log_path): + # factory is called by caktin_tools executor + def init_proxy(*args, **kwargs): + # init_proxy is called by asyncio + return cls(label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs) + return init_proxy + return factory + def get_installed_files(path): """Get a set of files installed by a CMake package as specified by an diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 9baec6b1..2187898c 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -142,7 +142,8 @@ def test_workspace( test_job_kwargs = dict( context=context, package=pkg, - package_path=pkg_path) + package_path=pkg_path, + verbose=not quiet) # Create the job based on the build type build_type = pkg.get_build_type() From a8bc55d4223c74e06ad0cac86bef942078e56328 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 20 Apr 2021 19:00:27 +0200 Subject: [PATCH 49/68] catkin test: improve filtering of run_tests output --- catkin_tools/jobs/commands/cmake.py | 42 ++++++++++++----------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index 1b3a1d28..c3ecdb88 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -161,8 +161,10 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k def on_stdout_received(self, data): super(CMakeMakeIOBufferProtocol, self).on_stdout_received(data) + self.send_progress(data) - # Parse CMake Make completion progress + def send_progress(self, data): + """Parse CMake Make completion progress""" progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data)) if progress_matches is not None: self.event_queue.put(ExecutionEvent( @@ -172,7 +174,7 @@ def on_stdout_received(self, data): percent=str(progress_matches.groups()[0]))) -class CMakeMakeRunTestsIOBufferProtocol(IOBufferProtocol): +class CMakeMakeRunTestsIOBufferProtocol(CMakeMakeIOBufferProtocol): """An IOBufferProtocol which parses the output of `make run_tests`.""" def __init__(self, label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs): super(CMakeMakeRunTestsIOBufferProtocol, self).__init__( @@ -182,38 +184,28 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, verbose, * # Each is a 2-tuple: # - regular expression # - output formatting line - filters = [ - (r'^-- run_tests.py:', '@!@{kf}{}@|'), - (r'^Removing test result files from ', '@!@{kf}{}@|'), - (r'^- removing ', '@!@{kf}{}@|'), + self.filters = [ + (re.compile(r'^-- run_tests.py:'), '@!@{kf}{}@|'), ] - self.filters = [(re.compile(p), r) for (p, r) in filters] - self.progress = '0' + self.in_test_output = False self.verbose = verbose def on_stdout_received(self, data): - # Parse CMake Make completion progress - progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data)) - # CMake also has output 'Scanning dependencies of target...', filter them - scanning_dependencies_matches = re.search(r'Scanning dependencies of target ', self._decode(data)) - if progress_matches: - self.progress = str(progress_matches.groups()[0]) - self.event_queue.put(ExecutionEvent( - 'STAGE_PROGRESS', - job_id=self.job_id, - stage_label=self.stage_label, - percent=self.progress)) - super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(data) - elif self.progress == '100' or self.verbose: - # Only when make is finished or we are verbose, write to stdout - if scanning_dependencies_matches and not self.verbose: - return + self.send_progress(data) + + data = self._decode(data) + if data.startswith('-- run_tests.py: execute command'): + self.in_test_output = True + elif data.startswith('-- run_tests.py: verify result'): + self.in_test_output = False + + if self.verbose or self.in_test_output: colored = self.colorize_run_tests(data) super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(colored.encode()) def colorize_run_tests(self, line): - cline = sanitize(line.decode()).rstrip() + cline = sanitize(line).rstrip() for p, r in self.filters: if p.match(cline): lines = [fmt(r).format(line) for line in cline.splitlines()] From fc65a6b004f19cd8a828d3be2f0d16c595e65e17 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 14:03:08 +0200 Subject: [PATCH 50/68] catkin test: check for circular dependencies --- catkin_tools/verbs/catkin_test/test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 2187898c..76e8018f 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -85,6 +85,13 @@ def test_workspace( # Get list of packages to test ordered_packages = topological_order_packages(workspace_packages) + + # Check if topological_order_packages determined any circular dependencies, if so print an error and fail. + # If this is the case, the last entry of ordered packages is a tuple that starts with nil. + if ordered_packages and ordered_packages[-1][0] is None: + guilty_packages = ", ".join(ordered_packages[-1][1:]) + sys.exit("[test] Circular dependency detected in the following packages: {}".format(guilty_packages)) + workspace_packages = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) packages_to_test = [] if packages: From e3f9f7c69329e461e470208e566b27a4e440b89f Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 14:11:12 +0200 Subject: [PATCH 51/68] catkin test: add --make-args --- catkin_tools/jobs/catkin.py | 2 +- catkin_tools/jobs/cmake.py | 2 +- catkin_tools/verbs/catkin_test/cli.py | 3 +++ docs/verbs/cli/catkin_test.txt | 10 +++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index 9844e622..7d2bf4aa 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -622,7 +622,7 @@ def create_catkin_test_job( # Make command stages.append(CommandStage( 'make', - [MAKE_EXEC, test_target], + [MAKE_EXEC, test_target] + context.make_args, cwd=build_space, logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory_factory(verbose), )) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index cdead64f..61ca84cf 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -448,7 +448,7 @@ def create_cmake_test_job( # Make command stages.append(CommandStage( 'make', - [MAKE_EXEC, test_target], + [MAKE_EXEC, test_target] + context.make_args, cwd=build_space, logger_factory=IOBufferProtocol.factory, )) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 576a97bd..29206759 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -59,6 +59,9 @@ def prepare_arguments(parser): help='Make target to run for tests (default is "run_tests" for catkin and "test" for cmake)') add('--catkin-test-target', metavar='TARGET', default=None, type=str, help='Make target to run for tests for catkin packages, overwrites --test-target (default is "run_tests")') + add('--make-args', metavar='ARG', dest='make_args', nargs='+', required=False, type=str, default=None, + help='Arbitrary arguments which are passed to make. ' + 'It collects all of following arguments until a "--" is read.') interface_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') add = interface_group.add_argument diff --git a/docs/verbs/cli/catkin_test.txt b/docs/verbs/cli/catkin_test.txt index df36c774..d099d973 100644 --- a/docs/verbs/cli/catkin_test.txt +++ b/docs/verbs/cli/catkin_test.txt @@ -1,8 +1,8 @@ usage: catkin test [-h] [--workspace WORKSPACE] [--profile PROFILE] [--this] [--continue-on-failure] [-p PACKAGE_JOBS] [-t TARGET] - [--catkin-test-target TARGET] [--verbose] - [--interleave-output] [--no-status] [--summarize] - [--no-notify] + [--catkin-test-target TARGET] [--make-args ARG [ARG ...]] + [--verbose] [--interleave-output] [--no-status] + [--summarize] [--no-notify] [PKGNAME ...] Test one or more packages in a catkin workspace. This invokes `make run_tests` @@ -39,6 +39,10 @@ Config: --catkin-test-target TARGET Make target to run for tests for catkin packages, overwrites --test-target (default is "run_tests") + --make-args ARG [ARG ...] + Arbitrary arguments which are passed to make. It + collects all of following arguments until a "--" is + read. Interface: The behavior of the command-line interface. From a3cab742915df4c578657c0ebb816f3f22730397 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 18 Jul 2021 14:23:52 +0200 Subject: [PATCH 52/68] catkin test: call cmake_check_build_system before running the tests --- catkin_tools/jobs/catkin.py | 13 +++++++++++++ catkin_tools/jobs/commands/cmake.py | 16 ++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index 7d2bf4aa..c0acf16a 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -588,6 +588,8 @@ def create_catkin_test_job( ): """Generate a job that tests a package""" + # Package source space path + pkg_dir = os.path.join(context.source_space_abs, package_path) # Package build space path build_space = context.package_build_space(package) # Environment dictionary for the job, which will be built @@ -608,6 +610,17 @@ def create_catkin_test_job( verbose=False, )) + # Check buildsystem command + # The stdout is suppressed here instead of globally because for the actual tests, + # stdout contains important information, but for cmake it is only relevant when verbose + stages.append(CommandStage( + 'check', + [MAKE_EXEC, 'cmake_check_build_system'], + cwd=build_space, + logger_factory=CMakeIOBufferProtocol.factory_factory(pkg_dir, suppress_stdout=not verbose), + occupy_job=True + )) + # Check if the test target exists # make -q target_name returns 2 if the target does not exist, in that case we want to terminate this test job # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index c3ecdb88..eb2ca46f 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -50,9 +50,11 @@ def abspath(self, groups): """Group filter that turns source-relative paths into absolute paths.""" return (groups[0] if groups[0].startswith(os.sep) else os.path.join(self.source_path, groups[0]),) + groups[1:] - def __init__(self, label, job_id, stage_label, event_queue, log_path, source_path, *args, **kwargs): + def __init__(self, label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args, + **kwargs): super(CMakeIOBufferProtocol, self).__init__(label, job_id, stage_label, event_queue, log_path, *args, **kwargs) self.source_path = source_path + self.suppress_stdout = suppress_stdout # These are buffers for incomplete lines that we want to wait to parse # until we have received them completely @@ -78,9 +80,10 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, source_pat self.filters = [(re.compile(p), r, f) for (p, r, f) in filters] def on_stdout_received(self, data): - data_head, self.stdout_tail = split_to_last_line_break(self.stdout_tail + data) - colored = self.color_lines(data_head) - super(CMakeIOBufferProtocol, self).on_stdout_received(colored) + if not self.suppress_stdout: + data_head, self.stdout_tail = split_to_last_line_break(self.stdout_tail + data) + colored = self.color_lines(data_head) + super(CMakeIOBufferProtocol, self).on_stdout_received(colored) def on_stderr_received(self, data): data_head, self.stderr_tail = split_to_last_line_break(self.stderr_tail + data) @@ -116,13 +119,14 @@ def color_lines(self, data): return encoded_data @classmethod - def factory_factory(cls, source_path): + def factory_factory(cls, source_path, suppress_stdout=False): """Factory factory for constructing protocols that know the source path for this CMake package.""" def factory(label, job_id, stage_label, event_queue, log_path): # factory is called by caktin_tools executor def init_proxy(*args, **kwargs): # init_proxy is called by asyncio - return cls(label, job_id, stage_label, event_queue, log_path, source_path, *args, **kwargs) + return cls(label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args, + **kwargs) return init_proxy return factory From 142dfa115364963c68d7ab7b591ceebece58b1a5 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 27 Jul 2021 11:29:24 +0200 Subject: [PATCH 53/68] Fix environment variable parsing (#686) --- catkin_tools/common.py | 34 +++++++++++++------------- catkin_tools/resultspace.py | 21 ++-------------- catkin_tools/verbs/catkin_build/cli.py | 2 +- catkin_tools/verbs/catkin_env/cli.py | 4 +-- 4 files changed, 22 insertions(+), 39 deletions(-) diff --git a/catkin_tools/common.py b/catkin_tools/common.py index 9cfe55dc..24b8e27e 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -21,8 +21,6 @@ import sys from fnmatch import fnmatch from itertools import chain -from shlex import split as cmd_split -from shlex import quote as cmd_quote from catkin_pkg.packages import find_packages @@ -572,11 +570,15 @@ def mkdir_p(path): raise -def format_env_dict(environ): +def format_env_dict(environ, human_readable=True): """Format an environment dict for printing to console similarly to `typeset` builtin.""" - return '\n'.join([ - 'typeset -x {}={}'.format(k, cmd_quote(v)) + if human_readable: + separator = '\n' + else: + separator = '\x00' + return separator.join([ + '{}={}'.format(k, v) for k, v in environ.items() ]) @@ -586,18 +588,16 @@ def parse_env_str(environ_str): environ_str must be encoded in utf-8 """ - - try: - split_envs = [e.split('=', 1) for e in cmd_split(environ_str.decode())] - return { - e[0]: e[1] for e - in split_envs - if len(e) == 2 - } - except ValueError: - print('WARNING: Could not parse env string: `{}`'.format(environ_str), - file=sys.stderr) - raise + variables = environ_str.decode().rstrip('\x00').split('\x00') + environment = {} + for v in variables: + try: + key, value = v.split('=', 1) + environment[key] = value + except ValueError: + print('WARNING: Could not parse env string: `{}`'.format(v), + file=sys.stderr) + return environment def expand_glob_package(pattern, all_workspace_packages): diff --git a/catkin_tools/resultspace.py b/catkin_tools/resultspace.py index 8efedf0f..7d98f2dc 100644 --- a/catkin_tools/resultspace.py +++ b/catkin_tools/resultspace.py @@ -16,7 +16,6 @@ import subprocess import sys from hashlib import md5 -from osrf_pycommon.process_utils import execute_process from shlex import quote as cmd_quote from .common import parse_env_str @@ -113,17 +112,11 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c ) # Construct a command list which sources the setup file and prints the env to stdout - norc_flags = { - 'bash': '--norc', - 'zsh': '-f' - } - command = ' '.join([ cmd_quote(setup_file_path), shell_path, - norc_flags[shell_name], '-c', - '"typeset -px"' + '"env --null"', ]) # Define some "blacklisted" environment variables which shouldn't be copied @@ -132,17 +125,7 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c try: # Run the command synchronously to get the resultspace environmnet - if 0: - # NOTE: This sometimes fails to get all output (returns prematurely) - lines = '' - for ret in execute_process(command, cwd=os.getcwd(), env=base_env, emulate_tty=False, shell=True): - if type(ret) is bytes: - ret = ret.decode() - if isinstance(ret, str): - lines += ret - else: - p = subprocess.Popen(command, cwd=os.getcwd(), env=base_env, shell=True, stdout=subprocess.PIPE) - lines, _ = p.communicate() + lines = subprocess.check_output(command, cwd=os.getcwd(), env=base_env, shell=True) # Extract the environment variables env_dict = { diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 23420d18..268538a8 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -203,7 +203,7 @@ def print_build_env(context, package_name): if pkg.name == package_name: environ = dict(os.environ) loadenv(None, None, environ, pkg, context) - print(format_env_dict(environ)) + print(format_env_dict(environ, human_readable=sys.stdout.isatty())) return 0 print('[build] Error: Package `{}` not in workspace.'.format(package_name), file=sys.stderr) diff --git a/catkin_tools/verbs/catkin_env/cli.py b/catkin_tools/verbs/catkin_env/cli.py index 7c709cd6..91ecb6f5 100644 --- a/catkin_tools/verbs/catkin_env/cli.py +++ b/catkin_tools/verbs/catkin_env/cli.py @@ -30,7 +30,7 @@ def prepare_arguments(parser): help='Start with an empty environment.') add('-s', '--stdin', default=False, action='store_true', help='Read environment variable definitions from stdin. ' - 'Variables should be given in NAME=VALUE format. ') + 'Variables should be given in NAME=VALUE format, separated by null-bytes.') add('envs_', metavar='NAME=VALUE', nargs='*', type=str, default=[], help='Explicitly set environment variables for the subcommand. ' @@ -102,7 +102,7 @@ def main(opts): # Update environment from stdin if opts.stdin: - input_env_str = sys.stdin.read() + input_env_str = sys.stdin.read().strip() environ.update(parse_env_str(input_env_str.encode())) # Finally, update with explicit vars From 5d9d2723a097a176813f97da219581e73a678b6f Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 3 Aug 2021 12:01:19 +0200 Subject: [PATCH 54/68] Update installation docs for python 3 (#687) --- docs/installing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installing.rst b/docs/installing.rst index 5b5244ac..3fded3ff 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -26,7 +26,7 @@ Once you have added that repository, run these commands to install ``catkin_tool .. code-block:: bash $ sudo apt-get update - $ sudo apt-get install python-catkin-tools + $ sudo apt-get install python3-catkin-tools Installing on other platforms with pip ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -35,7 +35,7 @@ Simply install it with ``pip``: .. code-block:: bash - $ sudo pip install -U catkin_tools + $ sudo pip3 install -U catkin_tools Installing from source ^^^^^^^^^^^^^^^^^^^^^^ @@ -51,13 +51,13 @@ Then install the dependencies with ``pip``: .. code-block:: bash - $ pip install -r requirements.txt --upgrade + $ pip3 install -r requirements.txt --upgrade Then install with the ``setup.py`` file: .. code-block:: bash - $ python setup.py install --record install_manifest.txt + $ python3 setup.py install --record install_manifest.txt .. note:: @@ -74,14 +74,14 @@ To setup ``catkin_tools`` for fast iteration during development, use the ``devel .. code-block:: bash - $ python setup.py develop + $ python3 setup.py develop Now the commands, like ``catkin``, will be in the system path and the local source files located in the ``catkin_tools`` folder will be on the ``PYTHONPATH``. When you are done with your development, undo this by running this command: .. code-block:: bash - $ python setup.py develop -u + $ python3 setup.py develop -u Uninstalling from Source From bbe4eff9f36f8639421be5412dc2d3ce8d617ee0 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Sun, 3 Oct 2021 14:17:33 +0200 Subject: [PATCH 55/68] catkin test: fix parsing of make arguments --- catkin_tools/argument_parsing.py | 4 ++-- catkin_tools/verbs/catkin_test/__init__.py | 3 +++ catkin_tools/verbs/catkin_test/cli.py | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/catkin_tools/argument_parsing.py b/catkin_tools/argument_parsing.py index 1ae9e33d..f4e9f0ae 100644 --- a/catkin_tools/argument_parsing.py +++ b/catkin_tools/argument_parsing.py @@ -392,12 +392,12 @@ def argument_preprocessor(args): jobs_args = extract_jobs_flags(' '.join(args)) if jobs_args: # Remove jobs flags from cli args if they're present - args = re.sub(' '.join(jobs_args), '', ' '.join(args)).split() + args = [arg for arg in args if arg not in jobs_args] elif make_args is not None: jobs_args = extract_jobs_flags(' '.join(make_args)) if jobs_args: # Remove jobs flags from cli args if they're present - make_args = re.sub(' '.join(jobs_args), '', ' '.join(make_args)).split() + make_args = [arg for arg in make_args if arg not in jobs_args] extras = { 'cmake_args': cmake_args, diff --git a/catkin_tools/verbs/catkin_test/__init__.py b/catkin_tools/verbs/catkin_test/__init__.py index 9094837b..62dbed3d 100644 --- a/catkin_tools/verbs/catkin_test/__init__.py +++ b/catkin_tools/verbs/catkin_test/__init__.py @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from catkin_tools.argument_parsing import argument_preprocessor + from .cli import main from .cli import prepare_arguments @@ -19,4 +21,5 @@ description="Tests a catkin workspace.", main=main, prepare_arguments=prepare_arguments, + argument_preprocessor=argument_preprocessor, ) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index 29206759..f9af6b74 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -15,6 +15,7 @@ from catkin_pkg.package import InvalidPackage from catkin_tools.argument_parsing import add_context_args +from catkin_tools.argument_parsing import configure_make_args from catkin_tools.common import is_tty from catkin_tools.common import getcwd from catkin_tools.common import find_enclosing_package @@ -121,6 +122,10 @@ def main(opts): sys.exit(clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) + # Extract make arguments + make_args, _, _, _ = configure_make_args(ctx.make_args, ctx.jobs_args, ctx.use_internal_make_jobserver) + ctx.make_args = make_args + # Get parallel toplevel jobs try: parallel_jobs = int(opts.parallel_jobs) From e83062fb657f3eb20210f4a10428481dcb18358b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Thu, 14 Oct 2021 11:38:40 +0200 Subject: [PATCH 56/68] https in docs, properties over vars, sphinx cleanup (#690) * Fix typo * Use install property * Use https * Remove unused sphinxcontrib-programoutput doc requirement * Simplify environment get * Replace more dunder vars by their property * Fix verb in test --- .github/ISSUE_TEMPLATE.md | 4 ++-- README.md | 2 +- catkin_tools/context.py | 20 ++++++++++---------- catkin_tools/verbs/catkin_build/cli.py | 2 +- catkin_tools/verbs/catkin_create/cli.py | 2 +- catkin_tools/verbs/catkin_test/test.py | 2 +- docs/conf.py | 2 -- requirements.txt | 1 - setup.py | 2 +- 9 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6ab4fc7d..ed8a771a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -13,10 +13,10 @@ ### Build / Run Issue [//]: # (If you are migrating from catkin_make, please follow the migration guide:) -[//]: # (http://catkin-tools.readthedocs.org/en/latest/migration.html) +[//]: # (https://catkin-tools.readthedocs.org/en/latest/migration.html) [//]: # (Please also check for solved issues here:) -[//]: # (http://catkin-tools.readthedocs.org/en/latest/troubleshooting.html) +[//]: # (https://catkin-tools.readthedocs.org/en/latest/troubleshooting.html) [//]: # (And check for open issues here:) [//]: # (https://github.com/catkin/catkin_tools/issues?q=is%3Aopen+is%3Aissue+label%3Abug) diff --git a/README.md b/README.md index f7137ac7..0d049321 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,4 @@ Command line tools for working with [catkin](https://github.com/ros/catkin) -Documentation: http://catkin-tools.readthedocs.org/ +Documentation: https://catkin-tools.readthedocs.org/ diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 40cbb5bc..8356966d 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -90,7 +90,7 @@ def space_setter(self, value): if self.__locked: raise RuntimeError("Setting of context members is not allowed while locked.") setattr(self, '__%s_space' % space, value) - setattr(self, '__%s_space_abs' % space, os.path.realpath(os.path.join(self.__workspace, value))) + setattr(self, '__%s_space_abs' % space, os.path.realpath(os.path.join(self.workspace, value))) def space_exists(self): """ @@ -379,7 +379,7 @@ def __init__( if len(kwargs) > 0: print('Warning: Unhandled config context options: {}'.format(kwargs), file=sys.stderr) - self.destdir = os.environ['DESTDIR'] if 'DESTDIR' in os.environ else None + self.destdir = os.environ.get('DESTDIR', None) # Handle package whitelist/blacklist self.whitelist = whitelist or [] @@ -576,8 +576,8 @@ def existence_str(path, used=True): return clr(' @{bf}[unused]@|') install_layout = 'None' - if self.__install: - install_layout = 'merged' if not self.__isolate_install else 'isolated' + if self.install: + install_layout = 'merged' if not self.isolate_install else 'isolated' def quote(argument): # Distinguish in the printout if space separates two arguments or if we @@ -597,17 +597,17 @@ def quote(argument): 'extend': extend_value, 'install_layout': install_layout, 'cmake_prefix_path': (self.cmake_prefix_path or ['Empty']), - 'cmake_args': ' '.join([quote(a) for a in self.__cmake_args or ['None']]), - 'make_args': ' '.join(self.__make_args + self.__jobs_args or ['None']), - 'catkin_make_args': ', '.join(self.__catkin_make_args or ['None']), + 'cmake_args': ' '.join([quote(a) for a in self.cmake_args or ['None']]), + 'make_args': ' '.join(self.make_args + self.jobs_args or ['None']), + 'catkin_make_args': ', '.join(self.catkin_make_args or ['None']), 'source_missing': existence_str(self.source_space_abs), 'log_missing': existence_str(self.log_space_abs), 'build_missing': existence_str(self.build_space_abs), 'devel_missing': existence_str(self.devel_space_abs), - 'install_missing': existence_str(self.install_space_abs, used=self.__install), + 'install_missing': existence_str(self.install_space_abs, used=self.install), 'destdir_missing': existence_str(self.destdir, used=self.destdir), - 'whitelisted_packages': ' '.join(self.__whitelist or ['None']), - 'blacklisted_packages': ' '.join(self.__blacklist or ['None']), + 'whitelisted_packages': ' '.join(self.whitelist or ['None']), + 'blacklisted_packages': ' '.join(self.blacklist or ['None']), } subs.update(**self.__dict__) # Get the width of the shell diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 268538a8..0ba1338f 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -138,7 +138,7 @@ def prepare_arguments(parser): add('--no-summarize', '--no-summary', action='store_false', dest='summarize', help='Explicitly disable the end of build summary') add('--override-build-tool-check', action='store_true', default=False, - help='use to override failure due to using differnt build tools on the same workspace.') + help='use to override failure due to using different build tools on the same workspace.') # Deprecated args now handled by main catkin command add('--no-color', action='store_true', help=argparse.SUPPRESS) diff --git a/catkin_tools/verbs/catkin_create/cli.py b/catkin_tools/verbs/catkin_create/cli.py index f3f2b633..a10fd5da 100644 --- a/catkin_tools/verbs/catkin_create/cli.py +++ b/catkin_tools/verbs/catkin_create/cli.py @@ -55,7 +55,7 @@ def prepare_arguments(parser): # default='catkin', # help='The buildtool to use to build the package. (default: catkin)') - rosdistro_name = os.environ['ROS_DISTRO'] if 'ROS_DISTRO' in os.environ else None + rosdistro_name = os.environ.get('ROS_DISTRO', None) add('--rosdistro', required=rosdistro_name is None, default=rosdistro_name, help='The ROS distro (default: environment variable ROS_DISTRO if defined)') diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 76e8018f..2209fc4b 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -202,7 +202,7 @@ def test_workspace( # Block while running N jobs asynchronously try: all_succeeded = run_until_complete(execute_jobs( - 'build', + 'test', jobs, locks, event_queue, diff --git a/docs/conf.py b/docs/conf.py index d42392d0..2106a275 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,8 +34,6 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode', - #'sphinxcontrib.programoutput', - #'sphinxcontrib.ansi', ] # Conditionally add spelling diff --git a/requirements.txt b/requirements.txt index 06ceec8a..59061505 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ catkin_pkg osrf_pycommon pyyaml setuptools -sphinxcontrib-programoutput diff --git a/setup.py b/setup.py index 68397817..deb787e7 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ def run(self): author_email='william@osrfoundation.org', maintainer='William Woodall', maintainer_email='william@osrfoundation.org', - url='http://catkin-tools.readthedocs.org/', + url='https://catkin-tools.readthedocs.org/', keywords=['catkin'], classifiers=[ 'Environment :: Console', From 9f81a46b3a28c922bd2eb9650482cb07a0f1cf8c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 15 Oct 2021 12:08:04 +0200 Subject: [PATCH 57/68] More cleanup (#692) * replace http by https * Fix unfilled parameter * Fix update_active_metadata * Remove redundant parentheses * Fix typo's * Convert methods to static methods * Fix styling in test_jobs * Cleanup docstrings --- catkin_tools/argument_parsing.py | 6 ++-- catkin_tools/commands/catkin.py | 2 +- catkin_tools/common.py | 14 ++++++++-- catkin_tools/context.py | 14 +++++----- catkin_tools/execution/controllers.py | 2 +- catkin_tools/execution/events.py | 2 +- catkin_tools/execution/executor.py | 17 +++++++---- catkin_tools/execution/io.py | 11 ++++++-- catkin_tools/execution/job_server.py | 7 +++-- catkin_tools/execution/stages.py | 5 +--- catkin_tools/jobs/catkin.py | 15 ++++++---- catkin_tools/jobs/commands/cmake.py | 8 +++--- catkin_tools/jobs/utils.py | 16 +++++------ catkin_tools/metadata.py | 28 +++++++++++++------ .../catkin_build_notifier_src/README.markdown | 2 +- catkin_tools/resultspace.py | 12 +++++--- catkin_tools/terminal_color.py | 6 ++-- catkin_tools/utils.py | 2 +- catkin_tools/verbs/catkin_build/build.py | 8 +++++- catkin_tools/verbs/catkin_build/color.py | 2 +- catkin_tools/verbs/catkin_clean/clean.py | 4 ++- catkin_tools/verbs/catkin_clean/color.py | 2 +- catkin_tools/verbs/catkin_config/cli.py | 2 +- catkin_tools/verbs/catkin_create/cli.py | 4 +-- catkin_tools/verbs/catkin_env/cli.py | 2 +- catkin_tools/verbs/catkin_init/cli.py | 2 +- catkin_tools/verbs/catkin_list/__init__.py | 2 +- catkin_tools/verbs/catkin_profile/cli.py | 10 +++---- tests/unit/test_jobs.py | 18 ++++++------ 29 files changed, 134 insertions(+), 91 deletions(-) diff --git a/catkin_tools/argument_parsing.py b/catkin_tools/argument_parsing.py index f4e9f0ae..f6b8ca67 100644 --- a/catkin_tools/argument_parsing.py +++ b/catkin_tools/argument_parsing.py @@ -322,6 +322,8 @@ def configure_make_args(make_args, jobs_args, use_internal_make_jobserver): :param make_args: arguments to be passed to GNU Make :type make_args: list + :param jobs_args: job arguments overriding make flags + :type jobs_args: list :param use_internal_make_jobserver: if true, use the internal jobserver :type make_args: bool :rtype: tuple (final make_args, using makeflags, using cliflags, using jobserver) @@ -329,7 +331,7 @@ def configure_make_args(make_args, jobs_args, use_internal_make_jobserver): # Configure default jobs options: use all CPUs in each package try: - # NOTE: this will yeild greater than 100% CPU utilization + # NOTE: this will yield greater than 100% CPU utilization n_cpus = cpu_count() jobs_flags = { 'jobs': n_cpus, @@ -376,7 +378,7 @@ def argument_preprocessor(args): :param args: system arguments from which special arguments need to be extracted :type args: list - :returns: a tuple contianing a list of the arguments which can be handled + :returns: a tuple containing a list of the arguments which can be handled by argparse and a dict of the extra arguments which this function has extracted :rtype: tuple diff --git a/catkin_tools/commands/catkin.py b/catkin_tools/commands/catkin.py index 16c9123f..d1a7a41f 100644 --- a/catkin_tools/commands/catkin.py +++ b/catkin_tools/commands/catkin.py @@ -189,7 +189,7 @@ def catkin_main(sysargs): date.today().year) ) print('catkin_tools is released under the Apache License,' - ' Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)') + ' Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0)') print('---') print('Using Python {}'.format(''.join(sys.version.split('\n')))) sys.exit(0) diff --git a/catkin_tools/common.py b/catkin_tools/common.py index 24b8e27e..981da43c 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -60,7 +60,7 @@ def getcwd(symlinks=True): realpath = os.getcwd() # The `PWD` environment variable should contain the path that we took to - # get here, includng symlinks + # get here, including symlinks if symlinks: cwd = os.environ.get('PWD', '') @@ -138,7 +138,7 @@ def format_time_delta_short(delta): def get_cached_recursive_build_depends_in_workspace(package, workspace_packages): - """Returns cached or calculated recursive build dependes for a given package + """Returns cached or calculated recursive build depends for a given package If the recursive build depends for this package and this set of workspace packages has already been calculated, the cached results are returned. @@ -423,7 +423,7 @@ def slice_to_printed_length(string, length): def printed_fill(string, length): - """Textwrapping for strings with esacpe characters.""" + """Textwrapping for strings with escape characters.""" splat = string.replace('\\n', ' \\n ').split() count = 0 @@ -528,7 +528,15 @@ def find_enclosing_package(search_start_path=None, ws_path=None, warnings=None, """Get the package containing a specific directory. :param search_start_path: The path to crawl upward to find a package, CWD if None + :type search_start_path: str :param ws_path: The path at which the search should stop + :type ws_path: str + :param warnings: Print warnings if None or return them in the given list + :type warnings: list + :param symlinks: If True, then get the path considering symlinks. If false, + resolve the path to the actual path. + :type symlinks: bool + :returns: """ search_path = search_start_path or getcwd(symlinks=symlinks) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 8356966d..7fb1302c 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -316,7 +316,7 @@ def __init__( :type source_space: str :param log_space: relative location of log space, defaults to '/logs' :type log_space: str - :param build_space: relativetarget location of build space, defaults to '/build' + :param build_space: relative target location of build space, defaults to '/build' :type build_space: str :param devel_space: relative target location of devel space, defaults to '/devel' :type devel_space: str @@ -334,7 +334,7 @@ def __init__( :type make_args: list :param jobs_args: -j and -l jobs args :type jobs_args: list - :param use_internal_make_jobserver: true if this configuration should use an internal make jobserv + :param use_internal_make_jobserver: true if this configuration should use an internal make jobserver :type use_internal_make_jobserver: bool :param use_env_cache: true if this configuration should cache job environments loaded from resultspaces :type use_env_cache: bool @@ -471,7 +471,7 @@ def load_env(self): "requires the `catkin` CMake package in your source space " "in order to be built.")] - # Add warnings based on conflicing CMAKE_PREFIX_PATH + # Add warnings based on conflicting CMAKE_PREFIX_PATH elif self.cached_cmake_prefix_path and self.extend_path: ep_not_in_lcpp = any([self.extend_path in p for p in self.cached_cmake_prefix_path.split(':')]) if not ep_not_in_lcpp: @@ -637,7 +637,7 @@ def quote(argument): return (divider + "\n" + ("\n" + divider + "\n").join(groups) + "\n" + divider + "\n" + - ((("\n\n").join(notes) + "\n" + divider) if notes else '') + + (("\n\n".join(notes) + "\n" + divider) if notes else '') + warnings_joined) @property @@ -849,7 +849,7 @@ def extends(self, value): @property def private_devel_path(self): """The path to the hidden directory in the develspace that - contains the symbollically-linked isolated develspaces.""" + contains the symbolically-linked isolated develspaces.""" return os.path.join(self.devel_space_abs, '.private') def package_private_devel_path(self, package): @@ -871,7 +871,7 @@ def package_devel_space(self, package): elif self.link_devel: return os.path.join(self.private_devel_path, package.name) else: - raise ValueError('Unkown devel space layout: {}'.format(self.devel_layout)) + raise ValueError('Unknown devel space layout: {}'.format(self.devel_layout)) def package_install_space(self, package): """Get the install directory for a specific package. @@ -883,7 +883,7 @@ def package_install_space(self, package): elif self.isolate_install: return os.path.join(self.install_space_abs, package.name) else: - raise ValueError('Unkown install space layout: {}'.format(self.devel_layout)) + raise ValueError('Unknown install space layout: {}'.format(self.devel_layout)) def package_dest_path(self, package): """Get the intermediate destination into which a specific package is built.""" diff --git a/catkin_tools/execution/controllers.py b/catkin_tools/execution/controllers.py index 895b2b81..55b61412 100644 --- a/catkin_tools/execution/controllers.py +++ b/catkin_tools/execution/controllers.py @@ -34,7 +34,7 @@ from catkin_tools.execution import job_server -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' diff --git a/catkin_tools/execution/events.py b/catkin_tools/execution/events.py index afedce83..16320ce4 100644 --- a/catkin_tools/execution/events.py +++ b/catkin_tools/execution/events.py @@ -21,7 +21,7 @@ class ExecutionEvent(object): Events can be jobs starting/finishing, commands starting/failing/finishing, commands producing output (each line is an event), or when the executor - quits or failes. + quits or fails. """ # TODO: Make this a map of ID -> fields diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index 4f908188..89088a64 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -40,9 +40,12 @@ def split(values, cond): async def async_job(verb, job, threadpool, locks, event_queue, log_path): """Run a sequence of Stages from a Job and collect their output. + :param verb: Current command verb :param job: A Job instance - :threadpool: A thread pool executor for blocking stages - :event_queue: A queue for asynchronous events + :param threadpool: A thread pool executor for blocking stages + :param locks: Dict containing the locks to acquire + :param event_queue: A queue for asynchronous events + :param log_path: The path in which logfiles can be written """ # Initialize success flag @@ -95,7 +98,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): # Get the logger protocol_type = stage.logger_factory(verb, job.jid, stage.label, event_queue, log_path) - # Start asynchroonous execution + # Start asynchronous execution transport, logger = await ( async_execute_process( protocol_type, @@ -120,7 +123,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): # Asynchronously yield until this command is completed retcode = await logger.complete except: # noqa: E722 - # Bare except is permissable here because the set of errors which the CommandState might raise + # Bare except is permissible here because the set of errors which the CommandState might raise # is unbounded. We capture the traceback here and save it to the build's log files. logger = IOBufferLogger(verb, job.jid, stage.label, event_queue, log_path) logger.err(str(traceback.format_exc())) @@ -136,7 +139,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): logger, event_queue) except: # noqa: E722 - # Bare except is permissable here because the set of errors which the FunctionStage might raise + # Bare except is permissible here because the set of errors which the FunctionStage might raise # is unbounded. We capture the traceback here and save it to the build's log files. logger.err('Stage `{}` failed with arguments:'.format(stage.label)) for arg_val in stage.args: @@ -176,7 +179,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): lock.release() # Finally, return whether all stages of the job completed - return (job.jid, all_stages_succeeded) + return job.jid, all_stages_succeeded async def execute_jobs( @@ -190,7 +193,9 @@ async def execute_jobs( continue_without_deps=False): """Process a number of jobs asynchronously. + :param verb: Current command verb :param jobs: A list of topologically-sorted Jobs with no circular dependencies. + :param locks: Dict containing the locks to acquire :param event_queue: A python queue for reporting events. :param log_path: The path in which logfiles can be written :param max_toplevel_jobs: Max number of top-level jobs diff --git a/catkin_tools/execution/io.py b/catkin_tools/execution/io.py index 9d416989..ecb26ad1 100644 --- a/catkin_tools/execution/io.py +++ b/catkin_tools/execution/io.py @@ -129,13 +129,15 @@ def get_stderr_log(self): except UnicodeDecodeError: return "stderr_log: some output cannot be displayed.\n" - def _encode(self, data): + @staticmethod + def _encode(data): """Encode a Python str into bytes. :type data: str """ return _encode(data) - def _decode(self, data): + @staticmethod + def _decode(data): """Decode bytes into Python str. :type data: bytes """ @@ -169,6 +171,7 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k def out(self, data, end='\n'): """ :type data: str + :type end: str """ # Buffer the encoded data data += end @@ -189,6 +192,7 @@ def out(self, data, end='\n'): def err(self, data, end='\n'): """ :type data: str + :type end: str """ # Buffer the encoded data data += end @@ -225,7 +229,8 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k self.intermediate_stdout_buffer = b'' self.intermediate_stderr_buffer = b'' - def _split(self, data): + @staticmethod + def _split(data): try: last_break = data.rindex(b'\n') + 1 return data[0:last_break], data[last_break:] diff --git a/catkin_tools/execution/job_server.py b/catkin_tools/execution/job_server.py index c3ebefc6..8e14b604 100644 --- a/catkin_tools/execution/job_server.py +++ b/catkin_tools/execution/job_server.py @@ -72,7 +72,7 @@ def memory_usage(): def test_gnu_make_support_common(makefile_content): """ - Test if "make -f MAKEFILE -j2" runs successfullyn when MAKEFILE + Test if "make -f MAKEFILE -j2" runs successfully when MAKEFILE contains makefile_content. """ @@ -83,7 +83,7 @@ def test_gnu_make_support_common(makefile_content): ret = subprocess.call(['make', '-f', makefile, '-j2'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) os.unlink(makefile) - return (ret == 0) + return ret == 0 def test_gnu_make_support_old(): @@ -294,6 +294,7 @@ def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=Fals :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value (see _set_max_mem for additional documentation) + :param gnu_make_enabled: Set gnu make compatibility enabled """ # Check initialization @@ -308,7 +309,7 @@ def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=Fals log(clr('@!@{yf}WARNING:@| Make job server not supported. The number of Make ' 'jobs may exceed the number of CPU cores.@|')) - # Set gnu make compatibilty enabled + # Set gnu make compatibility enabled JobServer._gnu_make_enabled = gnu_make_enabled # Set the maximum number of jobs diff --git a/catkin_tools/execution/stages.py b/catkin_tools/execution/stages.py index 758c1593..a63fdc68 100644 --- a/catkin_tools/execution/stages.py +++ b/catkin_tools/execution/stages.py @@ -72,10 +72,7 @@ def __init__( success_retcodes=(0,)): """ :param label: The label for the stage - :param command: A list of strings composing a system command - :param protocol: A protocol class to use for this stage - - :parma cmd: The command to run + :param cmd: The command to run :param cwd: The directory in which to run the command (default: os.getcwd()) :param env: The base environment. (default: {}) :param env_overrides: The variables that override the base environment. (default: {}) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index c0acf16a..e3d652e8 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -84,13 +84,14 @@ def clean_linked_files( files_that_collide, files_to_clean, dry_run): - """Removes a list of files and adjusts collison counts for colliding files. + """Removes a list of files and adjusts collision counts for colliding files. This function synchronizes access to the devel collisions file. - :param devel_space_abs: absolute path to merged devel space + :param metadata_path: absolute path to the general metadata directory :param files_that_collide: list of absolute paths to files that collide :param files_to_clean: list of absolute paths to files to clean + :param dry_run: Perform a dry-run """ # Get paths @@ -166,8 +167,10 @@ def unlink_devel_products( :param devel_space_abs: Path to a merged devel space. :param private_devel_path: Path to the private devel space - :param devel_manifest_path: Path to the directory containing the package's + :param metadata_path: Path to the directory containing the general metadata + :param package_metadata_path: Path to the directory containing the package's catkin_tools metadata + :param dry_run: Perform a dry-run """ # Check paths @@ -202,7 +205,7 @@ def unlink_devel_products( # Clean the file or decrement the collision count files_to_clean.append(dest_file) - # Remove all listed symli and empty directories which have been removed + # Remove all listed symlinks and empty directories which have been removed # after this build, and update the collision file clean_linked_files(logger, event_queue, metadata_path, [], files_to_clean, dry_run) @@ -261,7 +264,7 @@ def link_devel_products( logger.out('Linked: ({}, {})'.format(source_dir, dest_dir)) else: # Create a symlink - logger.out('Symlinking %s' % (dest_dir)) + logger.out('Symlinking %s' % dest_dir) try: os.symlink(source_dir, dest_dir) except OSError: @@ -306,7 +309,7 @@ def link_devel_products( logger.out('Linked: ({}, {})'.format(source_file, dest_file)) else: # Create the symlink - logger.out('Symlinking %s' % (dest_file)) + logger.out('Symlinking %s' % dest_file) try: os.symlink(source_file, dest_file) except OSError: diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index eb2ca46f..981f8d45 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -110,7 +110,7 @@ def color_lines(self, data): """Apply colorization rules to each line in data""" decoded_data = self._decode(data) # TODO: This will only work if all lines are received at once. Instead - # of direclty splitting lines, we should buffer the data lines until + # of directly splitting lines, we should buffer the data lines until # the last character is a line break lines = decoded_data.splitlines(True) # Keep line breaks colored_lines = [self.colorize_cmake(line) for line in lines] @@ -122,7 +122,7 @@ def color_lines(self, data): def factory_factory(cls, source_path, suppress_stdout=False): """Factory factory for constructing protocols that know the source path for this CMake package.""" def factory(label, job_id, stage_label, event_queue, log_path): - # factory is called by caktin_tools executor + # factory is called by catkin_tools executor def init_proxy(*args, **kwargs): # init_proxy is called by asyncio return cls(label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args, @@ -157,7 +157,7 @@ def colorize_cmake(self, line): class CMakeMakeIOBufferProtocol(IOBufferProtocol): - """An IOBufferProtocol which parses CMake's progree prefixes and emits corresponding STAGE_PROGRESS events.""" + """An IOBufferProtocol which parses CMake's progress prefixes and emits corresponding STAGE_PROGRESS events.""" def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs): super(CMakeMakeIOBufferProtocol, self).__init__( @@ -220,7 +220,7 @@ def colorize_run_tests(self, line): def factory_factory(cls, verbose): """Factory factory for constructing protocols that know the verbosity.""" def factory(label, job_id, stage_label, event_queue, log_path): - # factory is called by caktin_tools executor + # factory is called by catkin_tools executor def init_proxy(*args, **kwargs): # init_proxy is called by asyncio return cls(label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs) diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index 37ddc69b..431c218f 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -24,7 +24,7 @@ class CommandMissing(Exception): - '''A required command is missing.''' + """A required command is missing.""" def __init__(self, name): super(CommandMissing, self).__init__( @@ -44,7 +44,7 @@ def get_env_loaders(package, context): if (context.install and context.isolate_install) or (not context.install and context.isolate_devel): # Source each package's install or devel space space = context.install_space_abs if context.install else context.devel_space_abs - # Get the recursive dependcies + # Get the recursive dependencies depends = get_cached_recursive_build_depends_in_workspace(package, context.packages) # For each dep add a line to source its setup file for dep_pth, dep in depends: @@ -62,12 +62,12 @@ def get_env_loaders(package, context): def merge_envs(job_env, overlay_envs): - ''' + """ In the merged/linked case of single env, this function amounts to a straight assignment, but a more complex merge is required with isolated result spaces, since a package's build environment may require extending that of multiple other result spaces. - ''' + """ merge_path_values = {} for overlay_env in overlay_envs: @@ -149,16 +149,16 @@ def rmfile(logger, event_queue, path): return 0 -def rmdirs(logger, event_queue, paths): +def rmdirs(logger, event_queue, paths, dry_run): """FunctionStage functor that removes a directory tree.""" - return rmfiles(logger, event_queue, paths, remove_empty=False) + return rmfiles(logger, event_queue, paths, dry_run, remove_empty=False) def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root='/'): """FunctionStage functor that removes a list of files and directories. If remove_empty is True, then this will also remove directories which - become emprt after deleting the files in `paths`. It will delete files up + become empty after deleting the files in `paths`. It will delete files up to the path specified by `empty_root`. """ @@ -201,7 +201,7 @@ def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root= if dir_descendants[path] == 0: paths.append(path) - # REmove the paths + # Remove the paths for index, path in enumerate(paths): # Remove the path diff --git a/catkin_tools/metadata.py b/catkin_tools/metadata.py index 9f17c88c..3e962140 100644 --- a/catkin_tools/metadata.py +++ b/catkin_tools/metadata.py @@ -99,7 +99,7 @@ def get_paths(workspace_path, profile_name, verb=None): # Get the metadata for this verb metadata_file_path = os.path.join(metadata_path, '%s.yaml' % verb) if profile_name and verb else None - return (metadata_path, metadata_file_path) + return metadata_path, metadata_file_path def find_enclosing_workspace(search_start_path): @@ -210,7 +210,7 @@ def init_metadata_root(workspace_path, reset=False): if not os.path.exists(workspace_path): raise IOError( "Can't initialize Catkin workspace in path %s because it does " - "not exist." % (workspace_path)) + "not exist." % workspace_path) # Check if the desired workspace is enclosed in another workspace marked_workspace = find_enclosing_workspace(workspace_path) @@ -228,7 +228,7 @@ def init_metadata_root(workspace_path, reset=False): if os.path.exists(metadata_root_path): # Reset the directory if requested if reset: - print("Deleting existing metadata from catkin_tools metadata directory: %s" % (metadata_root_path)) + print("Deleting existing metadata from catkin_tools metadata directory: %s" % metadata_root_path) shutil.rmtree(metadata_root_path) os.mkdir(metadata_root_path) else: @@ -254,6 +254,8 @@ def init_profile(workspace_path, profile_name, reset=False): :type workspace_path: str :param profile_name: The catkin_tools metadata profile name to initialize :type profile_name: str + :param reset: Delete profile with the same name if existing + :type reset: bool """ init_metadata_root(workspace_path) @@ -264,7 +266,7 @@ def init_profile(workspace_path, profile_name, reset=False): if os.path.exists(profile_path): # Reset the directory if requested if reset: - print("Deleting existing profile from catkin_tools profile directory: %s" % (profile_path)) + print("Deleting existing profile from catkin_tools profile directory: %s" % profile_path) shutil.rmtree(profile_path) os.mkdir(profile_path) else: @@ -396,7 +398,7 @@ def get_metadata(workspace_path, profile, verb): return yaml.safe_load(metadata_file) -def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, merge=True): +def update_metadata(workspace_path, profile, verb, new_data=None, no_init=False, merge=True): """Update the catkin_tools verb metadata for a given profile. :param workspace_path: The path to the root of a catkin workspace @@ -407,7 +409,13 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m :type verb: str :param new_data: A python dictionary or array to write to the metadata file :type new_data: dict + :param no_init: Do not init metadata root and/or profile folder (default: False) + :type no_init: bool + :param merge: Merge new data with current data or ignore current data (default: True) + :type merge: bool """ + if new_data is None: + new_data = {} migrate_metadata(workspace_path) @@ -418,7 +426,7 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m init_metadata_root(workspace_path) init_profile(workspace_path, profile) - # Get the curent metadata for this verb + # Get the current metadata for this verb if merge: data = get_metadata(workspace_path, profile, verb) else: @@ -430,7 +438,7 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m with open(metadata_file_path, 'w') as metadata_file: yaml.dump(data, metadata_file, default_flow_style=False) except PermissionError: - print("Could not write to metadata file '%s'!" % (metadata_file_path)) + print("Could not write to metadata file '%s'!" % metadata_file_path) return data @@ -451,7 +459,7 @@ def get_active_metadata(workspace_path, verb): get_metadata(workspace_path, active_profile, verb) -def update_active_metadata(workspace_path, verb, new_data={}): +def update_active_metadata(workspace_path, verb, new_data=None): """Update the catkin_tools verb metadata for the active profile. :param workspace_path: The path to the root of a catkin workspace @@ -461,6 +469,8 @@ def update_active_metadata(workspace_path, verb, new_data={}): :param new_data: A python dictionary or array to write to the metadata file :type new_data: dict """ + if new_data is None: + new_data = {} active_profile = get_active_profile(workspace_path) - update_active_metadata(workspace_path, active_profile, verb, new_data) + update_metadata(workspace_path, active_profile, verb, new_data) diff --git a/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown b/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown index b3ef53cf..521f625e 100644 --- a/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown +++ b/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown @@ -74,7 +74,7 @@ Some examples are: ``` $ echo 'Piped Message Data!' | terminal-notifier -sound default -$ terminal-notifier -title '💰' -message 'Check your Apple stock!' -open 'http://finance.yahoo.com/q?s=AAPL' +$ terminal-notifier -title '💰' -message 'Check your Apple stock!' -open 'https://finance.yahoo.com/q?s=AAPL' $ terminal-notifier -group 'address-book-sync' -title 'Address Book Sync' -subtitle 'Finished' -message 'Imported 42 contacts.' -activate 'com.apple.AddressBook' ``` diff --git a/catkin_tools/resultspace.py b/catkin_tools/resultspace.py index 7d98f2dc..6242e780 100644 --- a/catkin_tools/resultspace.py +++ b/catkin_tools/resultspace.py @@ -28,13 +28,15 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, cached=True, strict=True): - """Get the environemt variables which result from sourcing another catkin + """Get the environment variables which result from sourcing another catkin workspace's setup files as the string output of `cmake -E environment`. This cmake command is used to be as portable as possible. :param result_space_path: path to a Catkin result-space whose environment should be loaded, ``str`` :type result_space_path: str - :param quiet: don't throw exceptions, ``bool`` + :param base_env: Base environment dictionary (default: os.environ) + :type base_env: dict + :param quiet: don't throw exceptions :type quiet: bool :param cached: use the cached environment :type cached: bool @@ -124,7 +126,7 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c env_dict = {} try: - # Run the command synchronously to get the resultspace environmnet + # Run the command synchronously to get the resultspace environment lines = subprocess.check_output(command, cwd=os.getcwd(), env=base_env, shell=True) # Extract the environment variables @@ -151,11 +153,13 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c def load_resultspace_environment(result_space_path, base_env=None, cached=True): - """Load the environemt variables which result from sourcing another + """Load the environment variables which result from sourcing another workspace path into this process's environment. :param result_space_path: path to a Catkin result-space whose environment should be loaded, ``str`` :type result_space_path: str + :param base_env: Base environment dictionary (default: os.environ) + :type base_env: dict :param cached: use the cached environment :type cached: bool """ diff --git a/catkin_tools/terminal_color.py b/catkin_tools/terminal_color.py index 97ff6fd1..6eb139ed 100644 --- a/catkin_tools/terminal_color.py +++ b/catkin_tools/terminal_color.py @@ -153,7 +153,7 @@ class ColorMapper(object): functionality to convert them into colorized version. """ - # This map translates more human reable format strings into colorized versions + # This map translates more human readable format strings into colorized versions default_color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|') @@ -162,7 +162,7 @@ class ColorMapper(object): def __init__(self, color_map={}): """Create a color mapper with a given map. - :param color_map: A dictionary of format strings and colorized verisons + :param color_map: A dictionary of format strings and colorized versions :type color_map: dict """ self.color_map = ColorMapper.default_color_translation_map @@ -171,7 +171,7 @@ def __init__(self, color_map={}): def clr(self, key): """Returns a colorized version of the string given. - This is occomplished by either returning a hit from the color translation + This is accomplished by either returning a hit from the color translation map or by calling :py:func:`fmt` on the string and returning it. :param key: string to be colorized diff --git a/catkin_tools/utils.py b/catkin_tools/utils.py index e116fb81..af229c05 100644 --- a/catkin_tools/utils.py +++ b/catkin_tools/utils.py @@ -18,7 +18,7 @@ def which(program): """Custom version of the ``which`` built-in shell command. - Searches the pathes in the ``PATH`` environment variable for a given + Searches the paths in the ``PATH`` environment variable for a given executable name. It returns the full path to the first instance of the executable found or None if it was not found. diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index acfbb96f..5e8f2bde 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -63,6 +63,8 @@ def determine_packages_to_be_built(packages, context, workspace_packages): :type packages: list :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` + :param workspace_packages: list of all packages in the workspace + :type workspace_packages: list :returns: tuple of packages to be built and those package's deps :rtype: tuple """ @@ -206,10 +208,14 @@ def build_isolated_workspace( :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool + :param unbuilt: Handle unbuilt packages + :type unbuilt: bool :param n_jobs: number of parallel package build n_jobs :type n_jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool + :param pre_clean: Clean current build before building + :type pre_clean: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error @@ -671,7 +677,7 @@ def _create_unmerged_devel_setup_for_install(context): # This file is aggregates the many setup.sh files in the various # unmerged devel spaces in this folder. -# This is occomplished by sourcing each leaf package and all the +# This is accomplished by sourcing each leaf package and all the # recursive run dependencies of those leaf packages # Source the first package's setup.sh without the --extend option diff --git a/catkin_tools/verbs/catkin_build/color.py b/catkin_tools/verbs/catkin_build/color.py index 60382031..0c2d1a9c 100644 --- a/catkin_tools/verbs/catkin_build/color.py +++ b/catkin_tools/verbs/catkin_build/color.py @@ -19,7 +19,7 @@ from catkin_tools.terminal_color import sanitize from catkin_tools.terminal_color import ColorMapper -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|'), diff --git a/catkin_tools/verbs/catkin_clean/clean.py b/catkin_tools/verbs/catkin_clean/clean.py index 4c42ba03..9fbf05ad 100644 --- a/catkin_tools/verbs/catkin_clean/clean.py +++ b/catkin_tools/verbs/catkin_clean/clean.py @@ -45,6 +45,8 @@ def determine_packages_to_be_cleaned(context, include_dependents, packages): :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` + :param include_dependents: Also clean dependents of the packages to be cleaned + :type include_dependents: bool :param packages: list of package names to be cleaned :type packages: list :returns: full list of package names to be cleaned @@ -126,7 +128,7 @@ def clean_packages( # It's a problem if there aren't any build types available if len(clean_job_creators) == 0: - sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.') + sys.exit('Error: No build types available. Please check your catkin_tools installation.') # Determine the job parameters clean_job_kwargs = dict( diff --git a/catkin_tools/verbs/catkin_clean/color.py b/catkin_tools/verbs/catkin_clean/color.py index 3ab39dbc..d00ffa46 100644 --- a/catkin_tools/verbs/catkin_clean/color.py +++ b/catkin_tools/verbs/catkin_clean/color.py @@ -19,7 +19,7 @@ from catkin_tools.terminal_color import sanitize from catkin_tools.terminal_color import ColorMapper -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|'), diff --git a/catkin_tools/verbs/catkin_config/cli.py b/catkin_tools/verbs/catkin_config/cli.py index b49abf35..c2657c90 100644 --- a/catkin_tools/verbs/catkin_config/cli.py +++ b/catkin_tools/verbs/catkin_config/cli.py @@ -64,7 +64,7 @@ def prepare_arguments(parser): add('--maintainers', metavar=('NAME', 'EMAIL'), dest='maintainers', nargs='+', required=False, type=str, default=None, help='Set the default maintainers of created packages') - add('--licenses', metavar=('LICENSE'), dest='licenses', nargs='+', required=False, type=str, default=None, + add('--licenses', metavar='LICENSE', dest='licenses', nargs='+', required=False, type=str, default=None, help='Set the default licenses of created packages') lists_group = parser.add_argument_group( diff --git a/catkin_tools/verbs/catkin_create/cli.py b/catkin_tools/verbs/catkin_create/cli.py index a10fd5da..e35d09c9 100644 --- a/catkin_tools/verbs/catkin_create/cli.py +++ b/catkin_tools/verbs/catkin_create/cli.py @@ -33,14 +33,14 @@ def prepare_arguments(parser): parser_pkg.description = ( "Create a new Catkin package. Note that while the " "default options used by this command are sufficient for prototyping and " - "local usage, it is important that any publically-available packages have " + "local usage, it is important that any publicly-available packages have " "a valid license and a valid maintainer e-mail address.") add = parser_pkg.add_argument add('name', metavar='PKG_NAME', nargs='+', help='The name of one or more packages to create. This name should be ' - 'completely lower-case with individual words separated by undercores.') + 'completely lower-case with individual words separated by underscores.') add('-p', '--path', action='store', default=os.getcwd(), help='The path into which the package should be generated.') diff --git a/catkin_tools/verbs/catkin_env/cli.py b/catkin_tools/verbs/catkin_env/cli.py index 91ecb6f5..402c78c5 100644 --- a/catkin_tools/verbs/catkin_env/cli.py +++ b/catkin_tools/verbs/catkin_env/cli.py @@ -52,7 +52,7 @@ def argument_preprocessor(args): :param args: system arguments from which special arguments need to be extracted :type args: list - :returns: a tuple contianing a list of the arguments which can be handled + :returns: a tuple containing a list of the arguments which can be handled by argparse and a dict of the extra arguments which this function has extracted :rtype: tuple diff --git a/catkin_tools/verbs/catkin_init/cli.py b/catkin_tools/verbs/catkin_init/cli.py index 4c601742..2004a75d 100644 --- a/catkin_tools/verbs/catkin_init/cli.py +++ b/catkin_tools/verbs/catkin_init/cli.py @@ -40,7 +40,7 @@ def main(opts): # Initialize the workspace if necessary if ctx: - print('Catkin workspace `%s` is already initialized. No action taken.' % (ctx.workspace)) + print('Catkin workspace `%s` is already initialized. No action taken.' % ctx.workspace) else: print('Initializing catkin workspace in `%s`.' % (opts.workspace or os.getcwd())) # initialize the workspace diff --git a/catkin_tools/verbs/catkin_list/__init__.py b/catkin_tools/verbs/catkin_list/__init__.py index 74d360ed..98cff3dc 100644 --- a/catkin_tools/verbs/catkin_list/__init__.py +++ b/catkin_tools/verbs/catkin_list/__init__.py @@ -18,7 +18,7 @@ # This describes this command to the loader description = dict( verb='list', - description="Lists catkin packages in the workspace or other arbitray folders.", + description="Lists catkin packages in the workspace or other arbitrary folders.", main=main, prepare_arguments=prepare_arguments, ) diff --git a/catkin_tools/verbs/catkin_profile/cli.py b/catkin_tools/verbs/catkin_profile/cli.py index 63dac974..180157be 100644 --- a/catkin_tools/verbs/catkin_profile/cli.py +++ b/catkin_tools/verbs/catkin_profile/cli.py @@ -128,11 +128,11 @@ def main(opts): elif opts.subcommand == 'add': if opts.name in profiles: if opts.force: - print(clr('[profile] @{yf}Warning:@| Overwriting existing profile named @{cf}%s@|' % (opts.name))) + print(clr('[profile] @{yf}Warning:@| Overwriting existing profile named @{cf}%s@|' % opts.name)) else: print(clr('catkin profile: error: A profile named ' '@{cf}%s@| already exists. Use `--force` to ' - 'overwrite.' % (opts.name))) + 'overwrite.' % opts.name)) return 1 if opts.copy_active: ctx.profile = opts.name @@ -157,7 +157,7 @@ def main(opts): else: new_ctx = Context(workspace=ctx.workspace, profile=opts.name) Context.save(new_ctx) - print(clr('[profile] Created a new profile named @{cf}%s@| with default settings.' % (opts.name))) + print(clr('[profile] Created a new profile named @{cf}%s@| with default settings.' % opts.name)) profiles = get_profile_names(ctx.workspace) active_profile = get_active_profile(ctx.workspace) @@ -183,11 +183,11 @@ def main(opts): if opts.new_name in profiles: if opts.force: print(clr('[profile] @{yf}Warning:@| Overwriting ' - 'existing profile named @{cf}%s@|' % (opts.new_name))) + 'existing profile named @{cf}%s@|' % opts.new_name)) else: print(clr('catkin profile: error: A profile named ' '@{cf}%s@| already exists. Use `--force` to ' - 'overwrite.' % (opts.new_name))) + 'overwrite.' % opts.new_name)) return 1 ctx.profile = opts.new_name Context.save(ctx) diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py index 94b5e760..4d99342a 100644 --- a/tests/unit/test_jobs.py +++ b/tests/unit/test_jobs.py @@ -2,11 +2,11 @@ def test_merge_envs_basic(): - job_env = { 'PATH': '/usr/local/bin:/usr/bin', 'FOO': 'foo' } + job_env = {'PATH': '/usr/local/bin:/usr/bin', 'FOO': 'foo'} merge_envs(job_env, [ - { 'PATH': '/usr/local/bin:/bar/baz/bin' }, - { 'BAR': 'bar' } ]) + {'PATH': '/usr/local/bin:/bar/baz/bin'}, + {'BAR': 'bar'}]) # Validate that the known path was not moved from the existing order, and the unfamiliar # path was correctly prepended. @@ -20,14 +20,14 @@ def test_merge_envs_basic(): def test_merge_envs_complex(): - ''' Confirm that merged paths are deduplicated and that order is maintained. ''' - job_env = { 'PATH': 'C:B:A' } - merge_envs(job_env, [{ 'PATH': 'D:C' }, { 'PATH': 'E:A:C' }]) + """Confirm that merged paths are deduplicated and that order is maintained.""" + job_env = {'PATH': 'C:B:A'} + merge_envs(job_env, [{'PATH': 'D:C'}, {'PATH': 'E:A:C'}]) assert job_env['PATH'] == 'E:D:C:B:A', job_env['PATH'] def test_merge_envs_nonpaths(): - ''' Confirm that non-path vars are simply overwritten on a last-wins policy. ''' - job_env = { 'FOO': 'foo:bar' } - merge_envs(job_env, [{ 'FOO': 'bar:baz' }, { 'FOO': 'baz:bar' }]) + """Confirm that non-path vars are simply overwritten on a last-wins policy.""" + job_env = {'FOO': 'foo:bar'} + merge_envs(job_env, [{'FOO': 'bar:baz'}, {'FOO': 'baz:bar'}]) assert job_env['FOO'] == 'baz:bar' From 6895d4b3fb0e5a6a611e317045f8d4ee988c0055 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Fri, 15 Oct 2021 12:28:41 +0200 Subject: [PATCH 58/68] add arguments to catkin_test for backwards compatibility (#693) --- catkin_tools/verbs/catkin_test/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index f9af6b74..c71d3442 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -9,6 +9,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import argparse import os import sys @@ -77,6 +78,13 @@ def prepare_arguments(parser): add('--no-notify', action='store_true', default=False, help='Suppresses system pop-up notification.') + # Deprecated arguments + # colors are handled by the main catkin command + add('--no-color', action='store_true', help=argparse.SUPPRESS) + add('--force-color', action='store_true', help=argparse.SUPPRESS) + # no-deps became the default + add('--no-deps', action='store_true', help=argparse.SUPPRESS) + return parser From 116e2038cbc6195222ac99d88aebb13ac7f8bede Mon Sep 17 00:00:00 2001 From: Balint <75276107+balint-bg@users.noreply.github.com> Date: Sun, 17 Oct 2021 14:54:30 -0400 Subject: [PATCH 59/68] Improve wording in documentation (#694) Just noticed a couple of words out-of-order. --- docs/quick_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick_start.rst b/docs/quick_start.rst index 7531d5ae..9dc073b7 100644 --- a/docs/quick_start.rst +++ b/docs/quick_start.rst @@ -80,7 +80,7 @@ Now that there are some packages in the workspace, Catkin has something to build .. note:: Catkin utilizes an "out-of-source" and "aggregated" build pattern. - This means that temporary or final build will products never be placed in a package's source directory (or anywhere in the **source space**. + This means that temporary or final build products will never be placed in a package's source directory (or anywhere in the **source space**. Instead all build directories are aggregated in the **build space** and all final build products like executables, libraries, etc., will be put in the **devel space**. Building the Workspace From e8cd5a2af20104f260bc6c6f369ed09259eef15f Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Thu, 18 Nov 2021 14:50:18 +0100 Subject: [PATCH 60/68] Add changelog of version 0.7.2 --- CHANGELOG.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a8f83b6a..e3f0b772 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,13 @@ Changelog for package catkin_tools Upcoming -------- +0.7.2 (2021-11-18) +------------------ + +* Update installation docs for python 3 (#687) +* Fix environment variable parsing (#686) +* Switch from Travis CI to GitHub actions (#684) +* Regenerate setup files when the install space was cleaned (#682) * Add possibility to clean individual package with isolated devel space (#683) * Fix regeneration of setup file when the install space was cleaned (#682) * Fix workspace generation with catkin build --this and --start-with-this (#685) diff --git a/setup.py b/setup.py index deb787e7..b7b8feef 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): setup( name='catkin_tools', - version='0.7.1', + version='0.7.2', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={ From 374febcc0a824cce34049f597a3b349ad1d2d392 Mon Sep 17 00:00:00 2001 From: Jim Aldon D'Souza Date: Fri, 26 Nov 2021 13:53:55 +0000 Subject: [PATCH 61/68] Added doc for custom build, devel, install dirs (#697) --- docs/migration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration.rst b/docs/migration.rst index d972444a..3a15a66a 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -52,7 +52,7 @@ To make iterating easier, use ``catkin_make`` with build and devel spaces with t .. code-block:: bash cd /path/to/ws - catkin_make --cmake-args [CMAKE_ARGS...] --make-args [MAKE_ARGS...] + catkin_make --build build_cm --cmake-args -DCATKIN_DEVEL_PREFIX=devel_cm -DCMAKE_INSTALL_PREFIX=install_cm [CMAKE_ARGS...] --make-args [MAKE_ARGS...] If your packages build and other appropriate tests pass, continue to the next step. From 5163ee2d8edb800f45340f84342a0f3fb8c6d889 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 30 Nov 2021 00:15:12 +0100 Subject: [PATCH 62/68] Some space fixes (#691) * Add SPACE_missing subs dynamically * Make catkin locate accept other spaces than default ones * Make catkin clean accept other spaces than default ones * clean: use get_metadata_root_path * (clean) set context.packages * context: Add package_source_space * context: add package_SPACE_space dynamically if not existing * Align messages in spaces to clean Co-authored-by: Timon Engelke Co-authored-by: Timon Engelke --- catkin_tools/context.py | 37 ++++-- catkin_tools/verbs/catkin_clean/clean.py | 2 + catkin_tools/verbs/catkin_clean/cli.py | 143 +++++++++++------------ catkin_tools/verbs/catkin_locate/cli.py | 39 +++---- 4 files changed, 112 insertions(+), 109 deletions(-) diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 7fb1302c..3783f563 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -99,10 +99,21 @@ def space_exists(self): space_abs = getattr(self, '__%s_space_abs' % space) return os.path.exists(space_abs) and os.path.isdir(space_abs) + def package_space(self, package): + """ + Get the package specific path in a space + """ + space_abs = getattr(self, '__%s_space_abs' % space) + return os.path.join(space_abs, package.name) + setattr(cls, '%s_space' % space, property(space_getter, space_setter)) setattr(cls, '%s_space_abs' % space, property(space_abs_getter)) setattr(cls, '%s_space_exists' % space, space_exists) + package_space_name = "package_%s_space" % space + if not hasattr(cls, package_space_name): + setattr(cls, package_space_name, package_space) + @classmethod def setup_space_keys(cls): """ @@ -600,15 +611,18 @@ def quote(argument): 'cmake_args': ' '.join([quote(a) for a in self.cmake_args or ['None']]), 'make_args': ' '.join(self.make_args + self.jobs_args or ['None']), 'catkin_make_args': ', '.join(self.catkin_make_args or ['None']), - 'source_missing': existence_str(self.source_space_abs), - 'log_missing': existence_str(self.log_space_abs), - 'build_missing': existence_str(self.build_space_abs), - 'devel_missing': existence_str(self.devel_space_abs), - 'install_missing': existence_str(self.install_space_abs, used=self.install), 'destdir_missing': existence_str(self.destdir, used=self.destdir), 'whitelisted_packages': ' '.join(self.whitelist or ['None']), 'blacklisted_packages': ' '.join(self.blacklist or ['None']), } + for space, space_dict in sorted(Context.SPACES.items()): + key_missing = '{}_missing'.format(space) + space_abs = getattr(self, '{}_space_abs'.format(space)) + if hasattr(self, space): + subs[key_missing] = existence_str(space_abs, used=getattr(self, space)) + else: + subs[key_missing] = existence_str(space_abs) + subs.update(**self.__dict__) # Get the width of the shell width = terminal_width() @@ -856,9 +870,16 @@ def package_private_devel_path(self, package): """The path to the linked devel space for a given package.""" return os.path.join(self.private_devel_path, package.name) - def package_build_space(self, package): - """Get the build directory for a specific package.""" - return os.path.join(self.build_space_abs, package.name) + def package_source_space(self, package): + """Get the source directory of a specific package.""" + for pkg_name, pkg in self.packages: + if pkg_name == package.name: + pkg_dir = os.path.dirname(pkg.filename) + # Need to check if the pkg_dir is the source space as it can also be loaded from the metadata + if os.path.commonpath([self.source_space_abs, pkg_dir]) == self.source_space_abs: + return pkg_dir + + return None def package_devel_space(self, package): """Get the devel directory for a specific package. diff --git a/catkin_tools/verbs/catkin_clean/clean.py b/catkin_tools/verbs/catkin_clean/clean.py index 9fbf05ad..996a2332 100644 --- a/catkin_tools/verbs/catkin_clean/clean.py +++ b/catkin_tools/verbs/catkin_clean/clean.py @@ -57,6 +57,8 @@ def determine_packages_to_be_cleaned(context, include_dependents, packages): workspace_packages = find_packages(context.package_metadata_path(), exclude_subspaces=True, warnings=[]) # Order the packages by topology ordered_packages = topological_order_packages(workspace_packages) + # Set the packages in the workspace for the context + context.packages = ordered_packages # Create a dict of all packages in the workspace by name workspace_packages_by_name = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index 9432aaea..fb7dc690 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -31,10 +31,10 @@ import catkin_tools.execution.job_server as job_server from catkin_tools.metadata import find_enclosing_workspace +from catkin_tools.metadata import get_metadata_root_path from catkin_tools.metadata import get_paths as get_metadata_paths from catkin_tools.metadata import get_profile_names from catkin_tools.metadata import update_metadata -from catkin_tools.metadata import METADATA_DIR_NAME from catkin_tools.terminal_color import ColorMapper @@ -104,18 +104,18 @@ def prepare_arguments(parser): ' the workspace.') # Basic group - basic_group = parser.add_argument_group( + spaces_group = parser.add_argument_group( 'Spaces', 'Clean workspace subdirectories for the selected profile.') - add = basic_group.add_argument - add('-l', '--logs', action='store_true', default=False, - help='Remove the entire log space.') - add('-b', '--build', action='store_true', default=False, - help='Remove the entire build space.') - add('-d', '--devel', action='store_true', default=False, - help='Remove the entire devel space.') - add('-i', '--install', action='store_true', default=False, - help='Remove the entire install space.') + Context.setup_space_keys() + add = spaces_group.add_argument + for space, space_dict in Context.SPACES.items(): + if space == 'source': + continue + flags = ['--{}-space'.format(space)] + flags.extend([space_dict['short_flag']] if 'short_flag' in space_dict else []) + add(*flags, dest='spaces', action='append_const', const=space, + help='Remove the entire {} space.'.format(space)) # Packages group packages_group = parser.add_argument_group( @@ -168,20 +168,22 @@ def clean_profile(opts, profile): profile = ctx.profile # Check if the user wants to do something explicit - actions = [ - 'build', 'devel', 'install', 'logs', - 'packages', 'clean_this', 'orphans', - 'deinit', 'setup_files'] + actions = ['spaces', 'packages', 'clean_this', 'orphans', 'deinit', 'setup_files'] - logs_exists = os.path.exists(ctx.log_space_abs) - build_exists = os.path.exists(ctx.build_space_abs) - devel_exists = os.path.exists(ctx.devel_space_abs) + paths = {} # noqa + paths_exists = {} # noqa - install_path = ( + paths['install'] = ( os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep)) if ctx.destdir else ctx.install_space_abs) - install_exists = os.path.exists(install_path) + paths_exists['install'] = os.path.exists(paths['install']) and os.path.isdir(paths['install']) + + for space in Context.SPACES.keys(): + if space in paths: + continue + paths[space] = getattr(ctx, '{}_space_abs'.format(space)) + paths_exists[space] = getattr(ctx, '{}_space_exists'.format(space))() # Default is to clean all products for this profile no_specific_action = not any([ @@ -191,21 +193,17 @@ def clean_profile(opts, profile): # Initialize action options if clean_all: - opts.logs = opts.build = opts.devel = opts.install = True + opts.spaces = [k for k in Context.SPACES.keys() if k != 'source'] - # Make sure the user intends to clena everything - spaces_to_clean = (opts.logs or opts.build or opts.devel or opts.install) + # Make sure the user intends to clean everything spaces_to_clean_msgs = [] - if spaces_to_clean and not (opts.yes or opts.dry_run): - if opts.logs and logs_exists: - spaces_to_clean_msgs.append(clr("[clean] Log Space: @{yf}{}").format(ctx.log_space_abs)) - if opts.build and build_exists: - spaces_to_clean_msgs.append(clr("[clean] Build Space: @{yf}{}").format(ctx.build_space_abs)) - if opts.devel and devel_exists: - spaces_to_clean_msgs.append(clr("[clean] Devel Space: @{yf}{}").format(ctx.devel_space_abs)) - if opts.install and install_exists: - spaces_to_clean_msgs.append(clr("[clean] Install Space: @{yf}{}").format(install_path)) + if opts.spaces and not (opts.yes or opts.dry_run): + for space in opts.spaces: + if getattr(ctx, '{}_space_exists'.format(space))(): + space_name = Context.SPACES[space]['space'] + space_abs = getattr(ctx, '{}_space_abs'.format(space)) + spaces_to_clean_msgs.append(clr("[clean] {:14} @{yf}{}").format(space_name + ':', space_abs)) if len(spaces_to_clean_msgs) == 0 and not opts.deinit: log("[clean] Nothing to be cleaned for profile: `{}`".format(profile)) @@ -232,50 +230,41 @@ def clean_profile(opts, profile): needs_force = False try: - # Remove all installspace files - if opts.install and install_exists: - log("[clean] Removing installspace: %s" % install_path) - if not opts.dry_run: - safe_rmtree(install_path, ctx.workspace, opts.force) - - # Remove all develspace files - if opts.devel: - if devel_exists: - log("[clean] Removing develspace: %s" % ctx.devel_space_abs) - if not opts.dry_run: - safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) - # Clear the cached metadata from the last build run - _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') - if os.path.exists(build_metadata_file): - os.unlink(build_metadata_file) - # Clear the cached packages data, if it exists - packages_metadata_path = ctx.package_metadata_path() - if os.path.exists(packages_metadata_path): - safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) - - # Remove all buildspace files - if opts.build and build_exists: - log("[clean] Removing buildspace: %s" % ctx.build_space_abs) - if not opts.dry_run: - safe_rmtree(ctx.build_space_abs, ctx.workspace, opts.force) + for space in opts.spaces: + if space == 'devel': + # Remove all develspace files + if paths_exists['devel']: + log("[clean] Removing {}: {}".format(Context.SPACES['devel']['space'], ctx.devel_space_abs)) + if not opts.dry_run: + safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) + # Clear the cached metadata from the last build run + _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') + if os.path.exists(build_metadata_file): + os.unlink(build_metadata_file) + # Clear the cached packages data, if it exists + packages_metadata_path = ctx.package_metadata_path() + if os.path.exists(packages_metadata_path): + safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) + + else: + if paths_exists[space]: + space_name = Context.SPACES[space]['space'] + space_path = paths[space] + log("[clean] Removing {}: {}".format(space_name, space_path)) + if not opts.dry_run: + safe_rmtree(space_path, ctx.workspace, opts.force) # Setup file removal if opts.setup_files: - if devel_exists: - log("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs) + if paths_exists['devel']: + log("[clean] Removing setup files from {}: {}".format(Context.SPACES['devel']['space'], paths['devel'])) opts.packages.append('catkin') opts.packages.append('catkin_tools_prebuild') else: - log("[clean] No develspace exists, no setup files to clean.") - - # Clean log files - if opts.logs and logs_exists: - log("[clean] Removing log space: {}".format(ctx.log_space_abs)) - if not opts.dry_run: - safe_rmtree(ctx.log_space_abs, ctx.workspace, opts.force) + log("[clean] No {} exists, no setup files to clean.".format(Context.SPACES['devel']['space'])) # Find orphaned packages - if ctx.link_devel or ctx.isolate_devel and not any([opts.build, opts.devel]): + if ctx.link_devel or ctx.isolate_devel and not ('devel' in opts.spaces or 'build' in opts.spaces): if opts.orphans: if os.path.exists(ctx.build_space_abs): log("[clean] Determining orphaned packages...") @@ -299,14 +288,14 @@ def clean_profile(opts, profile): else: log("[clean] No orphans in the workspace.") else: - log("[clean] No buildspace exists, no potential for orphans.") + log("[clean] No {} exists, no potential for orphans.".format(Context.SPACES['build']['space'])) # Remove specific packages if len(opts.packages) > 0 or opts.clean_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) - # Suppress warnings since this won't necessaraly find all packages + # Suppress warnings since this won't necessarily find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( @@ -361,21 +350,23 @@ def clean_profile(opts, profile): def main(opts): # Check for exclusivity full_options = opts.deinit - space_options = opts.logs or opts.build or opts.devel or opts.install package_options = len(opts.packages) > 0 or opts.orphans or opts.clean_this advanced_options = opts.setup_files + if opts.spaces is None: + opts.spaces = [] + if full_options: - if space_options or package_options or advanced_options: + if opts.spaces or package_options or advanced_options: log("[clean] Error: Using `--deinit` will remove all spaces, so" " additional partial cleaning options will be ignored.") - elif space_options: + elif opts.spaces: if package_options: log("[clean] Error: Package arguments are not allowed with space" - " arguments (--build, --devel, --install, --logs). See usage.") + " arguments. See usage.") elif advanced_options: log("[clean] Error: Advanced arguments are not allowed with space" - " arguments (--build, --devel, --install, --logs). See usage.") + " arguments. See usage.") # Check for all profiles option if opts.all_profiles: @@ -413,7 +404,7 @@ def main(opts): # Nuke .catkin_tools if opts.deinit: ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False) - metadata_dir = os.path.join(ctx.workspace, METADATA_DIR_NAME) + metadata_dir = get_metadata_root_path(ctx.workspace) log("[clean] Deinitializing workspace by removing catkin_tools config: %s" % metadata_dir) if not opts.dry_run: safe_rmtree(metadata_dir, ctx.workspace, opts.force) diff --git a/catkin_tools/verbs/catkin_locate/cli.py b/catkin_tools/verbs/catkin_locate/cli.py index 88fd60db..08e6b7e3 100644 --- a/catkin_tools/verbs/catkin_locate/cli.py +++ b/catkin_tools/verbs/catkin_locate/cli.py @@ -45,20 +45,17 @@ def prepare_arguments(parser): help="Suppress warning output.") # Path options - dir_group = parser.add_argument_group( + spaces_group = parser.add_argument_group( 'Sub-Space Options', 'Get the absolute path to one of the following locations in the given ' 'workspace with the given profile.') - dir_group_mut = dir_group.add_mutually_exclusive_group() - add = dir_group_mut.add_argument - add('-s', '--src', dest='space', action='store_const', const='src', - help="Get the path to the source space.") - add('-b', '--build', dest='space', action='store_const', const='build', - help="Get the path to the build space.") - add('-d', '--devel', dest='space', action='store_const', const='devel', - help="Get the path to the devel space.") - add('-i', '--install', dest='space', action='store_const', const='install', - help="Get the path to the install space.") + Context.setup_space_keys() + add = spaces_group.add_mutually_exclusive_group().add_argument + for space, space_dict in Context.SPACES.items(): + flags = ['--{}-space'.format(space)] + flags.extend([space_dict['short_flag']] if 'short_flag' in space_dict else []) + add(*flags, dest='space', action='store_const', const=space, + help='Get the path to the {} space.'.format(space)) pkg_group = parser.add_argument_group( 'Package Directories', @@ -120,15 +117,7 @@ def main(opts): path = None if opts.space: - # Get the subspace - if opts.space == 'src': - path = ctx.source_space_abs - elif opts.space == 'build': - path = ctx.build_space_abs - elif opts.space == 'devel': - path = ctx.devel_space_abs - elif opts.space == 'install': - path = ctx.install_space_abs + path = getattr(ctx, "{}_space_abs".format(opts.space)) package = None if opts.package or opts.this: @@ -148,11 +137,7 @@ def main(opts): package = opts.package # Get the path to the given package path = path or ctx.source_space_abs - if opts.space == 'build': - path = os.path.join(path, package) - elif opts.space in ['devel', 'install']: - path = os.path.join(path, 'share', package) - else: + if not opts.space or opts.space == 'source': try: packages = find_packages(path, warnings=[]) catkin_package = [pkg_path for pkg_path, p in packages.items() if p.name == package] @@ -163,6 +148,10 @@ def main(opts): (package, path))) except RuntimeError as e: sys.exit(clr('@{rf}ERROR: %s@|' % str(e))) + elif opts.space in ['devel', 'install']: + path = os.path.join(path, 'share', package) + else: + path = os.path.join(path, package) if not opts.space and package is None: # Get the path to the workspace root From cd83bb17a87ea6eb31934d4896730b71533cfbda Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 30 Nov 2021 00:33:10 +0100 Subject: [PATCH 63/68] 0.8.0 --- CHANGELOG.rst | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e3f0b772..e0282dfb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,16 @@ Changelog for package catkin_tools Upcoming -------- +0.8.0 (2021-11-30) +------------------ + +* Add catkin test verb (#676) +* Code cleanup (#692) +* Documentation: https in docs, properties over vars, sphinx cleanup (#690) +* Documentation: Improve wording (#694) +* Documentation: Add custom build, devel, install dirs (#697) +* Contributors: Balint, Jim Aldon D'Souza, Matthijs van der Burgh, mobangjack, pseyfert, Timon Engelke + 0.7.2 (2021-11-18) ------------------ diff --git a/setup.py b/setup.py index b7b8feef..0ace5af2 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): setup( name='catkin_tools', - version='0.7.2', + version='0.8.0', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={ From 3b4034c882442d45acd69ff29d730364f3bfa271 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 30 Nov 2021 13:09:50 +0100 Subject: [PATCH 64/68] test: add --limit-status-rate option (#699) --- catkin_tools/verbs/catkin_build/build.py | 2 +- catkin_tools/verbs/catkin_test/cli.py | 15 +++++++++++++-- catkin_tools/verbs/catkin_test/test.py | 8 ++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index 5e8f2bde..eca6dfc7 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -243,7 +243,7 @@ def build_isolated_workspace( # Assert that the limit_status_rate is valid if limit_status_rate < 0: - sys.exit("[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero.") + sys.exit("[build] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero.") # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py index c71d3442..a9554439 100644 --- a/catkin_tools/verbs/catkin_test/cli.py +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -71,10 +71,20 @@ def prepare_arguments(parser): help='Print output from commands in ordered blocks once the command finishes.') add('--interleave-output', '-i', action='store_true', default=False, help='Prevents ordering of command output when multiple commands are running at the same time.') - add('--no-status', action='store_true', default=False, - help='Suppresses status line, useful in situations where carriage return is not properly supported.') add('--summarize', '--summary', '-s', action='store_true', default=None, help='Adds a summary to the end of the log') + add('--no-status', action='store_true', default=False, + help='Suppresses status line, useful in situations where carriage return is not properly supported.') + + def status_rate_type(rate): + rate = float(rate) + if rate < 0: + raise argparse.ArgumentTypeError("must be greater than or equal to zero.") + return rate + + add('--limit-status-rate', '--status-rate', type=status_rate_type, default=10.0, + help='Limit the update rate of the status bar to this frequency. Zero means unlimited. ' + 'Must be positive, default is 10 Hz.') add('--no-notify', action='store_true', default=False, help='Suppresses system pop-up notification.') @@ -160,6 +170,7 @@ def main(opts): quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, + limit_status_rate=opts.limit_status_rate, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize, diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py index 2209fc4b..87a3f006 100644 --- a/catkin_tools/verbs/catkin_test/test.py +++ b/catkin_tools/verbs/catkin_test/test.py @@ -32,6 +32,7 @@ def test_workspace( quiet=False, interleave_output=False, no_status=False, + limit_status_rate=10.0, no_notify=False, continue_on_failure=False, summarize_build=False, @@ -52,6 +53,8 @@ def test_workspace( :type interleave_output: bool :param no_status: suppresses the bottom status line :type no_status: bool + :param limit_status_rate: rate to which status updates are limited; the default 0, places no limit. + :type limit_status_rate: float :param no_notify: suppresses system notifications :type no_notify: bool :param continue_on_failure: do not stop testing other packages on error @@ -65,6 +68,10 @@ def test_workspace( """ pre_start_time = time.time() + # Assert that the limit_status_rate is valid + if limit_status_rate < 0: + sys.exit("[test] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero.") + # Get all the packages in the context source space # Suppress warnings since this is a utility function try: @@ -193,6 +200,7 @@ def test_workspace( show_full_summary=summarize_build, show_stage_events=not quiet, pre_start_time=pre_start_time, + active_status_rate=limit_status_rate, ) status_thread.start() From 6d9f6fdc86c56850e521e6a665b89b033612a2f8 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 30 Nov 2021 13:10:37 +0100 Subject: [PATCH 65/68] 0.8.1 --- CHANGELOG.rst | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0282dfb..6555dd89 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,11 @@ Changelog for package catkin_tools Upcoming -------- +0.8.1 (2021-11-30) +------------------ + +* catkin test: add --limit-status-rate option (#699) + 0.8.0 (2021-11-30) ------------------ diff --git a/setup.py b/setup.py index 0ace5af2..9330ffd0 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): setup( name='catkin_tools', - version='0.8.0', + version='0.8.1', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={ From 6f7c54d21909f406cd935cee53a425e69ea098b4 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Wed, 1 Dec 2021 13:42:40 +0100 Subject: [PATCH 66/68] Add back flags that were removed in #691 (#702) * add back flags that were removed in #691 * put the short flag before the long flag --- catkin_tools/verbs/catkin_clean/cli.py | 5 +++-- catkin_tools/verbs/catkin_locate/cli.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index fb7dc690..44c0b653 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -112,8 +112,9 @@ def prepare_arguments(parser): for space, space_dict in Context.SPACES.items(): if space == 'source': continue - flags = ['--{}-space'.format(space)] - flags.extend([space_dict['short_flag']] if 'short_flag' in space_dict else []) + flags = [space_dict['short_flag']] if 'short_flag' in space_dict else [] + flags.append('--{}'.format(space_dict['default'])) + flags.append('--{}-space'.format(space)) add(*flags, dest='spaces', action='append_const', const=space, help='Remove the entire {} space.'.format(space)) diff --git a/catkin_tools/verbs/catkin_locate/cli.py b/catkin_tools/verbs/catkin_locate/cli.py index 08e6b7e3..091b3c81 100644 --- a/catkin_tools/verbs/catkin_locate/cli.py +++ b/catkin_tools/verbs/catkin_locate/cli.py @@ -52,8 +52,9 @@ def prepare_arguments(parser): Context.setup_space_keys() add = spaces_group.add_mutually_exclusive_group().add_argument for space, space_dict in Context.SPACES.items(): - flags = ['--{}-space'.format(space)] - flags.extend([space_dict['short_flag']] if 'short_flag' in space_dict else []) + flags = [space_dict['short_flag']] if 'short_flag' in space_dict else [] + flags.append('--{}'.format(space_dict['default'])) + flags.append('--{}-space'.format(space)) add(*flags, dest='space', action='store_const', const=space, help='Get the path to the {} space.'.format(space)) From 6b78becef628a56013bd946ff1384cb5eb657aa2 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Wed, 1 Dec 2021 13:43:05 +0100 Subject: [PATCH 67/68] update CLI documentation --- docs/verbs/cli/catkin_build.txt | 10 +++++++--- docs/verbs/cli/catkin_clean.txt | 16 ++++++++++------ docs/verbs/cli/catkin_config.txt | 24 ++++++++++++++---------- docs/verbs/cli/catkin_create_pkg.txt | 17 ++++++++--------- docs/verbs/cli/catkin_env.txt | 6 +++--- docs/verbs/cli/catkin_list.txt | 15 +++++++-------- docs/verbs/cli/catkin_locate.txt | 18 ++++++++++++------ docs/verbs/cli/catkin_profile_add.txt | 18 +++++++++++------- docs/verbs/cli/catkin_profile_remove.txt | 2 +- docs/verbs/cli/catkin_test.txt | 13 +++++++++---- 10 files changed, 82 insertions(+), 57 deletions(-) diff --git a/docs/verbs/cli/catkin_build.txt b/docs/verbs/cli/catkin_build.txt index 9bf5b229..c511ea4b 100644 --- a/docs/verbs/cli/catkin_build.txt +++ b/docs/verbs/cli/catkin_build.txt @@ -3,7 +3,8 @@ usage: catkin build [-h] [--workspace WORKSPACE] [--profile PROFILE] [--unbuilt] [--start-with PKGNAME | --start-with-this] [--continue-on-failure] [--force-cmake] [--pre-clean] [--no-install-lock] [--save-config] [-j JOBS] - [-p PACKAGE_JOBS] [--jobserver | --no-jobserver] + [-p PACKAGE_JOBS] [-l LOAD_AVERAGE] + [--jobserver | --no-jobserver] [--env-cache | --no-env-cache] [--cmake-args ARG [ARG ...] | --no-cmake-args] [--make-args ARG [ARG ...] | --no-make-args] [--catkin-make-args ARG [ARG ...] | @@ -11,7 +12,7 @@ usage: catkin build [-h] [--workspace WORKSPACE] [--profile PROFILE] [--no-status] [--summarize] [--no-summarize] [--override-build-tool-check] [--limit-status-rate LIMIT_STATUS_RATE] [--no-notify] - [PKGNAME [PKGNAME ...]] + [PKGNAME ...] Build one or more packages in a catkin workspace. This invokes `CMake`, `make`, and optionally `make install` for either all or the specified packages @@ -69,6 +70,9 @@ Config: -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS Maximum number of packages allowed to be built in parallel (default is cpu count) + -l LOAD_AVERAGE, --load-average LOAD_AVERAGE + Maximum load average before no new build jobs are + scheduled --jobserver Use the internal GNU Make job server which will limit the number of Make jobs across all active packages. --no-jobserver Disable the internal GNU Make job server, and use an @@ -113,7 +117,7 @@ Interface: --no-summarize, --no-summary Explicitly disable the end of build summary --override-build-tool-check - use to override failure due to using differnt build + use to override failure due to using different build tools on the same workspace. --limit-status-rate LIMIT_STATUS_RATE, --status-rate LIMIT_STATUS_RATE Limit the update rate of the status bar to this diff --git a/docs/verbs/cli/catkin_clean.txt b/docs/verbs/cli/catkin_clean.txt index f0ec5c32..a270e6b3 100644 --- a/docs/verbs/cli/catkin_clean.txt +++ b/docs/verbs/cli/catkin_clean.txt @@ -1,8 +1,8 @@ usage: catkin clean [-h] [--workspace WORKSPACE] [--profile PROFILE] [--dry-run] [--verbose] [--yes] [--force] [--all-profiles] - [--deinit] [-l] [-b] [-d] [-i] [--this] [--dependents] + [--deinit] [-b] [-d] [-i] [-L] [--this] [--dependents] [--orphans] [--setup-files] - [PKGNAME [PKGNAME ...]] + [PKGNAME ...] Deletes various products of the build verb. @@ -31,10 +31,14 @@ Full: Spaces: Clean workspace subdirectories for the selected profile. - -l, --logs Remove the entire log space. - -b, --build Remove the entire build space. - -d, --devel Remove the entire devel space. - -i, --install Remove the entire install space. + -b, --build, --build-space + Remove the entire build space. + -d, --devel, --devel-space + Remove the entire devel space. + -i, --install, --install-space + Remove the entire install space. + -L, --logs, --log-space + Remove the entire log space. Packages: Clean products from specific packages in the workspace. Note that these diff --git a/docs/verbs/cli/catkin_config.txt b/docs/verbs/cli/catkin_config.txt index 5aa928f4..2c740730 100644 --- a/docs/verbs/cli/catkin_config.txt +++ b/docs/verbs/cli/catkin_config.txt @@ -6,15 +6,16 @@ usage: catkin config [-h] [--workspace WORKSPACE] [--profile PROFILE] [--whitelist PKG [PKG ...] | --no-whitelist] [--blacklist PKG [PKG ...] | --no-blacklist] [--build-space BUILD_SPACE | --default-build-space] - [--log-space LOG_SPACE | --default-log-space] - [--install-space INSTALL_SPACE | --default-install-space] [--devel-space DEVEL_SPACE | --default-devel-space] + [--install-space INSTALL_SPACE | --default-install-space] + [--log-space LOG_SPACE | --default-log-space] [--source-space SOURCE_SPACE | --default-source-space] [-x SPACE_SUFFIX] [--link-devel | --merge-devel | --isolate-devel] [--install | --no-install] [--isolate-install | --merge-install] [-j JOBS] - [-p PACKAGE_JOBS] [--jobserver | --no-jobserver] + [-p PACKAGE_JOBS] [-l LOAD_AVERAGE] + [--jobserver | --no-jobserver] [--env-cache | --no-env-cache] [--cmake-args ARG [ARG ...] | --no-cmake-args] [--make-args ARG [ARG ...] | --no-make-args] @@ -83,17 +84,17 @@ Spaces: The path to the build space. --default-build-space Use the default path to the build space ("build") - --log-space LOG_SPACE, -l LOG_SPACE - The path to the log space. - --default-log-space Use the default path to the log space ("logs") - --install-space INSTALL_SPACE, -i INSTALL_SPACE - The path to the install space. - --default-install-space - Use the default path to the install space ("install") --devel-space DEVEL_SPACE, -d DEVEL_SPACE The path to the devel space. --default-devel-space Use the default path to the devel space ("devel") + --install-space INSTALL_SPACE, -i INSTALL_SPACE + The path to the install space. + --default-install-space + Use the default path to the install space ("install") + --log-space LOG_SPACE, -L LOG_SPACE + The path to the log space. + --default-log-space Use the default path to the log space ("logs") --source-space SOURCE_SPACE, -s SOURCE_SPACE The path to the source space. --default-source-space @@ -133,6 +134,9 @@ Build Options: -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS Maximum number of packages allowed to be built in parallel (default is cpu count) + -l LOAD_AVERAGE, --load-average LOAD_AVERAGE + Maximum load average before no new build jobs are + scheduled --jobserver Use the internal GNU Make job server which will limit the number of Make jobs across all active packages. --no-jobserver Disable the internal GNU Make job server, and use an diff --git a/docs/verbs/cli/catkin_create_pkg.txt b/docs/verbs/cli/catkin_create_pkg.txt index 94cd79fc..0a12ce8b 100644 --- a/docs/verbs/cli/catkin_create_pkg.txt +++ b/docs/verbs/cli/catkin_create_pkg.txt @@ -1,20 +1,19 @@ -usage: catkin create pkg [-h] [-p PATH] [--rosdistro ROSDISTRO] +usage: catkin create pkg [-h] [-p PATH] --rosdistro ROSDISTRO [-v MAJOR.MINOR.PATCH] [-l LICENSE] [-m NAME EMAIL] [-a NAME EMAIL] [-d DESCRIPTION] - [--catkin-deps [DEP [DEP ...]]] - [--system-deps [DEP [DEP ...]]] - [--boost-components [COMP [COMP ...]]] + [--catkin-deps [DEP ...]] [--system-deps [DEP ...]] + [--boost-components [COMP ...]] PKG_NAME [PKG_NAME ...] Create a new Catkin package. Note that while the default options used by this command are sufficient for prototyping and local usage, it is important that -any publically-available packages have a valid license and a valid maintainer +any publicly-available packages have a valid license and a valid maintainer e-mail address. positional arguments: PKG_NAME The name of one or more packages to create. This name should be completely lower-case with individual words - separated by undercores. + separated by underscores. optional arguments: -h, --help show this help message and exit @@ -41,15 +40,15 @@ Package Metadata: Description of the package. (default: empty) Package Dependencies: - --catkin-deps [DEP [DEP ...]], -c [DEP [DEP ...]] + --catkin-deps [DEP ...], -c [DEP ...] The names of one or more Catkin dependencies. These are Catkin-based packages which are either built as source or installed by your system's package manager. - --system-deps [DEP [DEP ...]], -s [DEP [DEP ...]] + --system-deps [DEP ...], -s [DEP ...] The names of one or more system dependencies. These are other packages installed by your operating system's package manager. C++ Options: - --boost-components [COMP [COMP ...]] + --boost-components [COMP ...] One or more boost components used by the package. diff --git a/docs/verbs/cli/catkin_env.txt b/docs/verbs/cli/catkin_env.txt index 9387d5a7..48f43d32 100644 --- a/docs/verbs/cli/catkin_env.txt +++ b/docs/verbs/cli/catkin_env.txt @@ -1,5 +1,4 @@ -usage: catkin env [-h] [-i] [-s] - [NAME=VALUE [NAME=VALUE ...]] [COMMAND] [ARG [ARG ...]] +usage: catkin env [-h] [-i] [-s] [NAME=VALUE ...] [COMMAND] [ARG ...] Run an arbitrary command in a modified environment. @@ -12,7 +11,8 @@ optional arguments: -i, --ignore-environment Start with an empty environment. -s, --stdin Read environment variable definitions from stdin. - Variables should be given in NAME=VALUE format. + Variables should be given in NAME=VALUE format, + separated by null-bytes. command: COMMAND Command to run. If omitted, the environment is printed diff --git a/docs/verbs/cli/catkin_list.txt b/docs/verbs/cli/catkin_list.txt index 30b1f621..d0cef84b 100644 --- a/docs/verbs/cli/catkin_list.txt +++ b/docs/verbs/cli/catkin_list.txt @@ -1,10 +1,9 @@ usage: catkin list [-h] [--workspace WORKSPACE] [--profile PROFILE] - [--deps | --rdeps] [--depends-on [PKG [PKG ...]]] - [--rdepends-on [PKG [PKG ...]]] [--this] - [--directory [DIRECTORY [DIRECTORY ...]]] [--quiet] - [--unformatted] + [--deps | --rdeps] [--depends-on [PKG ...]] + [--rdepends-on [PKG ...]] [--this] + [--directory [DIRECTORY ...]] [--quiet] [--unformatted] -Lists catkin packages in the workspace or other arbitray folders. +Lists catkin packages in the workspace or other arbitrary folders. optional arguments: -h, --help show this help message and exit @@ -25,15 +24,15 @@ Information: Packages: Control which packages are listed. - --depends-on [PKG [PKG ...]] + --depends-on [PKG ...] Only show packages that directly depend on specific package(s). - --rdepends-on [PKG [PKG ...]], --recursive-depends-on [PKG [PKG ...]] + --rdepends-on [PKG ...], --recursive-depends-on [PKG ...] Only show packages that recursively depend on specific package(s). --this Show the package which contains the current working directory. - --directory [DIRECTORY [DIRECTORY ...]], -d [DIRECTORY [DIRECTORY ...]] + --directory [DIRECTORY ...], -d [DIRECTORY ...] Pass list of directories process all packages in directory diff --git a/docs/verbs/cli/catkin_locate.txt b/docs/verbs/cli/catkin_locate.txt index 6042efd5..c3076393 100644 --- a/docs/verbs/cli/catkin_locate.txt +++ b/docs/verbs/cli/catkin_locate.txt @@ -1,6 +1,6 @@ usage: catkin locate [-h] [--workspace WORKSPACE] [--profile PROFILE] [-e] - [-r] [-q] [-s | -b | -d | -i] [--this] [--shell-verbs] - [--examples] + [-r] [-q] [-b | -d | -i | -L | -s] [--this] + [--shell-verbs] [--examples] [PACKAGE] Get the paths to various locations in a workspace. @@ -22,10 +22,16 @@ Sub-Space Options: Get the absolute path to one of the following locations in the given workspace with the given profile. - -s, --src Get the path to the source space. - -b, --build Get the path to the build space. - -d, --devel Get the path to the devel space. - -i, --install Get the path to the install space. + -b, --build, --build-space + Get the path to the build space. + -d, --devel, --devel-space + Get the path to the devel space. + -i, --install, --install-space + Get the path to the install space. + -L, --logs, --log-space + Get the path to the log space. + -s, --src, --source-space + Get the path to the source space. Package Directories: Get the absolute path to package directories in the given workspace and diff --git a/docs/verbs/cli/catkin_profile_add.txt b/docs/verbs/cli/catkin_profile_add.txt index f674b89e..284f4bdc 100644 --- a/docs/verbs/cli/catkin_profile_add.txt +++ b/docs/verbs/cli/catkin_profile_add.txt @@ -1,11 +1,15 @@ -usage: catkin profile add [-h] [-f] [--copy BASE_PROFILE | --copy-active] name +usage: catkin profile add [-h] [-f] + [--copy BASE_PROFILE | --copy-active | --extend PARENT_PROFILE] + name positional arguments: - name The new profile name. + name The new profile name. optional arguments: - -h, --help show this help message and exit - -f, --force Overwrite an existing profile. - --copy BASE_PROFILE Copy the settings from an existing profile. (default: - None) - --copy-active Copy the settings from the active profile. + -h, --help show this help message and exit + -f, --force Overwrite an existing profile. + --copy BASE_PROFILE Copy the settings from an existing profile. (default: + None) + --copy-active Copy the settings from the active profile. + --extend PARENT_PROFILE + Extend another profile diff --git a/docs/verbs/cli/catkin_profile_remove.txt b/docs/verbs/cli/catkin_profile_remove.txt index 0fd0b59f..3e9600d9 100644 --- a/docs/verbs/cli/catkin_profile_remove.txt +++ b/docs/verbs/cli/catkin_profile_remove.txt @@ -1,4 +1,4 @@ -usage: catkin profile remove [-h] [name [name ...]] +usage: catkin profile remove [-h] [name ...] positional arguments: name One or more profile names to remove. diff --git a/docs/verbs/cli/catkin_test.txt b/docs/verbs/cli/catkin_test.txt index d099d973..2963497a 100644 --- a/docs/verbs/cli/catkin_test.txt +++ b/docs/verbs/cli/catkin_test.txt @@ -1,8 +1,9 @@ usage: catkin test [-h] [--workspace WORKSPACE] [--profile PROFILE] [--this] [--continue-on-failure] [-p PACKAGE_JOBS] [-t TARGET] [--catkin-test-target TARGET] [--make-args ARG [ARG ...]] - [--verbose] [--interleave-output] [--no-status] - [--summarize] [--no-notify] + [--verbose] [--interleave-output] [--summarize] + [--no-status] [--limit-status-rate LIMIT_STATUS_RATE] + [--no-notify] [PKGNAME ...] Test one or more packages in a catkin workspace. This invokes `make run_tests` @@ -52,8 +53,12 @@ Interface: --interleave-output, -i Prevents ordering of command output when multiple commands are running at the same time. - --no-status Suppresses status line, useful in situations where - carriage return is not properly supported. --summarize, --summary, -s Adds a summary to the end of the log + --no-status Suppresses status line, useful in situations where + carriage return is not properly supported. + --limit-status-rate LIMIT_STATUS_RATE, --status-rate LIMIT_STATUS_RATE + Limit the update rate of the status bar to this + frequency. Zero means unlimited. Must be positive, + default is 10 Hz. --no-notify Suppresses system pop-up notification. From fd856e450f72441d32c015498e7f8d236596ded8 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Wed, 1 Dec 2021 13:44:36 +0100 Subject: [PATCH 68/68] 0.8.2 --- CHANGELOG.rst | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6555dd89..a3d520d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,12 @@ Changelog for package catkin_tools Upcoming -------- +0.8.2 (2021-12-01) +------------------ + +* Add back flags that were removed in #691 (#702) +* Documentation: Update CLI documentation + 0.8.1 (2021-11-30) ------------------ diff --git a/setup.py b/setup.py index 9330ffd0..a9876cb9 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def run(self): setup( name='catkin_tools', - version='0.8.1', + version='0.8.2', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={