diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 398de840718..c6cca0afece 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -2,28 +2,31 @@ ansicolors==1.0.2 beautifulsoup4>=4.3.2,<4.4 cffi==1.11.1 contextlib2==0.5.5 +boto3==1.4.4 coverage>=4.3.4,<4.4 docutils>=0.12,<0.13 fasteners==0.14.1 faulthandler==2.6 -futures==3.0.5 +futures<4.0.0,>=2.2.0 isort==4.2.5 Markdown==2.1.1 mock==2.0.0 +moto==1.2.0 packaging==16.8 pathspec==0.5.0 parameterized==0.6.1 -pep8==1.6.2 pex==1.2.16 +pep8==1.5.0 psutil==4.3.0 pyflakes==1.1.0 -Pygments==1.4 pyopenssl==17.3.0 +Pygments==2.1.3 +pyjavaproperties==0.6 pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 pywatchman==1.4.1 -requests[security]>=2.5.0,<2.19 +requests[security]>=2.8.14 scandir==1.2 setproctitle==1.1.10 setuptools==30.0.0 diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh old mode 100644 new mode 100755 index 31f11b53581..db128461060 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -82,7 +82,7 @@ function ensure_native_build_prerequisites() { log "A pants owned rustup installation could not be found, installing via the instructions at" \ "https://www.rustup.rs ..." local readonly rustup=$(mktemp -t pants.rustup.XXXXXX) - curl https://sh.rustup.rs -sSf > ${rustup} + curl https://sh.rustup.rs -sSf > ${rustup} || (echo "Bad curl, trying wget" && wget https://sh.rustup.rs -O- > ${rustup}) sh ${rustup} -y --no-modify-path --default-toolchain "${rust_toolchain}" 1>&2 rm -f ${rustup} fi diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 11742a098e3..6691605fcac 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -16,7 +16,7 @@ function run_local_pants() { # NB: Pants core does not have the ability to change its own version, so we compute the # suffix here and mutate the VERSION_FILE to affect the current version. readonly HEAD_SHA=$(git rev-parse --verify HEAD) -readonly PANTS_STABLE_VERSION="$(run_local_pants --version 2>/dev/null)" +readonly PANTS_STABLE_VERSION="$(cat "${ROOT}/src/python/pants/VERSION")" readonly PANTS_UNSTABLE_VERSION="${PANTS_STABLE_VERSION}+${HEAD_SHA:0:8}" readonly DEPLOY_DIR="${ROOT}/dist/deploy" @@ -682,7 +682,7 @@ function build_pex() { local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex" - activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.2.13" || die "Failed to install pex." + activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.2.16" || die "Failed to install pex." local requirements_string="" for pkg_name in $PANTS_PEX_PACKAGES; do @@ -764,7 +764,7 @@ function usage() { fi } -while getopts "hdntcloep" opt; do +while getopts "hdntcloepw" opt; do case ${opt} in h) usage ;; d) debug="true" ;; @@ -774,6 +774,7 @@ while getopts "hdntcloep" opt; do o) list_owners ; exit $? ;; e) fetch_and_check_prebuilt_wheels ; exit $? ;; p) build_pex ; exit $? ;; + w) list_prebuilt_wheels ; exit $? ;; *) usage "Invalid option: -${OPTARG}" ;; esac done diff --git a/contrib/go/examples/3rdparty/go/github.com/apache/thrift/BUILD b/contrib/go/examples/3rdparty/go/github.com/apache/thrift/BUILD new file mode 100644 index 00000000000..ba2358f263c --- /dev/null +++ b/contrib/go/examples/3rdparty/go/github.com/apache/thrift/BUILD @@ -0,0 +1,9 @@ +# Auto-generated by pants! +# To re-generate run: `pants buildgen.go --materialize --remote` + +go_remote_libraries( + rev='e1abc8b2f3aed139f43ee0f9d1eca95b7da4f312', + packages=[ + 'lib/go/thrift', + ] +) diff --git a/contrib/go/examples/src/go/duck/BUILD b/contrib/go/examples/src/go/duck/BUILD new file mode 100644 index 00000000000..61122228f09 --- /dev/null +++ b/contrib/go/examples/src/go/duck/BUILD @@ -0,0 +1,8 @@ +# Auto-generated by pants! +# To re-generate run: `pants buildgen.go --materialize --remote` + +go_library( + dependencies=[ + 'contrib/go/examples/src/go/duckthrift/gen', + ] +) diff --git a/contrib/go/examples/src/go/duck/duckreader.go b/contrib/go/examples/src/go/duck/duckreader.go new file mode 100644 index 00000000000..120c239f781 --- /dev/null +++ b/contrib/go/examples/src/go/duck/duckreader.go @@ -0,0 +1,10 @@ +package duck + +import ( + "duckthrift/gen" +) + +func reader(d duck) string { + d := duck.NewDuck() + return d.GetQuack() +} diff --git a/contrib/go/examples/src/go/duckthrift/gen/BUILD b/contrib/go/examples/src/go/duckthrift/gen/BUILD new file mode 100644 index 00000000000..e79e6458479 --- /dev/null +++ b/contrib/go/examples/src/go/duckthrift/gen/BUILD @@ -0,0 +1,3 @@ +go_thrift_library( + sources = globs('*.thrift'), +) \ No newline at end of file diff --git a/contrib/go/examples/src/go/duckthrift/gen/duck.thrift b/contrib/go/examples/src/go/duckthrift/gen/duck.thrift new file mode 100644 index 00000000000..ac8a78f00b1 --- /dev/null +++ b/contrib/go/examples/src/go/duckthrift/gen/duck.thrift @@ -0,0 +1,8 @@ +// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +namespace go duckthrift.gen + +struct Duck { + 1: optional string quack, +} \ No newline at end of file diff --git a/contrib/go/examples/src/go/libA/BUILD b/contrib/go/examples/src/go/libA/BUILD index de1d82b940d..b86e065fd29 100644 --- a/contrib/go/examples/src/go/libA/BUILD +++ b/contrib/go/examples/src/go/libA/BUILD @@ -3,7 +3,7 @@ go_library( dependencies=[ - 'contrib/go/examples/src/go/libB', 'contrib/go/examples/src/go/libC', + 'contrib/go/examples/src/go/libB', ] ) diff --git a/contrib/go/examples/src/go/libC/BUILD b/contrib/go/examples/src/go/libC/BUILD index 9e4482af9a2..12cba16d679 100644 --- a/contrib/go/examples/src/go/libC/BUILD +++ b/contrib/go/examples/src/go/libC/BUILD @@ -3,7 +3,7 @@ go_library( dependencies=[ - 'contrib/go/examples/src/go/libD', 'contrib/go/examples/src/go/libE', + 'contrib/go/examples/src/go/libD', ] ) diff --git a/contrib/go/examples/src/go/server/BUILD b/contrib/go/examples/src/go/server/BUILD index 0b9bf569f01..192a3a6da6f 100644 --- a/contrib/go/examples/src/go/server/BUILD +++ b/contrib/go/examples/src/go/server/BUILD @@ -3,9 +3,9 @@ go_binary( dependencies=[ - 'contrib/go/examples/3rdparty/go/github.com/gorilla/mux', + 'contrib/go/examples/3rdparty/go/gopkg.in/yaml.v2', 'contrib/go/examples/3rdparty/go/golang.org/x/net:http2', + 'contrib/go/examples/3rdparty/go/github.com/gorilla/mux', 'contrib/go/examples/3rdparty/go/google.golang.org/grpc', - 'contrib/go/examples/3rdparty/go/gopkg.in/yaml.v2', ] ) diff --git a/contrib/go/examples/src/thrift/duckthrift/gen/BUILD b/contrib/go/examples/src/thrift/duckthrift/gen/BUILD new file mode 100644 index 00000000000..e79e6458479 --- /dev/null +++ b/contrib/go/examples/src/thrift/duckthrift/gen/BUILD @@ -0,0 +1,3 @@ +go_thrift_library( + sources = globs('*.thrift'), +) \ No newline at end of file diff --git a/contrib/go/examples/src/thrift/duckthrift/gen/duck.thrift b/contrib/go/examples/src/thrift/duckthrift/gen/duck.thrift new file mode 100644 index 00000000000..ac8a78f00b1 --- /dev/null +++ b/contrib/go/examples/src/thrift/duckthrift/gen/duck.thrift @@ -0,0 +1,8 @@ +// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +namespace go duckthrift.gen + +struct Duck { + 1: optional string quack, +} \ No newline at end of file diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_buildgen.py b/contrib/go/src/python/pants/contrib/go/tasks/go_buildgen.py index 9e9c377dace..dfebaa997a5 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_buildgen.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_buildgen.py @@ -24,6 +24,7 @@ from pants.contrib.go.targets.go_library import GoLibrary from pants.contrib.go.targets.go_local_source import GoLocalSource from pants.contrib.go.targets.go_remote_library import GoRemoteLibrary +from pants.contrib.go.targets.go_thrift_library import GoThriftLibrary from pants.contrib.go.tasks.go_task import GoTask @@ -69,7 +70,7 @@ def _generate_missing(self, gopath, local_address, import_listing, visited): existing = self._build_graph.get_target(local_address) if not existing: self._build_graph.inject_synthetic_target(address=local_address, target_type=target_type) - elif existing and not isinstance(existing, target_type): + elif existing and not isinstance(existing, target_type) and not isinstance(existing, GoThriftLibrary): raise self.WrongLocalSourceTargetTypeError('{} should be a {}' .format(existing, target_type.__name__)) @@ -107,6 +108,15 @@ def _list_deps(self, gopath, local_address): src_path = os.path.join(gopath, 'src', import_path) safe_mkdir(src_path) package_src_root = os.path.join(get_buildroot(), local_address.spec_path) + internal = self._build_graph.get_target(local_address) + + if isinstance(internal, GoThriftLibrary): + + package = os.path.basename(import_path) + dummy_file = os.path.join(src_path, '{}.go'.format(package)) + with safe_open(dummy_file, 'w') as fp: + fp.write('package {}'.format(package)) + for source_file in os.listdir(package_src_root): source_path = os.path.join(package_src_root, source_file) if GoLocalSource.is_go_source(source_path): diff --git a/contrib/go/tests/python/pants_test/contrib/go/tasks/BUILD b/contrib/go/tests/python/pants_test/contrib/go/tasks/BUILD index 343d1d0c665..562ec1382d8 100644 --- a/contrib/go/tests/python/pants_test/contrib/go/tasks/BUILD +++ b/contrib/go/tests/python/pants_test/contrib/go/tasks/BUILD @@ -33,3 +33,21 @@ python_tests( tags={'integration'}, timeout=180, ) + + +python_tests( + name = 'buildgen', + sources = [ 'test_go_buildgen.py'], + dependencies=[ + 'contrib/go/src/python/pants/contrib/go/subsystems', + 'contrib/go/src/python/pants/contrib/go/targets', + 'contrib/go/src/python/pants/contrib/go/tasks', + 'contrib/go/src/python/pants/contrib/go:plugin', + 'src/python/pants/base:exceptions', + 'src/python/pants/build_graph', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/tasks:task_test_base', + ], +) diff --git a/contrib/release_packages.sh b/contrib/release_packages.sh index f4f61c71f90..6343c87704c 100644 --- a/contrib/release_packages.sh +++ b/contrib/release_packages.sh @@ -198,6 +198,18 @@ function pkg_avro_install_test() { --explain gen | grep "avro-java" &> /dev/null } +PKG_THRIFTY=( + "pantsbuild.pants.contrib.thrifty" + "//contrib/thrifty/src/python/pants/contrib/thrifty:plugin" + "pkg_thrifty_install_test" +) +function pkg_thrifty_install_test() { + local version=$1 + execute_packaged_pants_with_internal_backends \ + --plugins="['pantsbuild.pants.contrib.thrifty==${version}']" \ + --explain gen | grep "thrifty" &> /dev/null +} + # Once individual (new) package is declared above, insert it into the array below) CONTRIB_PACKAGES=( PKG_ANDROID @@ -215,4 +227,5 @@ CONTRIB_PACKAGES=( PKG_JAXWS PKG_MYPY PKG_AVRO + PKG_THRIFTY ) diff --git a/contrib/thrifty/src/python/pants/__init__.py b/contrib/thrifty/src/python/pants/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/thrifty/src/python/pants/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/thrifty/src/python/pants/contrib/__init__.py b/contrib/thrifty/src/python/pants/contrib/__init__.py new file mode 100644 index 00000000000..de40ea7ca05 --- /dev/null +++ b/contrib/thrifty/src/python/pants/contrib/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/BUILD b/contrib/thrifty/src/python/pants/contrib/thrifty/BUILD new file mode 100644 index 00000000000..bff04a0fd1f --- /dev/null +++ b/contrib/thrifty/src/python/pants/contrib/thrifty/BUILD @@ -0,0 +1,19 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library(sources=['java_thrifty_gen.py', 'java_thrifty_library.py']) + +contrib_plugin( + name='plugin', + dependencies=[ + ':thrifty', + 'src/python/pants/goal:task_registrar', + ], + distribution_name='pantsbuild.pants.contrib.thrifty', + description='Microsoft Thrifty thrift generator pants plugins.', + additional_classifiers=[ + 'Topic :: Software Development :: Code Generators' + ], + register_goals=True, + build_file_aliases=True, +) diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/__init__.py b/contrib/thrifty/src/python/pants/contrib/thrifty/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py new file mode 100644 index 00000000000..bd082976a1c --- /dev/null +++ b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py @@ -0,0 +1,89 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pants.backend.jvm.targets.java_library import JavaLibrary +from pants.backend.jvm.tasks.nailgun_task import NailgunTaskBase +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.java.jar.jar_dependency import JarDependency +from pants.task.simple_codegen_task import SimpleCodegenTask +from twitter.common.collections import OrderedSet + +from pants.contrib.thrifty.java_thrifty_library import JavaThriftyLibrary + + +class JavaThriftyGen(NailgunTaskBase, SimpleCodegenTask): + + gentarget_type = JavaThriftyLibrary + + @classmethod + def register_options(cls, register): + super(JavaThriftyGen, cls).register_options(register) + + def thrifty_jar(name): + return JarDependency(org='com.microsoft.thrifty', name=name, rev='0.4.3') + + cls.register_jvm_tool(register, + 'thrifty-runtime', + classpath=[thrifty_jar(name='thrifty-runtime')]) + cls.register_jvm_tool(register, + 'thrifty-compiler', + classpath=[thrifty_jar(name='thrifty-compiler')]) + + def synthetic_target_type(self, target): + return JavaLibrary + + def synthetic_target_extra_dependencies(self, target, target_workdir): + deps = OrderedSet(self.resolve_deps([self.get_options().thrifty_runtime])) + deps.update(target.dependencies) + return deps + + def synthetic_target_extra_exports(self, target, target_workdir): + return self.resolve_deps([self.get_options().thrifty_runtime]) + + def format_args_for_target(self, target, target_workdir): + sources = OrderedSet(target.sources_relative_to_buildroot()) + args = ['--out={0}'.format(target_workdir)] + for include_path in self._compute_include_paths(target): + args.append('--path={0}'.format(include_path)) + args.extend(sources) + return args + + def execute_codegen(self, target, target_workdir): + args = self.format_args_for_target(target, target_workdir) + if args: + result = self.runjava(classpath=self.tool_classpath('thrifty-compiler'), + main='com.microsoft.thrifty.compiler.ThriftyCompiler', + args=args, + workunit_name='compile', + workunit_labels=[WorkUnitLabel.TOOL]) + if result != 0: + raise TaskError('Thrifty compiler exited non-zero ({0})'.format(result)) + + def _compute_include_paths(self, target): + """Computes the set of paths that thrifty uses to lookup imports. + + The IDL files under these paths are not compiled, but they are required to compile + downstream IDL files. + + :param target: the JavaThriftyLibrary target to compile. + :return: an ordered set of directories to pass along to thrifty. + """ + paths = OrderedSet() + paths.add(os.path.join(get_buildroot(), target.target_base)) + + def collect_paths(dep): + if not dep.has_sources('.thrift'): + return + paths.add(os.path.join(get_buildroot(), dep.target_base)) + + collect_paths(target) + target.walk(collect_paths) + return paths diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_library.py b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_library.py new file mode 100644 index 00000000000..423accd65b6 --- /dev/null +++ b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_library.py @@ -0,0 +1,19 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.jvm.targets.jvm_target import JvmTarget + + +class JavaThriftyLibrary(JvmTarget): + """An Android-optimized Java library generated by the Microsoft Thrifty thrift compiler. + + Says Thrifty, "Thrifty is an implementation of the Apache Thrift software stack for Android, + which uses 1/4 of the method count taken by the Apache Thrift compiler." + + For details, see https://github.com/Microsoft/thrifty + """ + default_sources_globs = '*.thrift' diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/register.py b/contrib/thrifty/src/python/pants/contrib/thrifty/register.py new file mode 100644 index 00000000000..0590600a307 --- /dev/null +++ b/contrib/thrifty/src/python/pants/contrib/thrifty/register.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.goal.task_registrar import TaskRegistrar as task + +from pants.contrib.thrifty.java_thrifty_gen import JavaThriftyGen +from pants.contrib.thrifty.java_thrifty_library import JavaThriftyLibrary + + +def build_file_aliases(): + return BuildFileAliases( + targets={ + 'java_thrifty_library': JavaThriftyLibrary, + } + ) + + +def register_goals(): + task(name='thrifty', action=JavaThriftyGen).install('gen') diff --git a/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/BUILD b/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/BUILD new file mode 100644 index 00000000000..b46a2efa34b --- /dev/null +++ b/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/BUILD @@ -0,0 +1,9 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +junit_tests( + dependencies=[ + 'contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client', + 'contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common', + ], +) diff --git a/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/ThriftyStructTest.java b/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/ThriftyStructTest.java new file mode 100644 index 00000000000..f277271f0d4 --- /dev/null +++ b/contrib/thrifty/tests/java/org/pantsbuild/contrib/thrifty/ThriftyStructTest.java @@ -0,0 +1,27 @@ +// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +package org.pantsbuild.contrib.thrifty; + +import org.junit.Test; +import org.pantsbuild.contrib.thrifty.common.ClientLog; +import org.pantsbuild.contrib.thrifty.common.Common; + +import static org.junit.Assert.assertEquals; + +public class ThriftyStructTest { + + @Test + public void testThriftyStruct() { + Common common = new Common.Builder() + .timestamp(1L) + .hostname("fake") + .build(); + ClientLog clientLog = new ClientLog.Builder() + .common(common) + .message("fake_message") + .build(); + assertEquals(clientLog.toString(), + "ClientLog{common=Common{timestamp=1, hostname=fake}, message=fake_message}"); + } +} diff --git a/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/BUILD b/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/BUILD new file mode 100644 index 00000000000..aa4c0a49317 --- /dev/null +++ b/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/BUILD @@ -0,0 +1,10 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + dependencies=[ + 'contrib/thrifty/src/python/pants/contrib/thrifty', + 'tests/python/pants_test/tasks:task_test_base', + ], + sources=globs('*.py'), +) diff --git a/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/test_thrifty_gen.py b/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/test_thrifty_gen.py new file mode 100644 index 00000000000..ded2df3ac93 --- /dev/null +++ b/contrib/thrifty/tests/python/pants_test/pants/contrib/thrifty/test_thrifty_gen.py @@ -0,0 +1,58 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.codegen.wire.java.register import build_file_aliases as register_codegen +from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.build_graph.register import build_file_aliases as register_core +from pants.java.jar.jar_dependency import JarDependency +from pants_test.tasks.task_test_base import TaskTestBase + +from pants.contrib.thrifty.java_thrifty_gen import JavaThriftyGen +from pants.contrib.thrifty.java_thrifty_library import JavaThriftyLibrary + + +class JavaThriftyGenTest(TaskTestBase): + TARGET_WORKDIR = ".pants.d/bogus/workdir" + + @classmethod + def task_type(cls): + return JavaThriftyGen + + @property + def alias_groups(self): + return register_core().merge(register_codegen()) + + def _create_fake_thrifty_tool(self): + self.make_target(':thrifty-compiler', JarLibrary, jars=[ + JarDependency(org='com.microsoft.thrifty', name='thrifty-compiler', rev='0.4.3'), + ]) + + def test_compiler_args(self): + self._create_fake_thrifty_tool() + target = self.make_target('src/thrifty:simple-thrifty-target', JavaThriftyLibrary, + sources=['foo.thrift']) + context = self.context(target_roots=[target]) + task = self.create_task(context) + self.assertEquals([ + '--out={}'.format(self.TARGET_WORKDIR), + '--path={}/src/thrifty'.format(self.build_root), + 'src/thrifty/foo.thrift'], + task.format_args_for_target(target, self.TARGET_WORKDIR)) + + def test_compiler_args_deps(self): + self._create_fake_thrifty_tool() + upstream = self.make_target('src/thrifty:upstream', JavaThriftyLibrary, + sources=['upstream.thrift']) + downstream = self.make_target('src/thrifty:downstream', JavaThriftyLibrary, + sources=['downstream.thrift'], dependencies=[upstream]) + context = self.context(target_roots=[upstream, downstream]) + task = self.create_task(context) + self.assertEquals([ + '--out={}'.format(self.TARGET_WORKDIR), + '--path={}/src/thrifty'.format(self.build_root), + 'src/thrifty/downstream.thrift'], + task.format_args_for_target(downstream, self.TARGET_WORKDIR)) diff --git a/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/BUILD b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/BUILD new file mode 100644 index 00000000000..689dd02469f --- /dev/null +++ b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +java_thrifty_library( + dependencies=[ + 'contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common', + ], +) diff --git a/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/clientlog.thrift b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/clientlog.thrift new file mode 100644 index 00000000000..3f88d457b05 --- /dev/null +++ b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/client/clientlog.thrift @@ -0,0 +1,11 @@ +// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +namespace java org.pantsbuild.contrib.thrifty.common + +include "org/pantsbuild/contrib/thrifty/common/common.thrift" + +struct ClientLog { + 1: common.Common common; + 2: string message; +} diff --git a/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/BUILD b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/BUILD new file mode 100644 index 00000000000..20b78606599 --- /dev/null +++ b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/BUILD @@ -0,0 +1,4 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +java_thrifty_library() diff --git a/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/common.thrift b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/common.thrift new file mode 100644 index 00000000000..9d0b18ced02 --- /dev/null +++ b/contrib/thrifty/tests/thrift/org/pantsbuild/contrib/thrifty/common/common.thrift @@ -0,0 +1,9 @@ +// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +namespace java org.pantsbuild.contrib.thrifty.common + +struct Common { + 1: optional i64 timestamp + 2: required string hostname; +} diff --git a/pants b/pants index 9774344fad4..500507a1d2f 100755 --- a/pants +++ b/pants @@ -89,3 +89,4 @@ if [[ ! -z "${WRAPPER_SRCPATH}" ]]; then fi exec_pants_bare "$@" + diff --git a/pants.ini b/pants.ini index 72d044bad14..c40ba0cb0a1 100644 --- a/pants.ini +++ b/pants.ini @@ -39,6 +39,7 @@ pythonpath: [ "%(buildroot)s/contrib/python/src/python", "%(buildroot)s/contrib/scalajs/src/python", "%(buildroot)s/contrib/scrooge/src/python", + "%(buildroot)s/contrib/thrifty/src/python", "%(buildroot)s/pants-plugins/src/python", ] @@ -62,6 +63,7 @@ backend_packages: +[ "pants.contrib.node", "pants.contrib.python.checks", "pants.contrib.scrooge", + "pants.contrib.thrifty", ] # Path patterns to ignore for filesystem operations on top of the builtin patterns. @@ -148,6 +150,15 @@ deps: ["3rdparty:thrift-0.9.2"] deps: ["3rdparty/python:thrift"] +[gen.thrifty] +allow_dups: True + + +[gen.go-thrift] +thrift_import_target: contrib/go/examples/3rdparty/go/github.com/apache/thrift:lib/go/thrift +thrift_import: github.com/apache/thrift/lib/go/thrift + + [gen.antlr-py] antlr3_deps: ["3rdparty/python:antlr-3.1.3"] @@ -316,8 +327,6 @@ restrict_push_urls: [ # Any example or test targets that are meant to test interpreters outside pants own # acceptable set should specify an explicit compatibility constraint. interpreter_constraints: ["CPython>=2.7,<3"] -interpreter_cache_dir: %(pants_bootstrapdir)s/python_cache/interpreters -resolver_cache_dir: %(pants_bootstrapdir)s/python_cache/requirements [sign] diff --git a/src/java/org/pantsbuild/args4j/BUILD b/src/java/org/pantsbuild/args4j/BUILD index de6769b0ef0..e9a11fd592b 100644 --- a/src/java/org/pantsbuild/args4j/BUILD +++ b/src/java/org/pantsbuild/args4j/BUILD @@ -10,7 +10,7 @@ java_library( platform='java7', provides=artifact( org='org.pantsbuild', - name='args4j', + name='args4j-fs', repo=public, publication_metadata=pants_library(""" Utilities to make args4j more like com.twitter.common#args diff --git a/src/java/org/pantsbuild/junit/annotations/BUILD b/src/java/org/pantsbuild/junit/annotations/BUILD index 2a68f1b899c..77a6ae9f4fd 100644 --- a/src/java/org/pantsbuild/junit/annotations/BUILD +++ b/src/java/org/pantsbuild/junit/annotations/BUILD @@ -4,7 +4,7 @@ java_library( provides=artifact( org='org.pantsbuild', - name='junit-runner-annotations', + name='junit-runner-annotations-fs', repo=public, publication_metadata=pants_library(""" Annotations for use with org.pantsbuild#junit-runner that support running tests in parallel. diff --git a/src/java/org/pantsbuild/tools/junit/BUILD b/src/java/org/pantsbuild/tools/junit/BUILD index 105b0fb82cf..c866c04e7fb 100644 --- a/src/java/org/pantsbuild/tools/junit/BUILD +++ b/src/java/org/pantsbuild/tools/junit/BUILD @@ -4,7 +4,7 @@ java_library( provides=artifact( org='org.pantsbuild', - name='junit-runner', + name='junit-runner-fs', repo=public, publication_metadata=pants_library(""" A command line tool for running junit tests that provides functionality above and beyond diff --git a/src/java/org/pantsbuild/tools/junit/impl/ScalaTestUtil.java b/src/java/org/pantsbuild/tools/junit/impl/ScalaTestUtil.java index f28dcbdc8ea..945beafcb4a 100644 --- a/src/java/org/pantsbuild/tools/junit/impl/ScalaTestUtil.java +++ b/src/java/org/pantsbuild/tools/junit/impl/ScalaTestUtil.java @@ -5,36 +5,37 @@ public final class ScalaTestUtil { private ScalaTestUtil() {} + // Scalatest classes loaded once with runtime reflection to avoid the extra + // dependency. + private static Class suiteClass = null; + private static Class junitRunnerClass = null; + static { + try { + suiteClass = Class.forName("org.scalatest.Suite"); + junitRunnerClass = Class.forName("org.scalatest.junit.JUnitRunner"); + } catch (ClassNotFoundException e) { + // No scalatest tests on classpath + } + } + /** - * Returns a scalatest junit runner using reflection in the classloader of the test. + * Returns a scalatest junit runner using reflection. * @param clazz the test class * - * @return a new scala test junit runner + * @return a new scalatest junit runner */ - public static Runner getJUnitRunner(Class clazz) { - try { - Class junitRunnerClass = Class.forName("org.scalatest.junit.JUnitRunner", - true, clazz.getClassLoader()); - return (Runner)junitRunnerClass.getConstructor(Class.class).newInstance(clazz); - } catch (Exception e) { - // isScalaTest should fail if scala test isn't available so this is probably ok. - throw new RuntimeException(e); - } + public static Runner getJUnitRunner(Class clazz) throws Exception { + return (Runner) junitRunnerClass.getConstructor(Class.class).newInstance(clazz); } /** - * Checks if the passed in test clazz has an ancestor that is the scala test suite - * object (looked up in the test classes class loader). + * Checks if the passed in test clazz has an ancestor that is the scalatest suite + * trait. * @param clazz the test class * * @return true if the test class is a scalatest test, false if not. */ public static boolean isScalaTestTest(Class clazz) { - try { - Class suiteClass = Class.forName("org.scalatest.Suite", true, clazz.getClassLoader()); - return suiteClass.isAssignableFrom(clazz); - } catch (ClassNotFoundException e) { - return false; - } + return suiteClass != null && suiteClass.isAssignableFrom(clazz); } } diff --git a/src/java/org/pantsbuild/tools/junit/withretry/BUILD b/src/java/org/pantsbuild/tools/junit/withretry/BUILD index 1054c2f4b22..a7cc8d116c5 100644 --- a/src/java/org/pantsbuild/tools/junit/withretry/BUILD +++ b/src/java/org/pantsbuild/tools/junit/withretry/BUILD @@ -4,7 +4,7 @@ java_library( provides=artifact( org='org.pantsbuild', - name='junit-runner-withretry', + name='junit-runner-withretry-fs', repo=public, publication_metadata=pants_library(""" Provides an org.junit.runner.Runner that supports retries of failing tests. diff --git a/src/python/pants/VERSION b/src/python/pants/VERSION index 40b4880203a..e77c20ee303 100644 --- a/src/python/pants/VERSION +++ b/src/python/pants/VERSION @@ -1 +1 @@ -1.4.0rc1 +1.4.0+fs2 diff --git a/src/python/pants/backend/jvm/subsystems/junit.py b/src/python/pants/backend/jvm/subsystems/junit.py index 9a026a0800d..dcff933a640 100644 --- a/src/python/pants/backend/jvm/subsystems/junit.py +++ b/src/python/pants/backend/jvm/subsystems/junit.py @@ -20,7 +20,7 @@ class JUnit(JvmToolMixin, Subsystem): RUNNER_MAIN = 'org.pantsbuild.tools.junit.ConsoleRunner' LIBRARY_JAR = JarDependency(org='junit', name='junit', rev=LIBRARY_REV) - _RUNNER_JAR = JarDependency(org='org.pantsbuild', name='junit-runner', rev='1.0.23') + _RUNNER_JAR = JarDependency(org='org.pantsbuild', name='junit-runner-fs', rev='1.0.24') @classmethod def register_options(cls, register): diff --git a/src/python/pants/backend/jvm/tasks/BUILD b/src/python/pants/backend/jvm/tasks/BUILD index 41eb5c303b4..8b8cec66374 100644 --- a/src/python/pants/backend/jvm/tasks/BUILD +++ b/src/python/pants/backend/jvm/tasks/BUILD @@ -339,6 +339,7 @@ python_library( sources = ['javadoc_gen.py'], dependencies = [ ':jvmdoc_gen', + 'src/python/pants/backend/jvm/targets:scala', 'src/python/pants/java/distribution', 'src/python/pants/java:executor', 'src/python/pants/util:memo', diff --git a/src/python/pants/backend/jvm/tasks/javadoc_gen.py b/src/python/pants/backend/jvm/tasks/javadoc_gen.py index c4c2f1a1f5c..791791f47e9 100644 --- a/src/python/pants/backend/jvm/tasks/javadoc_gen.py +++ b/src/python/pants/backend/jvm/tasks/javadoc_gen.py @@ -7,6 +7,7 @@ import os +from pants.backend.jvm.targets.scala_library import ScalaLibrary from pants.backend.jvm.tasks.jvmdoc_gen import Jvmdoc, JvmdocGen from pants.java.distribution.distribution import DistributionLocator from pants.java.executor import SubprocessExecutor @@ -27,7 +28,7 @@ def subsystem_dependencies(cls): def execute(self): def is_java(target): - return target.has_sources('.java') + return target.has_sources('.java') and not isinstance(target, ScalaLibrary) self.generate_doc(is_java, self.create_javadoc_command) diff --git a/src/python/pants/base/workunit.py b/src/python/pants/base/workunit.py index d3b1f2e9dff..dac458ee946 100644 --- a/src/python/pants/base/workunit.py +++ b/src/python/pants/base/workunit.py @@ -140,9 +140,9 @@ def has_label(self, label): """ return label in self.labels - def start(self): + def start(self, start_time=None): """Mark the time at which this workunit started.""" - self.start_time = time.time() + self.start_time = start_time or time.time() def end(self): """Mark the time at which this workunit ended.""" diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py index d7ad6e00072..c16e3bfa428 100644 --- a/src/python/pants/bin/daemon_pants_runner.py +++ b/src/python/pants/bin/daemon_pants_runner.py @@ -21,7 +21,7 @@ from pants.java.nailgun_io import NailgunStreamStdinReader, NailgunStreamWriter from pants.java.nailgun_protocol import ChunkType, NailgunProtocol from pants.pantsd.process_manager import ProcessManager -from pants.util.contextutil import HardSystemExit, stdio_as +from pants.util.contextutil import HardSystemExit, hermetic_environment_as, stdio_as from pants.util.socket import teardown_socket @@ -171,6 +171,10 @@ def _raise_deferred_exc(self): # If `_deferred_exception` isn't a 3-item tuple, treat it like a bare exception. raise self._deferred_exception + def _maybe_get_client_start_time_from_env(self, env): + client_start_time = env.pop('PANTSD_RUNTRACKER_CLIENT_START_TIME', None) + return None if client_start_time is None else float(client_start_time) + def run(self): """Fork, daemonize and invoke self.post_fork_child() (via ProcessManager).""" with self._fork_lock: @@ -209,8 +213,8 @@ def post_fork_child(self): # Setup a SIGINT signal handler. self._setup_sigint_handler() - # Invoke a Pants run with stdio redirected. - with self._nailgunned_stdio(self._socket) as finalizer: + # Invoke a Pants run with stdio redirected and a proxied environment. + with self._nailgunned_stdio(self._socket) as finalizer, hermetic_environment_as(**self._env): try: # Setup the Exiter's finalizer. self._exiter.set_finalizer(finalizer) @@ -223,6 +227,7 @@ def post_fork_child(self): # Otherwise, conduct a normal run. runner = LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper) + runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env)) runner.set_preceding_graph_size(self._preceding_graph_size) runner.run() except KeyboardInterrupt: diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index 62aa52a23f6..f1345dd05c0 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -32,10 +32,14 @@ def __init__(self, exiter, args, env, daemon_build_graph=None, options_bootstrap self._daemon_build_graph = daemon_build_graph self._options_bootstrapper = options_bootstrapper self._preceding_graph_size = -1 + self._run_start_time = None def set_preceding_graph_size(self, size): self._preceding_graph_size = size + def set_start_time(self, start_time): + self._run_start_time = start_time + def run(self): profile_path = self._env.get('PANTS_PROFILE') with hard_exit_handler(), maybe_profiled(profile_path): @@ -63,7 +67,7 @@ def _run(self): # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() - reporting.initialize(run_tracker) + reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. diff --git a/src/python/pants/bin/pants_exe.py b/src/python/pants/bin/pants_exe.py index 806eec7b1c1..4234e1eab45 100644 --- a/src/python/pants/bin/pants_exe.py +++ b/src/python/pants/bin/pants_exe.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import os +import time from pants.base.exiter import Exiter from pants.bin.pants_runner import PantsRunner @@ -28,11 +29,13 @@ def test_env(): def main(): + start_time = time.time() + exiter = Exiter() exiter.set_except_hook() with maybe_profiled(os.environ.get('PANTSC_PROFILE')): try: - PantsRunner(exiter).run() + PantsRunner(exiter, start_time=start_time).run() except KeyboardInterrupt: exiter.exit_and_fail(b'Interrupted by user.') diff --git a/src/python/pants/bin/pants_runner.py b/src/python/pants/bin/pants_runner.py index 9eac08ed95b..c6969145a16 100644 --- a/src/python/pants/bin/pants_runner.py +++ b/src/python/pants/bin/pants_runner.py @@ -19,7 +19,7 @@ class PantsRunner(object): """A higher-level runner that delegates runs to either a LocalPantsRunner or RemotePantsRunner.""" - def __init__(self, exiter, args=None, env=None): + def __init__(self, exiter, args=None, env=None, start_time=None): """ :param Exiter exiter: The Exiter instance to use for this run. :param list args: The arguments (sys.argv) for this run. (Optional, default: sys.argv) @@ -28,6 +28,7 @@ def __init__(self, exiter, args=None, env=None): self._exiter = exiter self._args = args or sys.argv self._env = env or os.environ + self._start_time = start_time def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) @@ -42,7 +43,11 @@ def run(self): # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner - return LocalPantsRunner(self._exiter, - self._args, - self._env, - options_bootstrapper=options_bootstrapper).run() + runner = LocalPantsRunner( + self._exiter, + self._args, + self._env, + options_bootstrapper=options_bootstrapper + ) + runner.set_start_time(self._start_time) + return runner.run() diff --git a/src/python/pants/bin/remote_pants_runner.py b/src/python/pants/bin/remote_pants_runner.py index 6fafccbf722..b4dbef471a6 100644 --- a/src/python/pants/bin/remote_pants_runner.py +++ b/src/python/pants/bin/remote_pants_runner.py @@ -8,6 +8,7 @@ import logging import signal import sys +import time from contextlib import contextmanager from pants.console.stty_utils import STTYSettings @@ -43,6 +44,7 @@ def __init__(self, exiter, args, env, bootstrap_options, stdin=None, stdout=None :param file stdout: The stream representing stdout. :param file stderr: The stream representing stderr. """ + self._start_time = time.time() self._exiter = exiter self._args = args self._env = env @@ -93,6 +95,7 @@ def _connect_and_execute(self, port): # Merge the nailgun TTY capability environment variables with the passed environment dict. ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr) modified_env = combined_dict(self._env, ng_env) + modified_env['PANTSD_RUNTRACKER_CLIENT_START_TIME'] = str(self._start_time) assert isinstance(port, int), 'port {} is not an integer!'.format(port) diff --git a/src/python/pants/cache/BUILD b/src/python/pants/cache/BUILD index 2de81f7962d..2b1ca179173 100644 --- a/src/python/pants/cache/BUILD +++ b/src/python/pants/cache/BUILD @@ -3,6 +3,8 @@ python_library( dependencies = [ + '3rdparty/python:boto3', + '3rdparty/python:pyjavaproperties', '3rdparty/python:requests', '3rdparty/python:pyopenssl', '3rdparty/python:six', diff --git a/src/python/pants/cache/cache_setup.py b/src/python/pants/cache/cache_setup.py index 4adc51fd65e..dd0d6944a39 100644 --- a/src/python/pants/cache/cache_setup.py +++ b/src/python/pants/cache/cache_setup.py @@ -18,6 +18,7 @@ from pants.cache.pinger import BestUrlSelector, Pinger from pants.cache.resolver import NoopResolver, Resolver, RESTfulResolver from pants.cache.restful_artifact_cache import RESTfulArtifactCache +from pants.cache.s3_artifact_cache import S3ArtifactCache from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property @@ -61,7 +62,7 @@ def register_options(cls, register): 'instead of skipping them.') register('--resolver', advanced=True, choices=['none', 'rest'], default='none', help='Select which resolver strategy to use for discovering URIs that access ' - 'artifact caches. none: use URIs from static config options, i.e. ' + 'http(s) artifact caches. none: use URIs from static config options, i.e. ' '--read-from, --write-to. rest: look up URIs by querying a RESTful ' 'URL, which is a remote address from --read-from, --write-to.') register('--read-from', advanced=True, type=list, default=default_cache, @@ -86,6 +87,14 @@ def register_options(cls, register): help='number of times pinger tries a cache') register('--write-permissions', advanced=True, type=str, default=None, help='Permissions to use when writing artifacts to a local cache, in octal.') + # NOTE(mateo): It would probably be more clear to the consumer to include the Boto3 defaults + # explicitely, although it could bitrot. + register('--s3-config-file', advanced=True, type=str, + help='Boto config file in .ini syntax. Falls back to Boto3 defaults if unset.') + register('--s3-credentials-file', advanced=True, type=str, + help='AWS credentials file in .ini syntax. Falls back to Boto3 defaults if unset.') + register('--s3-profile', advanced=True, type=str, default='default', + help='Boto profile to use for accessing S3. Falls back to Boto3 defaults if unset.') @classmethod def create_cache_factory_for_task(cls, task, **kwargs): @@ -234,9 +243,14 @@ def is_local(string_spec): return string_spec.startswith('/') or string_spec.startswith('~') @staticmethod - def is_remote(string_spec): + def _is_s3(string_spec): + return string_spec.startswith('s3://') + + @classmethod + def is_remote(cls, string_spec): # both artifact cache and resolver use REST, add new protocols here once they are supported - return string_spec.startswith('http://') or string_spec.startswith('https://') + return (string_spec.startswith('http://') or string_spec.startswith('https://') or + cls._is_s3(string_spec)) def _baseurl(self, url): parsed_url = urlparse.urlparse(url) @@ -281,17 +295,28 @@ def create_local_cache(parent_path): dereference=self._options.dereference_symlinks) def create_remote_cache(remote_spec, local_cache): - urls = self.get_available_urls(remote_spec.split('|')) - + urls = remote_spec.split('|') if len(urls) > 0: - best_url_selector = BestUrlSelector( - ['{}/{}'.format(url.rstrip('/'), self._cache_dirname) for url in urls] - ) local_cache = local_cache or TempLocalArtifactCache(artifact_root, compression) + if any(map(self._is_s3, urls)): + if len(urls) != 1: + raise InvalidCacheSpecError('S3 Cache only supports a single entry, got: {0}'.format( + remote_spec)) + return S3ArtifactCache( + self._options.s3_credentials_file, + self._options.s3_config_file, + self._options.s3_profile, + artifact_root, + urls[0], + local_cache, + ) + best_url_selector = BestUrlSelector(['{}/{}'.format(url.rstrip('/'), self._stable_name) + for url in self.get_available_urls(urls)]) return RESTfulArtifactCache(artifact_root, best_url_selector, local_cache) local_cache = create_local_cache(spec.local) if spec.local else None remote_cache = create_remote_cache(spec.remote, local_cache) if spec.remote else None + if remote_cache: return remote_cache return local_cache diff --git a/src/python/pants/cache/s3_artifact_cache.py b/src/python/pants/cache/s3_artifact_cache.py new file mode 100644 index 00000000000..f3e9d6eed4f --- /dev/null +++ b/src/python/pants/cache/s3_artifact_cache.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import logging +import os +from ConfigParser import ConfigParser, NoSectionError +from textwrap import dedent + +import boto3 +from botocore import exceptions +from botocore.config import Config +from botocore.vendored.requests import ConnectionError, Timeout +from botocore.vendored.requests.packages.urllib3.exceptions import ClosedPoolError +from six.moves.urllib.parse import urlparse + +from pants.cache.artifact_cache import ArtifactCache, NonfatalArtifactCacheError, UnreadableArtifact +from pants.util.memo import memoized, memoized_property + + +logger = logging.getLogger(__name__) + +_NETWORK_ERRORS = [ + ConnectionError, Timeout, ClosedPoolError, + exceptions.EndpointConnectionError, exceptions.ChecksumError +] + +# NOTE(mateo): Boto2 has a different set of configs/variables, this likely supports boto3 only. +_BOTO3_ENVS = { + 'profile': 'aws_profile', + 'access_key': 'aws_access_key_id', + 'secret_key': 'aws_secret_access_key', +} + +# Used if the cache_setup options do not define a config_file option. +_BOTO3_CONFIG_DEFAULT_KWARGS = dict(connect_timeout=4, read_timeout=4) + +# TODO: Read size is exposed both here and the restful client, should be wired as an option. +_READ_SIZE_BYTES = 4 * 1024 * 1024 + + +class S3ConfigException(Exception): + """Indicate a problem parsing or finding the config files for the s3 connection.""" + + +@memoized +def _connect_to_s3(creds_file, config_file, profile_name): + # Downgrading the boto logging since it spams the logs. + # TODO(mateo): Wire a boto logging option. + boto3.set_stream_logger(name='boto3.resources', level=logging.WARN) + boto3.set_stream_logger(name='botocore', level=logging.WARN) + auth_kwargs = {} + if creds_file: + config = ConfigParser() + config.read(creds_file) + try: + auth_kwargs['aws_access_key_id'] = config.get(profile_name, 'aws_access_key_id') + auth_kwargs['aws_secret_access_key'] = config.get(profile_name, 'aws_secret_access_key') + except NoSectionError as e: + # Actually raise here, since in this case we know that the user has passed a misconfigured + # option or input and that would be surprisingly unapplied. + raise S3ConfigException(dedent( + """ + Credentials file appears malformed. Should approximate: + + [] + aws_access_key_id = + aws_secret_access_key = + + """ + )) + # If no cred_file is passed, we allow Boto to attempt to consume from its respected + # environmental variables and/or traditional credential locations. + session = boto3.Session(**auth_kwargs) + config = config_file or Config(**_BOTO3_CONFIG_DEFAULT_KWARGS) + return session.resource('s3', config=config) + + +def iter_content(body): + while True: + chunk = body.read(_READ_SIZE_BYTES) + if not chunk: + break + yield chunk + + +def _not_found_error(e): + if not isinstance(e, exceptions.ClientError): + return False + return e.response['Error']['Code'] in ('404', 'NoSuchKey') + + +def _network_error(e): + return any(isinstance(e, cls) for cls in _NETWORK_ERRORS) + +_NOT_FOUND = 0 +_NETWORK = 1 +_UNKNOWN = 2 + + +def _log_and_classify_error(e, verb, cache_key): + if _not_found_error(e): + logger.debug('Not Found During {0} {1}'.format(verb, cache_key)) + return _NOT_FOUND + if _network_error(e): + logger.debug('Failed to {0} (network) {1}: {2}'.format(verb, cache_key, str(e))) + return _NETWORK + logger.debug('Failed to {0} (client) {1}: {2}'.format(verb, cache_key, str(e))) + return _UNKNOWN + + +class S3ArtifactCache(ArtifactCache): + """An artifact cache that stores the artifacts on S3.""" + + def __init__(self, creds_file, config_file, profile_name, artifact_root, s3_url, local): + """ + :param str creds_file: Path that holds AWS credentials as understood by Boto. + :param str config_file: Path that holds Boto config file. + :param str profile_name: Specifies a profile set in the creds file for use byBoto. + :param str artifact_root: The path under which cacheable products will be read/written. + :param str s3_url: URL of the form s3://bucket/path/to/store/artifacts. + :param BaseLocalArtifactCache local: local cache instance for storing and creating artifacts. + """ + super(S3ArtifactCache, self).__init__(artifact_root) + url = urlparse(s3_url) + self._creds_file = creds_file + self._config_file = config_file + self._path = url.path + if self._path.startswith('/'): + self._path = self._path[1:] + self._localcache = local + self._bucket = url.netloc + self.profile_name = profile_name + + @memoized_property + def creds_file(self): + if self._creds_file and not os.path.isfile(self._creds_file): + raise S3ConfigException( + "Could not find passed AWS credentials file: {}".format(self._creds_file) + ) + return self._creds_file + + @memoized_property + def config_file(self): + if self._config_file and not os.path.isfile(self._config_file): + raise S3ConfigException( + "Could not find passed Boto config file: {}".format(self._config_file) + ) + return self._config_file + + @memoized_property + def connection(self): + return _connect_to_s3(self.creds_file, self.config_file, self.profile_name) + + def try_insert(self, cache_key, paths): + logger.debug('Insert {0}'.format(cache_key)) + # Delegate creation of artifacts to the local cache + with self._localcache.insert_paths(cache_key, paths) as tarfile: + with open(tarfile, 'rb') as infile: + # Upload artifact to the remote cache. + try: + response = self._get_object(cache_key).put(Body=infile) + response_status = response['ResponseMetadata']['HTTPStatusCode'] + if response_status < 200 or response_status >= 300: + raise NonfatalArtifactCacheError('Failed to PUT (http error) {0}: {1}'.format( + cache_key, response_status)) + except Exception as e: + raise NonfatalArtifactCacheError( + 'Failed to PUT (core error) {0}: {1}'.format(cache_key, str(e))) + + def has(self, cache_key): + logger.debug('Has {0}'.format(cache_key)) + if self._localcache.has(cache_key): + return True + try: + self._get_object(cache_key).load() + return True + except Exception as e: + _log_and_classify_error(e, 'HEAD', cache_key) + return False + + def use_cached_files(self, cache_key, results_dir=None): + logger.debug('GET {0}'.format(cache_key)) + if self._localcache.has(cache_key): + return self._localcache.use_cached_files(cache_key, results_dir) + + s3_object = self._get_object(cache_key) + try: + get_result = s3_object.get() + except Exception as e: + _log_and_classify_error(e, 'GET', cache_key) + return False + + # Delegate storage and extraction to local cache + body = get_result['Body'] + try: + return self._localcache.store_and_use_artifact( + cache_key, iter_content(body), results_dir) + except Exception as e: + result = _log_and_classify_error(e, 'GET', cache_key) + if result == _UNKNOWN: + return UnreadableArtifact(cache_key, e) + return False + finally: + body.close() + + def delete(self, cache_key): + logger.debug("Delete {0}".format(cache_key)) + self._localcache.delete(cache_key) + try: + self._get_object(cache_key).delete() + except Exception as e: + _log_and_classify_error(e, 'DELETE', cache_key) + + def _get_object(self, cache_key): + return self.connection.Object(self._bucket, self._path_for_key(cache_key)) + + def _path_for_key(self, cache_key): + return '{0}/{1}/{2}.tgz'.format(self._path, cache_key.id, cache_key.hash) diff --git a/src/python/pants/core_tasks/changed_target_tasks.py b/src/python/pants/core_tasks/changed_target_tasks.py index 26c108927e8..e8bfcdb2243 100644 --- a/src/python/pants/core_tasks/changed_target_tasks.py +++ b/src/python/pants/core_tasks/changed_target_tasks.py @@ -15,8 +15,7 @@ class CompileChanged(ChangedTargetTask): """Find and compile changed targets.""" @classmethod - @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD compile` instead.', - '`./pants compile-changed`') + # @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD compile` instead.', '`./pants compile-changed`') def prepare(cls, options, round_manager): super(CompileChanged, cls).prepare(options, round_manager) round_manager.require_data(NoopCompile.product_types()[0]) @@ -26,8 +25,7 @@ class TestChanged(ChangedTargetTask): """Find and test changed targets.""" @classmethod - @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD test` instead.', - '`./pants test-changed`') + # @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD test` instead.', '`./pants test-changed`') def prepare(cls, options, round_manager): super(TestChanged, cls).prepare(options, round_manager) round_manager.require_data(NoopTest.product_types()[0]) diff --git a/src/python/pants/core_tasks/what_changed.py b/src/python/pants/core_tasks/what_changed.py index e163fa8b555..29d78a81104 100644 --- a/src/python/pants/core_tasks/what_changed.py +++ b/src/python/pants/core_tasks/what_changed.py @@ -26,8 +26,7 @@ def register_options(cls, register): def subsystem_dependencies(cls): return super(WhatChanged, cls).subsystem_dependencies() + (Changed.Factory,) - @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD list` instead.', - '`./pants changed`') + # @deprecated('1.5.0.dev0', 'Use e.g. `./pants --changed-parent=HEAD list` instead.', '`./pants changed`') def console_output(self, _): # N.B. This task shares an options scope ('changed') with the `Changed` subsystem. options = self.get_options() diff --git a/src/python/pants/engine/legacy/BUILD b/src/python/pants/engine/legacy/BUILD index 293feb1bdc5..b04018ef08c 100644 --- a/src/python/pants/engine/legacy/BUILD +++ b/src/python/pants/engine/legacy/BUILD @@ -77,6 +77,7 @@ python_library( 'src/python/pants/base:specs', 'src/python/pants/build_graph', 'src/python/pants/source', + '3rdparty/python:six' ] ) diff --git a/src/python/pants/engine/legacy/source_mapper.py b/src/python/pants/engine/legacy/source_mapper.py index 274a2aabdf8..6b0df0e15bc 100644 --- a/src/python/pants/engine/legacy/source_mapper.py +++ b/src/python/pants/engine/legacy/source_mapper.py @@ -7,6 +7,8 @@ import os +import six + from pants.base.specs import AscendantAddresses, SingleAddress from pants.build_graph.address import parse_spec from pants.build_graph.source_mapper import SourceMapper @@ -105,13 +107,15 @@ def iter_target_addresses_for_sources(self, sources): sources_set = set(sources) subjects = [AscendantAddresses(directory=d) for d in self._unique_dirs_for_sources(sources_set)] + # Uniqify all transitive hydrated targets. + hydrated_target_to_address = {} for hydrated_targets in self._scheduler.product_request(HydratedTargets, subjects): for hydrated_target in hydrated_targets.dependencies: - legacy_address = hydrated_target.adaptor.address - - # Handle BUILD files. - if any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set): - yield legacy_address - else: - if any(self._owns_source(source, hydrated_target) for source in sources_set): - yield legacy_address + if hydrated_target not in hydrated_target_to_address: + hydrated_target_to_address[hydrated_target] = hydrated_target.adaptor.address + + for hydrated_target, legacy_address in six.iteritems(hydrated_target_to_address): + # Handle BUILD files. + if (any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set) or + any(self._owns_source(source, hydrated_target) for source in sources_set)): + yield legacy_address diff --git a/src/python/pants/engine/subsystem/native_engine_version b/src/python/pants/engine/subsystem/native_engine_version new file mode 100644 index 00000000000..4e807f43236 --- /dev/null +++ b/src/python/pants/engine/subsystem/native_engine_version @@ -0,0 +1 @@ +af2e5b09cb7eee743c8d4e065258ac046c8a19ac diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py index 16a32133905..e0f3aaa2a0e 100644 --- a/src/python/pants/goal/run_tracker.py +++ b/src/python/pants/goal/run_tracker.py @@ -194,7 +194,7 @@ def initialize(self): return run_id - def start(self, report): + def start(self, report, run_start_time=None): """Start tracking this pants run using the given Report. `RunTracker.initialize` must have been called first to create the run_info_dir and @@ -213,7 +213,8 @@ def start(self, report): self._main_root_workunit = WorkUnit(run_info_dir=self.run_info_dir, parent=None, name=RunTracker.DEFAULT_ROOT_NAME, cmd=None) self.register_thread(self._main_root_workunit) - self._main_root_workunit.start() + # Set the true start time in the case of e.g. the daemon. + self._main_root_workunit.start(run_start_time) self.report.start_workunit(self._main_root_workunit) # Log reporting details. diff --git a/src/python/pants/notes/1.4.x.rst b/src/python/pants/notes/1.4.x.rst index 458d5bd924c..1d31bcb3b75 100644 --- a/src/python/pants/notes/1.4.x.rst +++ b/src/python/pants/notes/1.4.x.rst @@ -3,6 +3,117 @@ This document describes releases leading up to the ``1.4.x`` ``stable`` series. +1.4.0 (03/10/2018) +------------------ + +The ``1.4.0`` stable release, with no additional changes since the ``rc5`` release. Thanks for +your patience! + +A quick summary of the changes since the ``1.3.x`` branch: + +* ``pantsd`` should be usable for almost all usecases, although it is not `quite` + ready to enable by default. Please try it out! +* There are a few new contrib plugins: ``codeanalysis`` (aka: kythe), ``thrifty``, ``avro``, + ``mypy``, ``confluence``. +* Both pytest and junit now have support for ``--chroot`` and a ``files(..)`` target was added + to support loose-file isolation in unit tests, and to improve the safety of test caching. +* junit also gained support for ``--no-fast``, allowing running each ``junit_test`` target in + a separate VM. +* Beta support for ``coursier`` for artifact resolving: expected to be stable in the + ``1.5.x`` release series. +* PEX files support bringing interpreter constraints with them to their runtime + environment: see `PR #5160 `_. +* Improvements to the ``missing-dep-suggest`` task that runs when ``strict_deps`` is enabled + for a Java or Scala target. +* The ``--cache-ignore`` option was added to support forcing a rebuild of any particular + task without requiring clean-all. Particularly helpful in these early days of test caching! +* Pants reports per-target compile and test times to its stats URL via a general + "target-level metrics" API. Add per-target stats for your Tasks! +* ``java_thrift_library`` now takes a ``compiler_args`` property, which deprecates ``rpc_style`` + and ``language``. +* "implicit sources" are now enabled by default! If you do not specify a ``sources`` argument + on your target, a default sources glob will be used. + See https://www.pantsbuild.org/build_files.html for more info. + +See the rest of this page for the complete list of changes. + +1.4.0rc5 (03/07/2018) +--------------------- + +The sixth release candidate for the ``1.4.x`` stable branch. + +Bugfixes +~~~~~~~~ + +* Improve the performance of v2 changed. (#5571) + `PR #5571 `_ + +1.4.0rc4 (03/06/2018) +--------------------- + +The fifth release candidate for the ``1.4.x`` stable branch. + +Bugfixes +~~~~~~~~ + +* Set thrifty build_file_aliases (#5559) (#5562) + `PR #5559 `_ + +1.4.0rc3 (03/05/2018) +--------------------- + +The fourth release candidate for the ``1.4.x`` stable branch. + +New Features +~~~~~~~~~~~~ + +* Thrifty support for pants (#5531) + `PR #5531 `_ + +Bugfixes +~~~~~~~~ + +* [pantsd] Repair end to end runtracker timing for pantsd runs. (#5526) + `PR #5526 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* [pantsd] Repair pantsd integration tests for execution via pantsd. (#5387) + `PR #5387 `_ + +1.4.0rc2 (02/23/2018) +--------------------- + +The third release candidate for the ``1.4.x`` stable branch; likely the last! + +New Features +~~~~~~~~~~~~ + +* Add pantsd client profiling (#5434) + `PR #5434 `_ + +* Release script allows wheel listing (#5431) + `PR #5431 `_ + +Bugfixes +~~~~~~~~ + +* [pantsd] Set the remote environment for pantsd-runner and child processes. (#5508) + `PR #5508 `_ + +* Bump release.sh to pex 1.2.16. (#5460) + `PR #5460 `_ + +* Get version from version file not by running pants (#5428) + `PR #5428 `_ + +* [pantsd] Don't invalidate on surface name changes to config/rc files. (#5408) + `PR #5408 `_ + +* Set the log level when capturing logs in tests. (#5418) + `PR #5418 `_ + 1.4.0rc1 (01/28/2018) --------------------- diff --git a/src/python/pants/reporting/reporting.py b/src/python/pants/reporting/reporting.py index 997bcdadd60..c4ab57f3207 100644 --- a/src/python/pants/reporting/reporting.py +++ b/src/python/pants/reporting/reporting.py @@ -46,7 +46,7 @@ def register_options(cls, register): '{workunits}. Possible formatting values are {formats}'.format( workunits=WorkUnitLabel.keys(), formats=ToolOutputFormat.keys())) - def initialize(self, run_tracker): + def initialize(self, run_tracker, start_time=None): """Initialize with the given RunTracker. TODO: See `RunTracker.start`. @@ -88,7 +88,7 @@ def initialize(self, run_tracker): run_tracker.run_info.add_info('report_url', 'http://localhost:{}/run/{}'.format(port, run_id)) # And start tracking the run. - run_tracker.start(report) + run_tracker.start(report, start_time) def _get_invalidation_report(self): return InvalidationReport() if self.get_options().invalidation_report else None diff --git a/src/python/pants/source/wrapped_globs.py b/src/python/pants/source/wrapped_globs.py index 1367ce218b2..29cb9d13944 100644 --- a/src/python/pants/source/wrapped_globs.py +++ b/src/python/pants/source/wrapped_globs.py @@ -13,7 +13,7 @@ from twitter.common.dirutil.fileset import Fileset from pants.base.build_environment import get_buildroot -from pants.util.dirutil import fast_relpath +from pants.util.dirutil import fast_relpath, fast_relpath_optional from pants.util.memo import memoized_property from pants.util.meta import AbstractClass @@ -104,8 +104,8 @@ def __repr__(self): return 'EagerFilesetWithSpec(rel_root={!r}, files={!r})'.format(self.rel_root, self.files) def matches(self, path_from_buildroot): - path_relative_to_rel_root = os.path.relpath(path_from_buildroot, self.rel_root) - return path_relative_to_rel_root in self._files + path_relative_to_rel_root = fast_relpath_optional(path_from_buildroot, self.rel_root) + return path_relative_to_rel_root is not None and path_relative_to_rel_root in self._files class LazyFilesetWithSpec(FilesetWithSpec): diff --git a/src/python/pants/util/contextutil.py b/src/python/pants/util/contextutil.py index c004823d371..118544a8795 100644 --- a/src/python/pants/util/contextutil.py +++ b/src/python/pants/util/contextutil.py @@ -55,6 +55,17 @@ def setenv(key, val): setenv(key, val) +@contextmanager +def hermetic_environment_as(**kwargs): + """Set the environment to the supplied values from an empty state.""" + old_environment, os.environ = os.environ, {} + try: + with environment_as(**kwargs): + yield + finally: + os.environ = old_environment + + @contextmanager def _stdio_stream_as(src_fd, dst_fd, dst_sys_attribute, mode): """Replace the given dst_fd and attribute on `sys` with an open handle to the given src_fd.""" diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py index 1c621281acb..a659c95f9bc 100644 --- a/tests/python/pants_test/base_test.py +++ b/tests/python/pants_test/base_test.py @@ -480,9 +480,16 @@ def warnings(self): return self._messages_for_level('WARNING') @contextmanager - def captured_logging(self): + def captured_logging(self, level=None): + root_logger = logging.getLogger() + + old_level = root_logger.level + root_logger.setLevel(level or logging.NOTSET) + handler = self.LoggingRecorder() - logger = logging.getLogger('') - logger.addHandler(handler) - yield handler - logger.removeHandler(handler) + root_logger.addHandler(handler) + try: + yield handler + finally: + root_logger.setLevel(old_level) + root_logger.removeHandler(handler) diff --git a/tests/python/pants_test/cache/BUILD b/tests/python/pants_test/cache/BUILD index fd136e7164b..eeff9b39e58 100644 --- a/tests/python/pants_test/cache/BUILD +++ b/tests/python/pants_test/cache/BUILD @@ -101,6 +101,19 @@ python_tests( ] ) +python_tests( + name = 's3_artifact_cache', + sources = ['test_s3_artifact_cache.py'], + dependencies = [ + '3rdparty/python:boto3', + '3rdparty/python:moto', + 'src/python/pants/cache', + 'src/python/pants/invalidation', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + ] +) + python_tests( name = 'cache_cleanup_integration', sources = ['test_cache_cleanup_integration.py'], diff --git a/tests/python/pants_test/cache/test_cache_setup.py b/tests/python/pants_test/cache/test_cache_setup.py index b574b51a691..accc23e3b8a 100644 --- a/tests/python/pants_test/cache/test_cache_setup.py +++ b/tests/python/pants_test/cache/test_cache_setup.py @@ -16,6 +16,7 @@ from pants.cache.local_artifact_cache import LocalArtifactCache from pants.cache.resolver import Resolver from pants.cache.restful_artifact_cache import RESTfulArtifactCache +from pants.cache.s3_artifact_cache import S3ArtifactCache from pants.subsystem.subsystem import Subsystem from pants.task.task import Task from pants.util.contextutil import temporary_dir @@ -186,6 +187,9 @@ def check(expected_type, spec, resolver=None): check(RESTfulArtifactCache, [cachedir, 'http://localhost/bar']) check(RESTfulArtifactCache, [cachedir, 'http://localhost/bar'], resolver=self.resolver) + check(S3ArtifactCache, ['s3://some-bucket/bar']) + check(S3ArtifactCache, [cachedir, 's3://some-bucket/bar']) + with self.assertRaises(CacheSpecFormatError): mk_cache(['foo']) @@ -201,6 +205,12 @@ def check(expected_type, spec, resolver=None): with self.assertRaises(TooManyCacheSpecsError): mk_cache([tmpdir, self.REMOTE_URI_1, self.REMOTE_URI_2]) + with self.assertRaises(InvalidCacheSpecError): + mk_cache(['s3://some-bucket/bar|http://localhost/bar']) + + with self.assertRaises(InvalidCacheSpecError): + mk_cache(['s3://some-bucket/bar|s3://some-other-bucket/foo']) + def test_read_cache_available(self): self.assertFalse(self.cache_factory(ignore=True, read=True, read_from=[self.EMPTY_URI]) .read_cache_available()) diff --git a/tests/python/pants_test/cache/test_s3_artifact_cache.py b/tests/python/pants_test/cache/test_s3_artifact_cache.py new file mode 100644 index 00000000000..49b978aea69 --- /dev/null +++ b/tests/python/pants_test/cache/test_s3_artifact_cache.py @@ -0,0 +1,198 @@ +# coding=utf-8 +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os +from contextlib import contextmanager + +import boto3 +import pytest +from moto import mock_s3 + +from pants.cache.artifact_cache import UnreadableArtifact +from pants.cache.local_artifact_cache import LocalArtifactCache, TempLocalArtifactCache +from pants.cache.s3_artifact_cache import S3ArtifactCache +from pants.invalidation.build_invalidator import CacheKey +from pants.util.contextutil import temporary_dir, temporary_file +from pants.util.dirutil import safe_mkdir + + +_TEST_CONTENT1 = b'fraggle' +_TEST_CONTENT2 = b'gobo' + +_TEST_BUCKET = 'verst-test-bucket' + + +@pytest.yield_fixture(scope="function") +def local_artifact_root(): + with temporary_dir() as artifact_root: + yield artifact_root + + +@pytest.yield_fixture(scope="function") +def local_cache(local_artifact_root): + with temporary_dir() as cache_root: + yield LocalArtifactCache( + local_artifact_root, cache_root, compression=1) + + +@pytest.fixture( + scope="function", + params=[True, False], + ids=['temp-local-cache', 'local-cache'] +) +def tmp_and_local_cache(request, local_artifact_root, local_cache): + if request.param: + return TempLocalArtifactCache(local_artifact_root, 0) + else: + return local_cache + + +@pytest.yield_fixture(scope="function", autouse=True) +def s3_fixture(): + mock_s3().start() + + try: + s3 = boto3.resource('s3') + s3.create_bucket(Bucket=_TEST_BUCKET) + yield s3 + finally: + mock_s3().stop() + + +@pytest.fixture(scope="function") +def s3_cache_instance(local_artifact_root, local_cache): + with temporary_dir() as config_root: + return S3ArtifactCache( + os.path.join(config_root, 'non-existant'), None, + local_artifact_root, 's3://' + _TEST_BUCKET, local_cache) + + +@pytest.fixture(scope="function") +def tmp_and_local_s3_cache_instance( + local_artifact_root, tmp_and_local_cache): + with temporary_dir() as config_root: + return S3ArtifactCache( + os.path.join(config_root, 'non-existant'), None, + local_artifact_root, 's3://' + _TEST_BUCKET, tmp_and_local_cache) + + +@pytest.fixture() +def cache_key(): + return CacheKey('some_test_key', '1dfa0d08e47406038dda4ca5019c05c7977cb28c') + + +@pytest.yield_fixture() +def artifact_path(local_artifact_root): + with setup_test_file(local_artifact_root) as path: + yield path + + +@pytest.yield_fixture(scope="function") +def other_machine_cache(): + with temporary_dir() as artifact_root: + with temporary_dir() as cache_root: + local_cache = LocalArtifactCache( + artifact_root, cache_root, compression=1) + + with temporary_dir() as config_root: + yield S3ArtifactCache( + os.path.join(config_root, 'non-existant'), None, + artifact_root, 's3://' + _TEST_BUCKET, local_cache) + + +@contextmanager +def setup_test_file(parent): + with temporary_file(parent) as f: + # Write the file. + f.write(_TEST_CONTENT1) + path = f.name + f.close() + yield path + + +def test_basic_combined_cache( + tmp_and_local_s3_cache_instance, cache_key, artifact_path): + instance = tmp_and_local_s3_cache_instance + assert not instance.has(cache_key) + assert not instance.use_cached_files(cache_key) + + instance.insert(cache_key, [artifact_path]) + assert instance.has(cache_key) + + # Stomp it. + with open(artifact_path, 'w') as outfile: + outfile.write(_TEST_CONTENT2) + + # Recover it from the cache. + assert instance.use_cached_files(cache_key) + + # Check that it was recovered correctly. + with open(artifact_path, 'r') as infile: + content = infile.read() + assert content == _TEST_CONTENT1 + + # Delete it. + instance.delete(cache_key) + assert not instance.has(cache_key) + + +def test_multi_machine_combined( + s3_cache_instance, other_machine_cache, cache_key, + local_cache, artifact_path): + # Insert it into S3 from the other machine. + other_machine_cache.insert(cache_key, [artifact_path]) + + # Our machine doesn't have it ... + assert not local_cache.has(cache_key) + assert not local_cache.has(cache_key) + + # But can use it. + assert s3_cache_instance.has(cache_key) + assert s3_cache_instance.use_cached_files(cache_key) + + # And now we should have backfilled local: + assert local_cache.has(cache_key) + assert local_cache.use_cached_files(cache_key) + + +def test_corrupted_cached_file_cleaned_up( + local_artifact_root, + s3_fixture, s3_cache_instance, other_machine_cache, + cache_key): + local_results_dir = os.path.join(local_artifact_root, 'a/sub/dir') + remote_results_dir = os.path.join( + other_machine_cache.artifact_root, 'a/sub/dir') + safe_mkdir(local_results_dir) + safe_mkdir(remote_results_dir) + assert os.path.exists(local_results_dir) + assert os.path.exists(remote_results_dir) + + with setup_test_file(remote_results_dir) as path: + other_machine_cache.insert(cache_key, [path]) + + object = s3_fixture.Object(_TEST_BUCKET, s3_cache_instance._path_for_key(cache_key)) + object.put(Body=b'not a valid tgz any more') + + result = s3_cache_instance.use_cached_files( + cache_key, results_dir=local_results_dir) + + assert isinstance(result, UnreadableArtifact) + + # The local artifact should not have been stored, and the results_dir should exist, + # but be empty. + + assert os.path.exists(local_results_dir) + assert len(os.listdir(local_results_dir)) == 0 + + +def test_delete_non_existant_key(s3_cache_instance): + s3_cache_instance.delete(CacheKey('foo', 'bar')) + # Should not raise an exception + + +def test_use_cached_files_non_existant_key(s3_cache_instance): + assert not s3_cache_instance.use_cached_files(CacheKey('foo', 'bar')) diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py index ecb7f0a159c..b4c761f22b8 100644 --- a/tests/python/pants_test/pantsd/test_pantsd_integration.py +++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py @@ -18,10 +18,10 @@ from pants.pantsd.process_manager import ProcessManager from pants.util.collections import combined_dict -from pants.util.contextutil import temporary_dir +from pants.util.contextutil import environment_as, temporary_dir from pants.util.dirutil import touch from pants_test.pants_run_integration_test import PantsRunIntegrationTest -from pants_test.testutils.process_test_util import assert_no_process_exists_by_command +from pants_test.testutils.process_test_util import no_lingering_process_by_command class PantsDaemonMonitor(ProcessManager): @@ -86,32 +86,32 @@ def join(): class TestPantsDaemonIntegration(PantsRunIntegrationTest): @contextmanager def pantsd_test_context(self, log_level='info'): - with self.temporary_workdir() as workdir_base: - pid_dir = os.path.join(workdir_base, '.pids') - workdir = os.path.join(workdir_base, '.workdir.pants.d') - print('\npantsd log is {}/pantsd/pantsd.log'.format(workdir)) - pantsd_config = { - 'GLOBAL': { - 'enable_pantsd': True, - # The absolute paths in CI can exceed the UNIX socket path limitation - # (>104-108 characters), so we override that here with a shorter path. - 'watchman_socket_path': '/tmp/watchman.{}.sock'.format(os.getpid()), - 'level': log_level, - 'pants_subprocessdir': pid_dir + with no_lingering_process_by_command('pantsd-runner'): + with self.temporary_workdir() as workdir_base: + pid_dir = os.path.join(workdir_base, '.pids') + workdir = os.path.join(workdir_base, '.workdir.pants.d') + print('\npantsd log is {}/pantsd/pantsd.log'.format(workdir)) + pantsd_config = { + 'GLOBAL': { + 'enable_pantsd': True, + # The absolute paths in CI can exceed the UNIX socket path limitation + # (>104-108 characters), so we override that here with a shorter path. + 'watchman_socket_path': '/tmp/watchman.{}.sock'.format(os.getpid()), + 'level': log_level, + 'pants_subprocessdir': pid_dir + } } - } - checker = PantsDaemonMonitor(pid_dir) - self.assert_success_runner(workdir, pantsd_config, ['kill-pantsd']) - try: - yield workdir, pantsd_config, checker - finally: - banner('BEGIN pantsd.log') - for line in read_pantsd_log(workdir): - print(line) - banner('END pantsd.log') + checker = PantsDaemonMonitor(pid_dir) self.assert_success_runner(workdir, pantsd_config, ['kill-pantsd']) - checker.assert_stopped() - assert_no_process_exists_by_command('pantsd-runner') + try: + yield workdir, pantsd_config, checker + finally: + banner('BEGIN pantsd.log') + for line in read_pantsd_log(workdir): + print(line) + banner('END pantsd.log') + self.assert_success_runner(workdir, pantsd_config, ['kill-pantsd']) + checker.assert_stopped() @contextmanager def pantsd_successful_run_context(self, log_level='info'): @@ -206,22 +206,21 @@ def test_pantsd_stacktrace_dump(self): self.assertIn('Current thread 0x', '\n'.join(read_pantsd_log(workdir))) def test_pantsd_pantsd_runner_doesnt_die_after_failed_run(self): - with self.pantsd_test_context() as (workdir, pantsd_config, checker): - # Run target that throws an exception in pants. - self.assert_failure( - self.run_pants_with_workdir( - ['bundle', 'testprojects/src/java/org/pantsbuild/testproject/bundle:missing-files'], - workdir, - pantsd_config) - ) - checker.await_pantsd() - - # Check for no stray pantsd-runner prcesses. - assert_no_process_exists_by_command('pantsd-runner') - - # Assert pantsd is in a good functional state. - self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) - checker.assert_running() + # Check for no stray pantsd-runner prcesses. + with no_lingering_process_by_command('pantsd-runner'): + with self.pantsd_test_context() as (workdir, pantsd_config, checker): + # Run target that throws an exception in pants. + self.assert_failure( + self.run_pants_with_workdir( + ['bundle', 'testprojects/src/java/org/pantsbuild/testproject/bundle:missing-files'], + workdir, + pantsd_config) + ) + checker.await_pantsd() + + # Assert pantsd is in a good functional state. + self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) + checker.assert_running() def test_pantsd_lifecycle_invalidation(self): """Runs pants commands with pantsd enabled, in a loop, alternating between options that @@ -292,15 +291,16 @@ def test_pantsd_stray_runners(self): # Allow env var overrides for local stress testing. attempts = int(os.environ.get('PANTS_TEST_PANTSD_STRESS_ATTEMPTS', 20)) cmd = os.environ.get('PANTS_TEST_PANTSD_STRESS_CMD', 'help').split() - with self.pantsd_successful_run_context('debug') as (pantsd_run, checker, _): - pantsd_run(cmd) - checker.await_pantsd() - for _ in range(attempts): + + with no_lingering_process_by_command('pantsd-runner'): + with self.pantsd_successful_run_context('debug') as (pantsd_run, checker, _): pantsd_run(cmd) - checker.assert_running() - # The runner can sometimes exit more slowly than the thin client caller. - time.sleep(3) - assert_no_process_exists_by_command('pantsd-runner') + checker.await_pantsd() + for _ in range(attempts): + pantsd_run(cmd) + checker.assert_running() + # The runner can sometimes exit more slowly than the thin client caller. + time.sleep(3) def test_pantsd_aligned_output(self): # Set for pytest output display. @@ -342,3 +342,43 @@ def test_pantsd_filesystem_invalidation(self): checker.assert_running() join() + + def test_pantsd_client_env_var_is_inherited_by_pantsd_runner_children(self): + EXPECTED_VALUE = '333' + with self.pantsd_successful_run_context() as (pantsd_run, checker, workdir): + # First, launch the daemon without any local env vars set. + pantsd_run(['help']) + checker.await_pantsd() + + # Then, set an env var on the secondary call. + with environment_as(TEST_ENV_VAR_FOR_PANTSD_INTEGRATION_TEST=EXPECTED_VALUE): + result = pantsd_run( + ['-q', + 'run', + 'testprojects/src/python/print_env', + '--', + 'TEST_ENV_VAR_FOR_PANTSD_INTEGRATION_TEST'] + ) + checker.assert_running() + + self.assertEquals(EXPECTED_VALUE, ''.join(result.stdout_data).strip()) + + def test_pantsd_launch_env_var_is_not_inherited_by_pantsd_runner_children(self): + with self.pantsd_test_context() as (workdir, pantsd_config, checker): + with environment_as(NO_LEAKS='33'): + self.assert_success( + self.run_pants_with_workdir( + ['help'], + workdir, + pantsd_config) + ) + checker.await_pantsd() + + self.assert_failure( + self.run_pants_with_workdir( + ['-q', 'run', 'testprojects/src/python/print_env', '--', 'NO_LEAKS'], + workdir, + pantsd_config + ) + ) + checker.assert_running() diff --git a/tests/python/pants_test/pantsd/test_process_manager.py b/tests/python/pants_test/pantsd/test_process_manager.py index ba7abdfbb2e..adcee500f57 100644 --- a/tests/python/pants_test/pantsd/test_process_manager.py +++ b/tests/python/pants_test/pantsd/test_process_manager.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import errno +import logging import os import sys from contextlib import contextmanager @@ -141,7 +142,7 @@ def test_readwrite_metadata_by_name(self): def test_deadline_until(self): with self.assertRaises(self.pmm.Timeout): - with self.captured_logging() as captured: + with self.captured_logging(logging.INFO) as captured: self.pmm._deadline_until(lambda: False, 'the impossible', timeout=.5, info_interval=.1) self.assertTrue(4 <= len(captured.infos()) <= 6, 'Expected between 4 and 6 infos, got: {}'.format(captured.infos())) diff --git a/tests/python/pants_test/testutils/process_test_util.py b/tests/python/pants_test/testutils/process_test_util.py index 7a8e1043170..b75b8b160ad 100644 --- a/tests/python/pants_test/testutils/process_test_util.py +++ b/tests/python/pants_test/testutils/process_test_util.py @@ -5,6 +5,8 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +from contextlib import contextmanager + import psutil @@ -12,14 +14,37 @@ class ProcessStillRunning(AssertionError): """Raised when a process shouldn't be running but is.""" -def assert_no_process_exists_by_command(name): - """Asserts that no process exists for a given command with a helpful error.""" +def _safe_iter_matching_processes(name): for proc in psutil.process_iter(): try: - cmdline = proc.cmdline() - if name in ''.join(cmdline): - raise ProcessStillRunning( - 'a {} process was detected at PID {} (cmdline={})'.format(name, proc.pid, cmdline) - ) + if name in ''.join([part.decode('utf-8') for part in proc.cmdline()]): + yield proc except (psutil.NoSuchProcess, psutil.AccessDenied): pass + + +def _make_process_table(processes): + line_tmpl = '{0:>7} {1:>7} {2}' + proc_tuples = [(p.pid, p.ppid(), ''.join(p.cmdline())) for p in processes] + return '\n'.join( + [ + line_tmpl.format('PID', 'PGID', 'CMDLINE') + ] + [ + line_tmpl.format(*t) for t in sorted(proc_tuples) + ] + ) + + +@contextmanager +def no_lingering_process_by_command(name): + """Asserts that no process exists for a given command with a helpful error, excluding + existing processes outside of the scope of the contextmanager.""" + before_processes = set(_safe_iter_matching_processes(name)) + yield + after_processes = set(_safe_iter_matching_processes(name)) + delta_processes = after_processes.difference(before_processes) + if delta_processes: + raise ProcessStillRunning( + '{} {} processes lingered after tests:\n{}' + .format(len(delta_processes), name, _make_process_table(delta_processes)) + ) diff --git a/tests/python/pants_test/util/test_contextutil.py b/tests/python/pants_test/util/test_contextutil.py index f9d4507a74e..2a018636647 100644 --- a/tests/python/pants_test/util/test_contextutil.py +++ b/tests/python/pants_test/util/test_contextutil.py @@ -18,9 +18,9 @@ import mock from pants.util.contextutil import (HardSystemExit, InvalidZipPath, Timer, environment_as, - exception_logging, hard_exit_handler, maybe_profiled, open_zip, - pushd, signal_handler_as, stdio_as, temporary_dir, - temporary_file) + exception_logging, hard_exit_handler, hermetic_environment_as, + maybe_profiled, open_zip, pushd, signal_handler_as, stdio_as, + temporary_dir, temporary_file) from pants.util.process_handler import subprocess @@ -59,6 +59,11 @@ def test_environment_negation(self): output.seek(0) self.assertEquals('False\n', output.read()) + def test_hermetic_environment(self): + self.assertIn('USER', os.environ) + with hermetic_environment_as(**{}): + self.assertNotIn('USER', os.environ) + def test_simple_pushd(self): pre_cwd = os.getcwd() with temporary_dir() as tempdir: diff --git a/tests/python/pants_test/util/test_osutil.py b/tests/python/pants_test/util/test_osutil.py index c9379470258..320a4cd1b37 100644 --- a/tests/python/pants_test/util/test_osutil.py +++ b/tests/python/pants_test/util/test_osutil.py @@ -5,6 +5,8 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import logging + from pants.util.osutil import OS_ALIASES, known_os_names, normalize_os_name from pants_test.base_test import BaseTest @@ -22,14 +24,14 @@ def test_keys_in_aliases(self): def test_no_warnings_on_known_names(self): for name in known_os_names(): - with self.captured_logging() as captured: + with self.captured_logging(logging.WARNING) as captured: normalize_os_name(name) self.assertEqual(0, len(captured.warnings()), 'Recieved unexpected warnings: {}'.format(captured.warnings())) def test_warnings_on_unknown_names(self): name = 'I really hope no one ever names an operating system with this string.' - with self.captured_logging() as captured: + with self.captured_logging(logging.WARNING) as captured: normalize_os_name(name) self.assertEqual(1, len(captured.warnings()), 'Expected exactly one warning, but got: {}'.format(captured.warnings()))