Mateo.1.4.0+fs x fork

3rdparty/python/requirements.txt

@@ -2,28 +2,31 @@ ansicolors==1.0.2
 beautifulsoup4>=4.3.2,<4.4
 cffi==1.11.1
 contextlib2==0.5.5
+boto3==1.4.4
 coverage>=4.3.4,<4.4
 docutils>=0.12,<0.13
 fasteners==0.14.1
 faulthandler==2.6
-futures==3.0.5
+futures<4.0.0,>=2.2.0
 isort==4.2.5
 Markdown==2.1.1
 mock==2.0.0
+moto==1.2.0
 packaging==16.8
 pathspec==0.5.0
 parameterized==0.6.1
-pep8==1.6.2
 pex==1.2.16
+pep8==1.5.0
 psutil==4.3.0
 pyflakes==1.1.0
-Pygments==1.4
 pyopenssl==17.3.0
+Pygments==2.1.3
+pyjavaproperties==0.6
 pystache==0.5.3
 pytest-cov>=2.4,<2.5
 pytest>=3.0.7,<4.0
 pywatchman==1.4.1
-requests[security]>=2.5.0,<2.19
+requests[security]>=2.8.14
 scandir==1.2
 setproctitle==1.1.10
 setuptools==30.0.0

build-support/bin/native/bootstrap.sh

@@ -82,7 +82,7 @@ function ensure_native_build_prerequisites() {
     log "A pants owned rustup installation could not be found, installing via the instructions at" \
         "https://www.rustup.rs ..."
     local readonly rustup=$(mktemp -t pants.rustup.XXXXXX)
-    curl https://sh.rustup.rs -sSf > ${rustup}
+    curl https://sh.rustup.rs -sSf > ${rustup} || (echo "Bad curl, trying wget" && wget https://sh.rustup.rs -O- > ${rustup})
     sh ${rustup} -y --no-modify-path --default-toolchain "${rust_toolchain}" 1>&2
     rm -f ${rustup}
   fi

build-support/bin/release.sh

@@ -16,7 +16,7 @@ function run_local_pants() {
 # NB: Pants core does not have the ability to change its own version, so we compute the
 # suffix here and mutate the VERSION_FILE to affect the current version.
 readonly HEAD_SHA=$(git rev-parse --verify HEAD)
-readonly PANTS_STABLE_VERSION="$(run_local_pants --version 2>/dev/null)"
+readonly PANTS_STABLE_VERSION="$(cat "${ROOT}/src/python/pants/VERSION")"
 readonly PANTS_UNSTABLE_VERSION="${PANTS_STABLE_VERSION}+${HEAD_SHA:0:8}"
 
 readonly DEPLOY_DIR="${ROOT}/dist/deploy"
@@ -682,7 +682,7 @@ function build_pex() {
 
   local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex"
 
-  activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.2.13" || die "Failed to install pex."
+  activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.2.16" || die "Failed to install pex."
 
   local requirements_string=""
   for pkg_name in $PANTS_PEX_PACKAGES; do
@@ -764,7 +764,7 @@ function usage() {
   fi
 }
 
-while getopts "hdntcloep" opt; do
+while getopts "hdntcloepw" opt; do
   case ${opt} in
     h) usage ;;
     d) debug="true" ;;
@@ -774,6 +774,7 @@ while getopts "hdntcloep" opt; do
     o) list_owners ; exit $? ;;
     e) fetch_and_check_prebuilt_wheels ; exit $? ;;
     p) build_pex ; exit $? ;;
+    w) list_prebuilt_wheels ; exit $? ;;
     *) usage "Invalid option: -${OPTARG}" ;;
   esac
 done

contrib/go/examples/3rdparty/go/github.com/apache/thrift/BUILD

@@ -0,0 +1,9 @@
+# Auto-generated by pants!
+# To re-generate run: `pants buildgen.go --materialize --remote`
+
+go_remote_libraries(
+  rev='e1abc8b2f3aed139f43ee0f9d1eca95b7da4f312',
+  packages=[
+    'lib/go/thrift',
+  ]
+)

contrib/go/examples/src/go/duck/BUILD

@@ -0,0 +1,8 @@
+# Auto-generated by pants!
+# To re-generate run: `pants buildgen.go --materialize --remote`
+
+go_library(
+  dependencies=[
+    'contrib/go/examples/src/go/duckthrift/gen',
+  ]
+)

contrib/go/examples/src/go/duck/duckreader.go

@@ -0,0 +1,10 @@
+package duck
+
+import (
+	"duckthrift/gen"
+)
+
+func reader(d duck) string {
+	d := duck.NewDuck()
+	return d.GetQuack()
+}

contrib/go/examples/src/go/duckthrift/gen/BUILD

@@ -0,0 +1,3 @@
+go_thrift_library(
+  sources = globs('*.thrift'),
+)

contrib/go/examples/src/go/duckthrift/gen/duck.thrift

@@ -0,0 +1,8 @@
+// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+namespace go duckthrift.gen
+
+struct Duck {
+  1: optional string quack,
+}

contrib/go/examples/src/go/libA/BUILD

@@ -3,7 +3,7 @@
 
 go_library(
   dependencies=[
-    'contrib/go/examples/src/go/libB',
     'contrib/go/examples/src/go/libC',
+    'contrib/go/examples/src/go/libB',
   ]
 )

contrib/go/examples/src/go/libC/BUILD

@@ -3,7 +3,7 @@
 
 go_library(
   dependencies=[
-    'contrib/go/examples/src/go/libD',
     'contrib/go/examples/src/go/libE',
+    'contrib/go/examples/src/go/libD',
   ]
 )

contrib/go/examples/src/go/server/BUILD

@@ -3,9 +3,9 @@
 
 go_binary(
   dependencies=[
-    'contrib/go/examples/3rdparty/go/github.com/gorilla/mux',
+    'contrib/go/examples/3rdparty/go/gopkg.in/yaml.v2',
     'contrib/go/examples/3rdparty/go/golang.org/x/net:http2',
+    'contrib/go/examples/3rdparty/go/github.com/gorilla/mux',
     'contrib/go/examples/3rdparty/go/google.golang.org/grpc',
-    'contrib/go/examples/3rdparty/go/gopkg.in/yaml.v2',
   ]
 )

contrib/go/examples/src/thrift/duckthrift/gen/BUILD

@@ -0,0 +1,3 @@
+go_thrift_library(
+  sources = globs('*.thrift'),
+)

contrib/go/examples/src/thrift/duckthrift/gen/duck.thrift

@@ -0,0 +1,8 @@
+// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+namespace go duckthrift.gen
+
+struct Duck {
+  1: optional string quack,
+}

contrib/go/src/python/pants/contrib/go/tasks/go_buildgen.py

@@ -24,6 +24,7 @@
 from pants.contrib.go.targets.go_library import GoLibrary
 from pants.contrib.go.targets.go_local_source import GoLocalSource
 from pants.contrib.go.targets.go_remote_library import GoRemoteLibrary
+from pants.contrib.go.targets.go_thrift_library import GoThriftLibrary
 from pants.contrib.go.tasks.go_task import GoTask
 
 
@@ -69,7 +70,7 @@ def _generate_missing(self, gopath, local_address, import_listing, visited):
     existing = self._build_graph.get_target(local_address)
     if not existing:
       self._build_graph.inject_synthetic_target(address=local_address, target_type=target_type)
-    elif existing and not isinstance(existing, target_type):
+    elif existing and not isinstance(existing, target_type) and not isinstance(existing, GoThriftLibrary):
       raise self.WrongLocalSourceTargetTypeError('{} should be a {}'
                                                  .format(existing, target_type.__name__))
 
@@ -107,6 +108,15 @@ def _list_deps(self, gopath, local_address):
     src_path = os.path.join(gopath, 'src', import_path)
     safe_mkdir(src_path)
     package_src_root = os.path.join(get_buildroot(), local_address.spec_path)
+    internal = self._build_graph.get_target(local_address)
+
+    if isinstance(internal, GoThriftLibrary):
+
+      package = os.path.basename(import_path)
+      dummy_file = os.path.join(src_path, '{}.go'.format(package))
+      with safe_open(dummy_file, 'w') as fp:
+        fp.write('package {}'.format(package))
+
     for source_file in os.listdir(package_src_root):
       source_path = os.path.join(package_src_root, source_file)
       if GoLocalSource.is_go_source(source_path):

contrib/go/tests/python/pants_test/contrib/go/tasks/BUILD

@@ -33,3 +33,21 @@ python_tests(
   tags={'integration'},
   timeout=180,
 )
+
+
+python_tests(
+  name = 'buildgen',
+  sources = [ 'test_go_buildgen.py'],
+  dependencies=[
+    'contrib/go/src/python/pants/contrib/go/subsystems',
+    'contrib/go/src/python/pants/contrib/go/targets',
+    'contrib/go/src/python/pants/contrib/go/tasks',
+    'contrib/go/src/python/pants/contrib/go:plugin',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/build_graph',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/util:dirutil',
+    'tests/python/pants_test/subsystem:subsystem_utils',
+    'tests/python/pants_test/tasks:task_test_base',
+  ],
+)

contrib/release_packages.sh

@@ -198,6 +198,18 @@ function pkg_avro_install_test() {
     --explain gen | grep "avro-java" &> /dev/null
 }
 
+PKG_THRIFTY=(
+  "pantsbuild.pants.contrib.thrifty"
+  "//contrib/thrifty/src/python/pants/contrib/thrifty:plugin"
+  "pkg_thrifty_install_test"
+)
+function pkg_thrifty_install_test() {
+  local version=$1
+  execute_packaged_pants_with_internal_backends \
+    --plugins="['pantsbuild.pants.contrib.thrifty==${version}']" \
+    --explain gen | grep "thrifty" &> /dev/null
+}
+
 # Once individual (new) package is declared above, insert it into the array below)
 CONTRIB_PACKAGES=(
   PKG_ANDROID
@@ -215,4 +227,5 @@ CONTRIB_PACKAGES=(
   PKG_JAXWS
   PKG_MYPY
   PKG_AVRO
+  PKG_THRIFTY
 )

contrib/thrifty/src/python/pants/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/thrifty/src/python/pants/contrib/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/thrifty/src/python/pants/contrib/thrifty/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(sources=['java_thrifty_gen.py', 'java_thrifty_library.py'])
+
+contrib_plugin(
+  name='plugin',
+  dependencies=[
+    ':thrifty',
+    'src/python/pants/goal:task_registrar',
+  ],
+  distribution_name='pantsbuild.pants.contrib.thrifty',
+  description='Microsoft Thrifty thrift generator pants plugins.',
+  additional_classifiers=[
+    'Topic :: Software Development :: Code Generators'
+  ],
+  register_goals=True,
+  build_file_aliases=True,
+)

contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py

@@ -0,0 +1,89 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.backend.jvm.targets.java_library import JavaLibrary
+from pants.backend.jvm.tasks.nailgun_task import NailgunTaskBase
+from pants.base.build_environment import get_buildroot
+from pants.base.exceptions import TaskError
+from pants.base.workunit import WorkUnitLabel
+from pants.java.jar.jar_dependency import JarDependency
+from pants.task.simple_codegen_task import SimpleCodegenTask
+from twitter.common.collections import OrderedSet
+
+from pants.contrib.thrifty.java_thrifty_library import JavaThriftyLibrary
+
+
+class JavaThriftyGen(NailgunTaskBase, SimpleCodegenTask):
+
+  gentarget_type = JavaThriftyLibrary
+
+  @classmethod
+  def register_options(cls, register):
+    super(JavaThriftyGen, cls).register_options(register)
+
+    def thrifty_jar(name):
+      return JarDependency(org='com.microsoft.thrifty', name=name, rev='0.4.3')
+
+    cls.register_jvm_tool(register,
+                          'thrifty-runtime',
+                          classpath=[thrifty_jar(name='thrifty-runtime')])
+    cls.register_jvm_tool(register,
+                          'thrifty-compiler',
+                          classpath=[thrifty_jar(name='thrifty-compiler')])
+
+  def synthetic_target_type(self, target):
+    return JavaLibrary
+
+  def synthetic_target_extra_dependencies(self, target, target_workdir):
+    deps = OrderedSet(self.resolve_deps([self.get_options().thrifty_runtime]))
+    deps.update(target.dependencies)
+    return deps
+
+  def synthetic_target_extra_exports(self, target, target_workdir):
+    return self.resolve_deps([self.get_options().thrifty_runtime])
+
+  def format_args_for_target(self, target, target_workdir):
+    sources = OrderedSet(target.sources_relative_to_buildroot())
+    args = ['--out={0}'.format(target_workdir)]
+    for include_path in self._compute_include_paths(target):
+      args.append('--path={0}'.format(include_path))
+    args.extend(sources)
+    return args
+
+  def execute_codegen(self, target, target_workdir):
+    args = self.format_args_for_target(target, target_workdir)
+    if args:
+      result = self.runjava(classpath=self.tool_classpath('thrifty-compiler'),
+                            main='com.microsoft.thrifty.compiler.ThriftyCompiler',
+                            args=args,
+                            workunit_name='compile',
+                            workunit_labels=[WorkUnitLabel.TOOL])
+      if result != 0:
+        raise TaskError('Thrifty compiler exited non-zero ({0})'.format(result))
+
+  def _compute_include_paths(self, target):
+    """Computes the set of paths that thrifty uses to lookup imports.
+
+    The IDL files under these paths are not compiled, but they are required to compile
+    downstream IDL files.
+
+    :param target: the JavaThriftyLibrary target to compile.
+    :return: an ordered set of directories to pass along to thrifty.
+    """
+    paths = OrderedSet()
+    paths.add(os.path.join(get_buildroot(), target.target_base))
+
+    def collect_paths(dep):
+      if not dep.has_sources('.thrift'):
+        return
+      paths.add(os.path.join(get_buildroot(), dep.target_base))
+
+    collect_paths(target)
+    target.walk(collect_paths)
+    return paths

contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_library.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.jvm.targets.jvm_target import JvmTarget
+
+
+class JavaThriftyLibrary(JvmTarget):
+  """An Android-optimized Java library generated by the Microsoft Thrifty thrift compiler.
+
+  Says Thrifty, "Thrifty is an implementation of the Apache Thrift software stack for Android,
+  which uses 1/4 of the method count taken by the Apache Thrift compiler."
+
+  For details, see https://github.com/Microsoft/thrifty
+  """
+  default_sources_globs = '*.thrift'