From 3ef802184ba05ef32695df6cababf1aa8edcfac1 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 17 Jul 2019 09:34:18 -0700 Subject: [PATCH 01/35] adding scoverage 1/2 --- BUILD.tools | 7 +++ pants.ini | 7 +++ src/python/pants/backend/jvm/register.py | 3 +- src/python/pants/backend/jvm/subsystems/BUILD | 10 +++++ src/python/pants/backend/jvm/targets/BUILD | 1 + .../backend/jvm/targets/scala_library.py | 43 ++++++++++++++++++- .../pants_test/backend/jvm/subsystems/BUILD | 20 +++++++++ 7 files changed, 89 insertions(+), 2 deletions(-) diff --git a/BUILD.tools b/BUILD.tools index 1e240dd0602..5eb84cebae9 100644 --- a/BUILD.tools +++ b/BUILD.tools @@ -27,6 +27,13 @@ jar_library(name = 'scrooge-linter', jar(org='com.twitter', name='scrooge-linter_2.11', rev=SCROOGE_REV) ]) +jar_library(name = 'scoverage', + jars = [ + scala_jar(org = 'com.twitter.scoverage', name = 'scalac-scoverage-plugin', rev = '1.0.1-twitter' ), + scala_jar(org = 'com.twitter.scoverage', name = 'scalac-scoverage-runtime', rev = '1.0.1-twitter' ), + ]) + + # Google doesn't publish Kythe jars (yet?). So we publish them to a custom repo # (https://github.com/toolchainlabs/binhost) for now. See build-support/ivy/ivysettings.xml # for more info. diff --git a/pants.ini b/pants.ini index f9b3f009053..12766df1807 100644 --- a/pants.ini +++ b/pants.ini @@ -220,6 +220,13 @@ no_warning_args: [ '-S-nowarn', ] +compiler_option_sets_enabled_args: { + # Needed to make scoverage CodeGrid highlighting work + 'scoverage': [ + '-S-Yrangepos', + ] + } + [compile.javac] args: [ '-encoding', 'UTF-8', diff --git a/src/python/pants/backend/jvm/register.py b/src/python/pants/backend/jvm/register.py index a0ab8ea91ea..8178c0558cc 100644 --- a/src/python/pants/backend/jvm/register.py +++ b/src/python/pants/backend/jvm/register.py @@ -8,6 +8,7 @@ from pants.backend.jvm.scala_artifact import ScalaArtifact from pants.backend.jvm.subsystems.jar_dependency_management import JarDependencyManagementSetup from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform from pants.backend.jvm.subsystems.shader import Shading from pants.backend.jvm.targets.annotation_processor import AnnotationProcessor from pants.backend.jvm.targets.benchmark import Benchmark @@ -138,7 +139,7 @@ def build_file_aliases(): def global_subsystems(): - return (ScalaPlatform,) + return (ScalaPlatform, ScalaCoveragePlatform, ) # TODO https://github.com/pantsbuild/pants/issues/604 register_goals diff --git a/src/python/pants/backend/jvm/subsystems/BUILD b/src/python/pants/backend/jvm/subsystems/BUILD index 4787b017a96..10252a62b63 100644 --- a/src/python/pants/backend/jvm/subsystems/BUILD +++ b/src/python/pants/backend/jvm/subsystems/BUILD @@ -111,6 +111,16 @@ python_library( ], ) +python_library( + name = 'scala_coverage_platform', + sources = ['scala_coverage_platform.py'], + dependencies = [ + 'src/python/pants/option', + 'src/python/pants/subsystem', + 'src/python/pants/backend/jvm/targets:jvm', + ], +) + python_library( name = 'shader', sources = ['shader.py'], diff --git a/src/python/pants/backend/jvm/targets/BUILD b/src/python/pants/backend/jvm/targets/BUILD index c8a1bfdbec7..f6b26fa5f86 100644 --- a/src/python/pants/backend/jvm/targets/BUILD +++ b/src/python/pants/backend/jvm/targets/BUILD @@ -79,6 +79,7 @@ python_library( '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', 'src/python/pants/base:exceptions', 'src/python/pants/base:validation', 'src/python/pants/build_graph', diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index 32ffb3507bf..e73aac68159 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform from pants.backend.jvm.targets.exportable_jvm_library import ExportableJvmLibrary from pants.backend.jvm.targets.junit_tests import JUnitTests from pants.base.exceptions import TargetDefinitionException @@ -27,7 +28,7 @@ class ScalaLibrary(ExportableJvmLibrary): @classmethod def subsystems(cls): - return super().subsystems() + (ScalaPlatform, ) + return super().subsystems() + (ScalaPlatform, ScalaCoveragePlatform, ) def __init__(self, java_sources=None, payload=None, **kwargs): """ @@ -47,6 +48,8 @@ def __init__(self, java_sources=None, payload=None, **kwargs): }) super().__init__(payload=payload, **kwargs) + self._scoverage = ScalaCoveragePlatform.global_instance().get_options().enable_scoverage + @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): for spec in super().compute_injectable_specs(kwargs, payload): @@ -65,6 +68,10 @@ def compute_dependency_specs(cls, kwargs=None, payload=None): for spec in ScalaPlatform.global_instance().injectables_specs_for_key('scala-library'): yield spec + if ScalaCoveragePlatform.global_instance().get_options().enable_scoverage: + for spec in ScalaCoveragePlatform.global_instance().injectables_specs_for_key('scoverage'): + yield spec + def get_jar_dependencies(self): for jar in super().get_jar_dependencies(): yield jar @@ -82,3 +89,37 @@ def java_sources(self): raise TargetDefinitionException(self, 'No such java target: {}'.format(spec)) targets.append(target) return targets + + @property + def scalac_plugins(self): + """The names of compiler plugins to use when compiling this target with scalac. + :return: See constructor. + :rtype: list of strings. + """ + if self._scoverage: + return ScalaCoveragePlatform.global_instance().get_scalac_plugins(self) + + return self.payload.scalac_plugins + + @property + def scalac_plugin_args(self): + """Map from scalac plugin name to list of args for that plugin. + :return: See constructor. + :rtype: map from string to list of strings. + """ + if self._scoverage: + return ScalaCoveragePlatform.global_instance().get_scalac_plugin_args(self) + + return self.payload.scalac_plugin_args + + @property + def compiler_option_sets(self): + """For every element in this list, enable the corresponding flags on compilation + of targets. + :return: See constructor. + :rtype: list + """ + if self._scoverage: + return ScalaCoveragePlatform.global_instance().get_compiler_option_sets(self) + + return self.payload.compiler_option_sets diff --git a/tests/python/pants_test/backend/jvm/subsystems/BUILD b/tests/python/pants_test/backend/jvm/subsystems/BUILD index 2775ef46d05..e680573f5c3 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/BUILD +++ b/tests/python/pants_test/backend/jvm/subsystems/BUILD @@ -5,6 +5,7 @@ python_tests( name='shader', sources=['test_shader.py'], dependencies=[ + '3rdparty/python:future', 'src/python/pants/backend/jvm/subsystems:shader', 'src/python/pants/java/distribution', 'src/python/pants/java:executor', @@ -19,6 +20,24 @@ python_tests( sources=['test_custom_scala.py'], dependencies=[ 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', + 'src/python/pants/java/jar', + 'src/python/pants/backend/jvm/targets:jvm', + 'src/python/pants/backend/jvm/targets:scala', + 'src/python/pants/backend/jvm/tasks:scalastyle', + 'src/python/pants/backend/jvm/tasks/jvm_compile/zinc', + 'tests/python/pants_test/jvm:nailgun_task_test_base', + 'tests/python/pants_test/subsystem:subsystem_utils', + ] +) + + +python_tests( + name='test_scala_coverage_platform', + sources=['test_scala_coverage_platform.py'], + dependencies=[ + 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', @@ -45,6 +64,7 @@ python_tests( name='shader_integration', sources=['test_shader_integration.py'], dependencies=[ + '3rdparty/python:future', 'src/python/pants/fs', 'src/python/pants/java/distribution', 'src/python/pants/util:contextutil', From 705a12b3dfdcc78234db92df4f6f09ee7d66c458 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 17 Jul 2019 09:34:18 -0700 Subject: [PATCH 02/35] adding scoverage 1/2 From 5ea8fc57f60fd893981304c7b636890de8829674 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 17 Jul 2019 09:38:57 -0700 Subject: [PATCH 03/35] adding scoverage 1/2 --- 3rdparty/jvm/org/scoverage/BUILD | 3 + new_blacklist_scoverage | 27 +++ .../jvm/subsystems/scala_coverage_platform.py | 94 ++++++++++ .../test_scala_coverage_platform.py | 161 ++++++++++++++++++ 4 files changed, 285 insertions(+) create mode 100644 3rdparty/jvm/org/scoverage/BUILD create mode 100644 new_blacklist_scoverage create mode 100644 src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py create mode 100644 tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py diff --git a/3rdparty/jvm/org/scoverage/BUILD b/3rdparty/jvm/org/scoverage/BUILD new file mode 100644 index 00000000000..eff94249694 --- /dev/null +++ b/3rdparty/jvm/org/scoverage/BUILD @@ -0,0 +1,3 @@ +target( + dependencies = ["//:scoverage"], +) diff --git a/new_blacklist_scoverage b/new_blacklist_scoverage new file mode 100644 index 00000000000..251aefee5c2 --- /dev/null +++ b/new_blacklist_scoverage @@ -0,0 +1,27 @@ +src/scala/com/twitter/botmaker/knowndevices:knowndevices +bizinsights/src/main/scala/com/twitter/bizinsights/jobs/events_entrypoints:events-entrypoints +src/scala/com/twitter/tormenta_internal/util:util +src/java/com/twitter/ml/vw/main:main +src/java/com/twitter/ml/api:api-base +src/java/com/twitter/ads/internal/feature_switch:feature_switch +twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine:prediction_engine +src/scala/com/twitter/pluck/source:remote-sources +src/scala/com/twitter/pluck/source/cassowary:cassowary +3rdparty/src/jvm/com/twitter/storehaus:core +src/scala/com/twitter/ml/api/decision_forest:decision_forest +birdherd/src/main/scala:scala +src/scala/com/twitter/ads/dataservice/scalding:scalding +src/java/com/twitter/search/common/file:file +birdherd/integration/options/src/main/scala:scala +src/scala/com/twitter/frigate/data_pipeline/features_aggregated/extractor:extractor +src/scala/com/twitter/frigate/data_pipeline/scalding:push-ddg-metrics-job +ads-data-infrastructure/ec/ecnext/src/main/scala/com/twitter/ads/ecnext/migration/readers:readers +geoduck/data/ipindex/src/main/scala/com/twitter/geoduck/data/ipindex/api/common:common +tsar/core/src/main/scala:scala +ads-data-infrastructure/ec/ecnext/src/main/scala/com/twitter/ads/ecnext/migration/readers:readers +src/java/com/twitter/ads/common/config:configmap +emoji/scala/src/main/scala/com/twitter/emoji:emoji +finatra-internal/authorization/src/test/scala:test-deps +kafka/provisioning/client/src/main/scala:scala +appsec/sanitization-lib/src/main/scala:scala +a/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blacked \ No newline at end of file diff --git a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py new file mode 100644 index 00000000000..e5d38064f86 --- /dev/null +++ b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.build_graph.injectables_mixin import InjectablesMixin +from pants.subsystem.subsystem import Subsystem + + +SCOVERAGE = "scoverage" +blacklist_file = 'new_blacklist_scoverage' + + +class ScalaCoveragePlatform(InjectablesMixin, Subsystem): + """The scala coverage platform.""" + + options_scope = 'scala-coverage' + + @classmethod + def register_options(cls, register): + super(ScalaCoveragePlatform, cls).register_options(register) + register('--enable-scoverage', + default=False, + type=bool, + help='Specifies whether to generate scoverage reports for scala test targets.' + 'Default value is False') + + register('--scoverage-target-path', + default='//:scoverage', + type=str, + help='Path to the scoverage dependency.') + + @property + def injectables_spec_mapping(self): + return { + 'scoverage': ['{}'.format(self.get_options().scoverage_target_path)], + } + + def get_scalac_plugins(self, target): + """ + Adds 'scoverage' to scalac_plugins in case scoverage is enabled for that [target]. + :return: modified scalac_plugins + :rtype: list of strings + """ + + # Prevent instrumenting generated targets and targets in blacklist. + if target.identifier.startswith(".pants.d.gen") or self.is_blacklisted(target): + return target.payload.scalac_plugins + + scalac_plugins = target.payload.scalac_plugins + if scalac_plugins: + scalac_plugins.append(SCOVERAGE) + else: + scalac_plugins = [SCOVERAGE] + return scalac_plugins + + def get_scalac_plugin_args(self, target): + """ + Adds 'scoverage' to scalac_plugins_args in case scoverage is enabled for that [target]. + :return: modified scalac_plugins_args + :rtype: map from string to list of strings. + """ + scalac_plugin_args = target.payload.scalac_plugin_args + if scalac_plugin_args: + scalac_plugin_args.update( + {"scoverage": ["writeToClasspath:true", "dataDir:{}".format(target.identifier)]}) + else: + scalac_plugin_args = { + "scoverage": ["writeToClasspath:true", "dataDir:{}".format(target.identifier)] + } + return scalac_plugin_args + + def get_compiler_option_sets(self, target): + """ + Adds 'scoverage' to compiler_options_sets in case scoverage is enabled for that [target]. + :return: modified compiler_option_sets + :rtype: see constructor + """ + compiler_option_sets = target.payload.compiler_option_sets + if compiler_option_sets: + list(compiler_option_sets).append(SCOVERAGE) + else: + compiler_option_sets = [SCOVERAGE] + return tuple(compiler_option_sets) + + def is_blacklisted(self, target): + """ + Checks if the [target] is blacklisted or not. + """ + if target.address.spec in open(blacklist_file).read(): + return True + else: + return False diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py new file mode 100644 index 00000000000..86e7acdf493 --- /dev/null +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py @@ -0,0 +1,161 @@ +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants_test.test_base import TestBase +from pants_test.subsystem.subsystem_util import init_subsystem +from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform +from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.targets.scala_library import ScalaLibrary +from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.java.jar.jar_dependency import JarDependency + + +from textwrap import dedent + + +class ScalaCoveragePlatformTest(TestBase): + scoverage_path = '//:scoverage' + + def setup_scala_coverage_platform(self): + options = { + ScalaPlatform.options_scope: { + 'version': 'custom', + 'suffix_version': '2.10', + } + } + + options2 = { + ScalaCoveragePlatform.options_scope: { + 'enable_scoverage' : 'False' + } + } + + init_subsystem(ScalaPlatform, options) + init_subsystem(ScalaCoveragePlatform, options2) + + self.make_target('//:scalastyle', + JarLibrary, + jars=[JarDependency('org.scalastyle', 'scalastyle_2.10', '0.3.2')] + ) + + self.make_target('//:scala-repl', + JarLibrary, + jars=[ + JarDependency(org = 'org.scala-lang', + name = 'jline', + rev = '2.10.5'), + JarDependency(org = 'org.scala-lang', + name = 'scala-compiler', + rev = '2.10.5')]) + + self.make_target('//:scalac', + JarLibrary, + jars=[JarDependency('org.scala-lang', 'scala-compiler', '2.10.5')]) + + self.make_target('//:scala-library', + JarLibrary, + jars=[JarDependency('org.scala-lang', 'scala-library', '2.10.5')]) + + self.make_target('//:scoverage', + JarLibrary, + jars=[JarDependency('com.twitter.scoverage', 'scalac-scoverage-plugin', '1.0.1-twitter'), + JarDependency('com.twitter.scoverage', 'scalac-scoverage-runtime', '1.0.1-twitter')]) + + + # ==========> TESTS <============= + # ================================ + def test_subsystem_defaults(self): + init_subsystem(ScalaCoveragePlatform) + + subsystem = ScalaCoveragePlatform.global_instance() + + self.assertEqual(False, subsystem.get_options().enable_scoverage) + self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) + + def test_subsystem_option_sets(self): + init_subsystem(ScalaCoveragePlatform) + ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + + subsystem = ScalaCoveragePlatform.global_instance() + + self.assertEqual(True, subsystem.get_options().enable_scoverage) + self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) + + + def test_library_scoverage_enabled(self): + self.setup_scala_coverage_platform() + ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + + self.create_file( + relpath='a/scala/pass.scala', + contents=dedent(""" + import java.util + object HelloWorld { + def main(args: Array[String]) { + println("Hello, world!") + } + } + """)) + + scala_target = self.make_target('a/scala:pass', ScalaLibrary, sources=['pass.scala']) + + self.assertIn('scoverage', scala_target.scalac_plugins) + self.assertIn('scoverage', scala_target.scalac_plugin_args) + self.assertIn('//:scoverage', list(map(lambda t: t.address.spec, scala_target.dependencies))) + self.assertIn('scoverage', list(scala_target.compiler_option_sets)) + + + def test_library_scoverage_disabled(self): + self.setup_scala_coverage_platform() + ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = False + + self.create_file( + relpath='a/scala/pass.scala', + contents=dedent(""" + import java.util + object HelloWorld { + def main(args: Array[String]) { + println("Hello, world!") + } + } + """)) + + scala_target = self.make_target('a/scala:pass', ScalaLibrary, sources=['pass.scala']) + + self.assertNotIn('scoverage', scala_target.scalac_plugins) + if scala_target.scalac_plugin_args: + self.assertNotIn('scoverage', scala_target.scalac_plugin_args) + self.assertNotIn('//:scoverage', list(map(lambda t: t.address.spec ,scala_target.dependencies))) + if scala_target.compiler_option_sets: + self.assertNotIn('scoverage', list(scala_target.compiler_option_sets)) + + + def test_blacklist(self): + """ + When a target is blacklisted, we do not instrument it. For achieving that, we only + want `scalac_plugins` to not contain `scoverage`. Thus, the target may still have + `scoverage` in `scalac_plugin_args` and in `dependencies` but it will not be + instrumented as long as `scalac_plugins` do not contain `scoverage`. + :return: + """ + self.setup_scala_coverage_platform() + ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + + blacklist_file = open("new_blacklist_scoverage", "a+") + blacklist_file.write("a/scala:blacked") + + self.create_file( + relpath='a/scala/pass.scala', + contents=dedent(""" + import java.util + object HelloWorld { + def main(args: Array[String]) { + println("Hello, world!") + } + } + """)) + + scala_target = self.make_target('a/scala:blacked', ScalaLibrary, sources=['pass.scala']) + + self.assertNotIn('scoverage', scala_target.scalac_plugins) + From fc48b64e8fb7924c0337208ecf0c920ea2923aac Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 17 Jul 2019 23:15:25 -0700 Subject: [PATCH 04/35] removing scoverage target from BUILD.tools --- BUILD.tools | 6 -- new_blacklist_scoverage | 27 --------- src/python/pants/backend/jvm/subsystems/BUILD | 1 + .../jvm/subsystems/scala_coverage_platform.py | 58 ++++++++++++++----- .../test_scala_coverage_platform.py | 16 +++-- 5 files changed, 55 insertions(+), 53 deletions(-) delete mode 100644 new_blacklist_scoverage diff --git a/BUILD.tools b/BUILD.tools index 5eb84cebae9..1e0acbce5de 100644 --- a/BUILD.tools +++ b/BUILD.tools @@ -27,12 +27,6 @@ jar_library(name = 'scrooge-linter', jar(org='com.twitter', name='scrooge-linter_2.11', rev=SCROOGE_REV) ]) -jar_library(name = 'scoverage', - jars = [ - scala_jar(org = 'com.twitter.scoverage', name = 'scalac-scoverage-plugin', rev = '1.0.1-twitter' ), - scala_jar(org = 'com.twitter.scoverage', name = 'scalac-scoverage-runtime', rev = '1.0.1-twitter' ), - ]) - # Google doesn't publish Kythe jars (yet?). So we publish them to a custom repo # (https://github.com/toolchainlabs/binhost) for now. See build-support/ivy/ivysettings.xml diff --git a/new_blacklist_scoverage b/new_blacklist_scoverage deleted file mode 100644 index 251aefee5c2..00000000000 --- a/new_blacklist_scoverage +++ /dev/null @@ -1,27 +0,0 @@ -src/scala/com/twitter/botmaker/knowndevices:knowndevices -bizinsights/src/main/scala/com/twitter/bizinsights/jobs/events_entrypoints:events-entrypoints -src/scala/com/twitter/tormenta_internal/util:util -src/java/com/twitter/ml/vw/main:main -src/java/com/twitter/ml/api:api-base -src/java/com/twitter/ads/internal/feature_switch:feature_switch -twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine:prediction_engine -src/scala/com/twitter/pluck/source:remote-sources -src/scala/com/twitter/pluck/source/cassowary:cassowary -3rdparty/src/jvm/com/twitter/storehaus:core -src/scala/com/twitter/ml/api/decision_forest:decision_forest -birdherd/src/main/scala:scala -src/scala/com/twitter/ads/dataservice/scalding:scalding -src/java/com/twitter/search/common/file:file -birdherd/integration/options/src/main/scala:scala -src/scala/com/twitter/frigate/data_pipeline/features_aggregated/extractor:extractor -src/scala/com/twitter/frigate/data_pipeline/scalding:push-ddg-metrics-job -ads-data-infrastructure/ec/ecnext/src/main/scala/com/twitter/ads/ecnext/migration/readers:readers -geoduck/data/ipindex/src/main/scala/com/twitter/geoduck/data/ipindex/api/common:common -tsar/core/src/main/scala:scala -ads-data-infrastructure/ec/ecnext/src/main/scala/com/twitter/ads/ecnext/migration/readers:readers -src/java/com/twitter/ads/common/config:configmap -emoji/scala/src/main/scala/com/twitter/emoji:emoji -finatra-internal/authorization/src/test/scala:test-deps -kafka/provisioning/client/src/main/scala:scala -appsec/sanitization-lib/src/main/scala:scala -a/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blackeda/scala:blacked \ No newline at end of file diff --git a/src/python/pants/backend/jvm/subsystems/BUILD b/src/python/pants/backend/jvm/subsystems/BUILD index 10252a62b63..07316ec9d56 100644 --- a/src/python/pants/backend/jvm/subsystems/BUILD +++ b/src/python/pants/backend/jvm/subsystems/BUILD @@ -117,6 +117,7 @@ python_library( dependencies = [ 'src/python/pants/option', 'src/python/pants/subsystem', + 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', ], ) diff --git a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py index e5d38064f86..a7734e55c86 100644 --- a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py @@ -1,12 +1,13 @@ -# coding=utf-8 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import, division, print_function, unicode_literals - +import os from pants.build_graph.injectables_mixin import InjectablesMixin from pants.subsystem.subsystem import Subsystem - +from pants.java.jar.jar_dependency import JarDependency +from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.build_graph.address import Address +from typing import Any, Dict, List, Optional, Union SCOVERAGE = "scoverage" blacklist_file = 'new_blacklist_scoverage' @@ -26,22 +27,50 @@ def register_options(cls, register): help='Specifies whether to generate scoverage reports for scala test targets.' 'Default value is False') + register('--blacklist-file', + default=blacklist_file, + type=str, + help='Path to files containing targets not to be instrumented.') + register('--scoverage-target-path', default='//:scoverage', type=str, help='Path to the scoverage dependency.') + def scoverage_jar(self): + return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), + JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', rev='1.0.1-twitter')] + + + def injectables(self,build_graph): + specs_to_create = [ + ('scoverage',self.scoverage_jar), + ] + + for spec_key, create_jardep_func in specs_to_create: + spec = self.injectables_spec_for_key(spec_key) + target_address = Address.parse(spec) + if not build_graph.contains_address(target_address): + target_jars = create_jardep_func() + jars = target_jars if isinstance(target_jars, list) else [target_jars] + build_graph.inject_synthetic_target(target_address, + JarLibrary, + jars=jars, + scope='forced') + elif not build_graph.get_target(target_address).is_synthetic: + raise build_graph.ManualSyntheticTargetError(target_address) + + @property def injectables_spec_mapping(self): return { - 'scoverage': ['{}'.format(self.get_options().scoverage_target_path)], + 'scoverage': [f"{self.get_options().scoverage_target_path}"], } - def get_scalac_plugins(self, target): + def get_scalac_plugins(self, target) -> List[str]: """ Adds 'scoverage' to scalac_plugins in case scoverage is enabled for that [target]. :return: modified scalac_plugins - :rtype: list of strings """ # Prevent instrumenting generated targets and targets in blacklist. @@ -55,19 +84,18 @@ def get_scalac_plugins(self, target): scalac_plugins = [SCOVERAGE] return scalac_plugins - def get_scalac_plugin_args(self, target): + def get_scalac_plugin_args(self, target) -> Dict[str, List[str]]: """ Adds 'scoverage' to scalac_plugins_args in case scoverage is enabled for that [target]. :return: modified scalac_plugins_args - :rtype: map from string to list of strings. """ scalac_plugin_args = target.payload.scalac_plugin_args if scalac_plugin_args: scalac_plugin_args.update( - {"scoverage": ["writeToClasspath:true", "dataDir:{}".format(target.identifier)]}) + {"scoverage": ["writeToClasspath:true", f"dataDir:{target.identifier}"]}) else: scalac_plugin_args = { - "scoverage": ["writeToClasspath:true", "dataDir:{}".format(target.identifier)] + "scoverage": ["writeToClasspath:true", f"dataDir:{target.identifier}"] } return scalac_plugin_args @@ -75,7 +103,6 @@ def get_compiler_option_sets(self, target): """ Adds 'scoverage' to compiler_options_sets in case scoverage is enabled for that [target]. :return: modified compiler_option_sets - :rtype: see constructor """ compiler_option_sets = target.payload.compiler_option_sets if compiler_option_sets: @@ -84,11 +111,14 @@ def get_compiler_option_sets(self, target): compiler_option_sets = [SCOVERAGE] return tuple(compiler_option_sets) - def is_blacklisted(self, target): + def is_blacklisted(self, target) -> bool: """ Checks if the [target] is blacklisted or not. """ - if target.address.spec in open(blacklist_file).read(): + if not os.path.exists(self.get_options().blacklist_file): + return False + + if target.address.spec in open(self.get_options().blacklist_file).read(): return True else: return False diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py index 86e7acdf493..02f5247e82b 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os from pants_test.test_base import TestBase from pants_test.subsystem.subsystem_util import init_subsystem from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform @@ -15,6 +16,7 @@ class ScalaCoveragePlatformTest(TestBase): scoverage_path = '//:scoverage' + blacklist_file_path = 'my/file/new_blacklist_scoverage_test' def setup_scala_coverage_platform(self): options = { @@ -56,10 +58,6 @@ def setup_scala_coverage_platform(self): JarLibrary, jars=[JarDependency('org.scala-lang', 'scala-library', '2.10.5')]) - self.make_target('//:scoverage', - JarLibrary, - jars=[JarDependency('com.twitter.scoverage', 'scalac-scoverage-plugin', '1.0.1-twitter'), - JarDependency('com.twitter.scoverage', 'scalac-scoverage-runtime', '1.0.1-twitter')]) # ==========> TESTS <============= @@ -141,8 +139,13 @@ def test_blacklist(self): self.setup_scala_coverage_platform() ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True - blacklist_file = open("new_blacklist_scoverage", "a+") - blacklist_file.write("a/scala:blacked") + tmp = self.create_file( + relpath=self.blacklist_file_path, + contents=dedent(""" + a/scala:blacked + """) + ) + ScalaCoveragePlatform.global_instance().get_options().blacklist_file = tmp self.create_file( relpath='a/scala/pass.scala', @@ -159,3 +162,4 @@ def main(args: Array[String]) { self.assertNotIn('scoverage', scala_target.scalac_plugins) + From 21a5dd4e7ffc3abce6484513d39e9a8081a600e7 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 17 Jul 2019 23:44:07 -0700 Subject: [PATCH 05/35] Update src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py Co-Authored-By: Eric Arellano --- .../pants/backend/jvm/subsystems/scala_coverage_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py index a7734e55c86..c84710ffc52 100644 --- a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py @@ -20,7 +20,7 @@ class ScalaCoveragePlatform(InjectablesMixin, Subsystem): @classmethod def register_options(cls, register): - super(ScalaCoveragePlatform, cls).register_options(register) + super().register_options(register) register('--enable-scoverage', default=False, type=bool, From 24293574b710b8e7ffc1cf8c15027c319fb1b3c3 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 18 Jul 2019 09:39:48 -0700 Subject: [PATCH 06/35] removing futures --- tests/python/pants_test/backend/jvm/subsystems/BUILD | 2 -- .../backend/jvm/subsystems/test_scala_coverage_platform.py | 4 ---- 2 files changed, 6 deletions(-) diff --git a/tests/python/pants_test/backend/jvm/subsystems/BUILD b/tests/python/pants_test/backend/jvm/subsystems/BUILD index e680573f5c3..fbafb4abe75 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/BUILD +++ b/tests/python/pants_test/backend/jvm/subsystems/BUILD @@ -5,7 +5,6 @@ python_tests( name='shader', sources=['test_shader.py'], dependencies=[ - '3rdparty/python:future', 'src/python/pants/backend/jvm/subsystems:shader', 'src/python/pants/java/distribution', 'src/python/pants/java:executor', @@ -64,7 +63,6 @@ python_tests( name='shader_integration', sources=['test_shader_integration.py'], dependencies=[ - '3rdparty/python:future', 'src/python/pants/fs', 'src/python/pants/java/distribution', 'src/python/pants/util:contextutil', diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py index 02f5247e82b..8dddd9e6558 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py @@ -1,7 +1,3 @@ -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import os from pants_test.test_base import TestBase from pants_test.subsystem.subsystem_util import init_subsystem from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform From f432d7bc56f1ad36677f8daed5a3a0f4b4b0b6a8 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 19 Jul 2019 00:23:47 -0700 Subject: [PATCH 07/35] changing name to scoverage_platform --- 3rdparty/jvm/org/scoverage/BUILD | 3 -- pants.ini | 7 ---- src/python/pants/backend/jvm/register.py | 4 +- src/python/pants/backend/jvm/subsystems/BUILD | 4 +- ...rage_platform.py => scoverage_platform.py} | 17 +++++---- src/python/pants/backend/jvm/targets/BUILD | 2 +- .../backend/jvm/targets/scala_library.py | 16 ++++---- .../tasks/jvm_compile/zinc/zinc_compile.py | 5 +++ .../pants_test/backend/jvm/subsystems/BUILD | 8 ++-- ...platform.py => test_scoverage_platform.py} | 37 ++++++++++--------- .../python/pants_test/backend/jvm/tasks/BUILD | 2 + .../backend/jvm/tasks/test_scalafmt.py | 2 + .../backend/jvm/tasks/test_scalastyle.py | 2 + 13 files changed, 57 insertions(+), 52 deletions(-) delete mode 100644 3rdparty/jvm/org/scoverage/BUILD rename src/python/pants/backend/jvm/subsystems/{scala_coverage_platform.py => scoverage_platform.py} (93%) rename tests/python/pants_test/backend/jvm/subsystems/{test_scala_coverage_platform.py => test_scoverage_platform.py} (80%) diff --git a/3rdparty/jvm/org/scoverage/BUILD b/3rdparty/jvm/org/scoverage/BUILD deleted file mode 100644 index eff94249694..00000000000 --- a/3rdparty/jvm/org/scoverage/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -target( - dependencies = ["//:scoverage"], -) diff --git a/pants.ini b/pants.ini index 12766df1807..f9b3f009053 100644 --- a/pants.ini +++ b/pants.ini @@ -220,13 +220,6 @@ no_warning_args: [ '-S-nowarn', ] -compiler_option_sets_enabled_args: { - # Needed to make scoverage CodeGrid highlighting work - 'scoverage': [ - '-S-Yrangepos', - ] - } - [compile.javac] args: [ '-encoding', 'UTF-8', diff --git a/src/python/pants/backend/jvm/register.py b/src/python/pants/backend/jvm/register.py index 8178c0558cc..ff0d81b8aaf 100644 --- a/src/python/pants/backend/jvm/register.py +++ b/src/python/pants/backend/jvm/register.py @@ -8,7 +8,7 @@ from pants.backend.jvm.scala_artifact import ScalaArtifact from pants.backend.jvm.subsystems.jar_dependency_management import JarDependencyManagementSetup from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform -from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.subsystems.shader import Shading from pants.backend.jvm.targets.annotation_processor import AnnotationProcessor from pants.backend.jvm.targets.benchmark import Benchmark @@ -139,7 +139,7 @@ def build_file_aliases(): def global_subsystems(): - return (ScalaPlatform, ScalaCoveragePlatform, ) + return (ScalaPlatform, ScoveragePlatform, ) # TODO https://github.com/pantsbuild/pants/issues/604 register_goals diff --git a/src/python/pants/backend/jvm/subsystems/BUILD b/src/python/pants/backend/jvm/subsystems/BUILD index 07316ec9d56..02439a53435 100644 --- a/src/python/pants/backend/jvm/subsystems/BUILD +++ b/src/python/pants/backend/jvm/subsystems/BUILD @@ -112,8 +112,8 @@ python_library( ) python_library( - name = 'scala_coverage_platform', - sources = ['scala_coverage_platform.py'], + name = 'scoverage_platform', + sources = ['scoverage_platform.py'], dependencies = [ 'src/python/pants/option', 'src/python/pants/subsystem', diff --git a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py similarity index 93% rename from src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py rename to src/python/pants/backend/jvm/subsystems/scoverage_platform.py index c84710ffc52..7c4caebaf3a 100644 --- a/src/python/pants/backend/jvm/subsystems/scala_coverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -9,14 +9,15 @@ from pants.build_graph.address import Address from typing import Any, Dict, List, Optional, Union + SCOVERAGE = "scoverage" blacklist_file = 'new_blacklist_scoverage' -class ScalaCoveragePlatform(InjectablesMixin, Subsystem): +class ScoveragePlatform(InjectablesMixin, Subsystem): """The scala coverage platform.""" - options_scope = 'scala-coverage' + options_scope = 'scoverage' @classmethod def register_options(cls, register): @@ -38,13 +39,14 @@ def register_options(cls, register): help='Path to the scoverage dependency.') def scoverage_jar(self): - return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), - JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', rev='1.0.1-twitter')] - + return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', + rev='1.0.1-twitter'), + JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', + rev='1.0.1-twitter')] - def injectables(self,build_graph): + def injectables(self, build_graph): specs_to_create = [ - ('scoverage',self.scoverage_jar), + ('scoverage', self.scoverage_jar), ] for spec_key, create_jardep_func in specs_to_create: @@ -60,7 +62,6 @@ def injectables(self,build_graph): elif not build_graph.get_target(target_address).is_synthetic: raise build_graph.ManualSyntheticTargetError(target_address) - @property def injectables_spec_mapping(self): return { diff --git a/src/python/pants/backend/jvm/targets/BUILD b/src/python/pants/backend/jvm/targets/BUILD index f6b26fa5f86..827c88f99a6 100644 --- a/src/python/pants/backend/jvm/targets/BUILD +++ b/src/python/pants/backend/jvm/targets/BUILD @@ -79,7 +79,7 @@ python_library( '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/subsystems:scala_platform', - 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/base:exceptions', 'src/python/pants/base:validation', 'src/python/pants/build_graph', diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index e73aac68159..dc6d751a3ee 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -2,7 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform -from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.exportable_jvm_library import ExportableJvmLibrary from pants.backend.jvm.targets.junit_tests import JUnitTests from pants.base.exceptions import TargetDefinitionException @@ -28,7 +28,7 @@ class ScalaLibrary(ExportableJvmLibrary): @classmethod def subsystems(cls): - return super().subsystems() + (ScalaPlatform, ScalaCoveragePlatform, ) + return super().subsystems() + (ScalaPlatform, ScoveragePlatform,) def __init__(self, java_sources=None, payload=None, **kwargs): """ @@ -48,7 +48,7 @@ def __init__(self, java_sources=None, payload=None, **kwargs): }) super().__init__(payload=payload, **kwargs) - self._scoverage = ScalaCoveragePlatform.global_instance().get_options().enable_scoverage + self._scoverage = ScoveragePlatform.global_instance().get_options().enable_scoverage @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): @@ -68,8 +68,8 @@ def compute_dependency_specs(cls, kwargs=None, payload=None): for spec in ScalaPlatform.global_instance().injectables_specs_for_key('scala-library'): yield spec - if ScalaCoveragePlatform.global_instance().get_options().enable_scoverage: - for spec in ScalaCoveragePlatform.global_instance().injectables_specs_for_key('scoverage'): + if ScoveragePlatform.global_instance().get_options().enable_scoverage: + for spec in ScoveragePlatform.global_instance().injectables_specs_for_key('scoverage'): yield spec def get_jar_dependencies(self): @@ -97,7 +97,7 @@ def scalac_plugins(self): :rtype: list of strings. """ if self._scoverage: - return ScalaCoveragePlatform.global_instance().get_scalac_plugins(self) + return ScoveragePlatform.global_instance().get_scalac_plugins(self) return self.payload.scalac_plugins @@ -108,7 +108,7 @@ def scalac_plugin_args(self): :rtype: map from string to list of strings. """ if self._scoverage: - return ScalaCoveragePlatform.global_instance().get_scalac_plugin_args(self) + return ScoveragePlatform.global_instance().get_scalac_plugin_args(self) return self.payload.scalac_plugin_args @@ -120,6 +120,6 @@ def compiler_option_sets(self): :rtype: list """ if self._scoverage: - return ScalaCoveragePlatform.global_instance().get_compiler_option_sets(self) + return ScoveragePlatform.global_instance().get_compiler_option_sets(self) return self.payload.compiler_option_sets diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py index d3faad52ff5..346ba5bb75f 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py @@ -322,6 +322,11 @@ def relative_to_exec_root(path): zinc_args.append('-transactional') compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets(compiler_option_sets) + + # Needed to make scoverage CodeGrid highlighting work + if 'scoverage' in scalac_plugin_map.keys(): + compiler_option_sets_args += ['-S-Yrangepos'] + zinc_args.extend(compiler_option_sets_args) if not self._clear_invalid_analysis: diff --git a/tests/python/pants_test/backend/jvm/subsystems/BUILD b/tests/python/pants_test/backend/jvm/subsystems/BUILD index fbafb4abe75..8591505c3f6 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/BUILD +++ b/tests/python/pants_test/backend/jvm/subsystems/BUILD @@ -19,7 +19,7 @@ python_tests( sources=['test_custom_scala.py'], dependencies=[ 'src/python/pants/backend/jvm/subsystems:scala_platform', - 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', @@ -32,11 +32,11 @@ python_tests( python_tests( - name='test_scala_coverage_platform', - sources=['test_scala_coverage_platform.py'], + name='test_scoverage_platform', + sources=['test_scoverage_platform.py'], dependencies=[ 'src/python/pants/backend/jvm/subsystems:scala_platform', - 'src/python/pants/backend/jvm/subsystems:scala_coverage_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py similarity index 80% rename from tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py rename to tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 8dddd9e6558..ccc678d2ed5 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scala_coverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -1,6 +1,9 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + from pants_test.test_base import TestBase from pants_test.subsystem.subsystem_util import init_subsystem -from pants.backend.jvm.subsystems.scala_coverage_platform import ScalaCoveragePlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform from pants.backend.jvm.targets.scala_library import ScalaLibrary from pants.backend.jvm.targets.jar_library import JarLibrary @@ -10,11 +13,11 @@ from textwrap import dedent -class ScalaCoveragePlatformTest(TestBase): +class ScoveragePlatformTest(TestBase): scoverage_path = '//:scoverage' blacklist_file_path = 'my/file/new_blacklist_scoverage_test' - def setup_scala_coverage_platform(self): + def setup_scoverage_platform(self): options = { ScalaPlatform.options_scope: { 'version': 'custom', @@ -23,13 +26,13 @@ def setup_scala_coverage_platform(self): } options2 = { - ScalaCoveragePlatform.options_scope: { + ScoveragePlatform.options_scope: { 'enable_scoverage' : 'False' } } init_subsystem(ScalaPlatform, options) - init_subsystem(ScalaCoveragePlatform, options2) + init_subsystem(ScoveragePlatform, options2) self.make_target('//:scalastyle', JarLibrary, @@ -59,26 +62,26 @@ def setup_scala_coverage_platform(self): # ==========> TESTS <============= # ================================ def test_subsystem_defaults(self): - init_subsystem(ScalaCoveragePlatform) + init_subsystem(ScoveragePlatform) - subsystem = ScalaCoveragePlatform.global_instance() + subsystem = ScoveragePlatform.global_instance() self.assertEqual(False, subsystem.get_options().enable_scoverage) self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) def test_subsystem_option_sets(self): - init_subsystem(ScalaCoveragePlatform) - ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + init_subsystem(ScoveragePlatform) + ScoveragePlatform.global_instance().get_options().enable_scoverage = True - subsystem = ScalaCoveragePlatform.global_instance() + subsystem = ScoveragePlatform.global_instance() self.assertEqual(True, subsystem.get_options().enable_scoverage) self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) def test_library_scoverage_enabled(self): - self.setup_scala_coverage_platform() - ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + self.setup_scoverage_platform() + ScoveragePlatform.global_instance().get_options().enable_scoverage = True self.create_file( relpath='a/scala/pass.scala', @@ -100,8 +103,8 @@ def main(args: Array[String]) { def test_library_scoverage_disabled(self): - self.setup_scala_coverage_platform() - ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = False + self.setup_scoverage_platform() + ScoveragePlatform.global_instance().get_options().enable_scoverage = False self.create_file( relpath='a/scala/pass.scala', @@ -132,8 +135,8 @@ def test_blacklist(self): instrumented as long as `scalac_plugins` do not contain `scoverage`. :return: """ - self.setup_scala_coverage_platform() - ScalaCoveragePlatform.global_instance().get_options().enable_scoverage = True + self.setup_scoverage_platform() + ScoveragePlatform.global_instance().get_options().enable_scoverage = True tmp = self.create_file( relpath=self.blacklist_file_path, @@ -141,7 +144,7 @@ def test_blacklist(self): a/scala:blacked """) ) - ScalaCoveragePlatform.global_instance().get_options().blacklist_file = tmp + ScoveragePlatform.global_instance().get_options().blacklist_file = tmp self.create_file( relpath='a/scala/pass.scala', diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD index 4acbd69a97f..a7585af817f 100644 --- a/tests/python/pants_test/backend/jvm/tasks/BUILD +++ b/tests/python/pants_test/backend/jvm/tasks/BUILD @@ -648,6 +648,7 @@ python_tests( sources = ['test_scalafmt.py'], dependencies = [ 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', 'src/python/pants/backend/jvm/tasks:scalafmt', @@ -676,6 +677,7 @@ python_tests( dependencies = [ 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/backend/jvm/targets:java', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', diff --git a/tests/python/pants_test/backend/jvm/tasks/test_scalafmt.py b/tests/python/pants_test/backend/jvm/tasks/test_scalafmt.py index 8b1aadc6f03..65089b94dbf 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_scalafmt.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_scalafmt.py @@ -5,6 +5,7 @@ from textwrap import dedent from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.junit_tests import JUnitTests from pants.backend.jvm.targets.scala_library import ScalaLibrary from pants.backend.jvm.tasks.scalafmt import ScalaFmtCheckFormat, ScalaFmtFormat @@ -30,6 +31,7 @@ def setUp(self): super().setUp() init_subsystem(ScalaPlatform) + init_subsystem(ScoveragePlatform) init_subsystem(SourceRootConfig) self.configuration = self.create_file( diff --git a/tests/python/pants_test/backend/jvm/tasks/test_scalastyle.py b/tests/python/pants_test/backend/jvm/tasks/test_scalastyle.py index 45e2800a81f..333af9fe615 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_scalastyle.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_scalastyle.py @@ -5,6 +5,7 @@ from textwrap import dedent from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.java_library import JavaLibrary from pants.backend.jvm.targets.scala_library import ScalaLibrary @@ -68,6 +69,7 @@ def setUp(self): 'version': '2.11' } }) + init_subsystem(ScoveragePlatform) def test_initialize_config_no_config_settings(self): with self.assertRaises(Scalastyle.UnspecifiedConfig): From c7f560a4aabefc458d4bceabfdec6f2497cfa11f Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 19 Jul 2019 17:02:29 -0700 Subject: [PATCH 08/35] adding scoverage dependency for unit tests to pass --- tests/python/pants_test/backend/project_info/tasks/BUILD | 1 + .../pants_test/backend/project_info/tasks/test_export.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/python/pants_test/backend/project_info/tasks/BUILD b/tests/python/pants_test/backend/project_info/tasks/BUILD index 6cc0b9b4ea9..c7c5478d5ef 100644 --- a/tests/python/pants_test/backend/project_info/tasks/BUILD +++ b/tests/python/pants_test/backend/project_info/tasks/BUILD @@ -47,6 +47,7 @@ python_tests( 'src/python/pants/backend/jvm/subsystems:junit', 'src/python/pants/backend/jvm/subsystems:jvm_platform', 'src/python/pants/backend/jvm/subsystems:scala_platform', + 'src/python/pants/backend/jvm/subsystems:scoverage_platform', 'src/python/pants/backend/jvm/targets:java', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/targets:scala', diff --git a/tests/python/pants_test/backend/project_info/tasks/test_export.py b/tests/python/pants_test/backend/project_info/tasks/test_export.py index e7dfecea788..4495c01321a 100644 --- a/tests/python/pants_test/backend/project_info/tasks/test_export.py +++ b/tests/python/pants_test/backend/project_info/tasks/test_export.py @@ -12,6 +12,7 @@ from pants.backend.jvm.subsystems.jvm_platform import JvmPlatform from pants.backend.jvm.subsystems.resolve_subsystem import JvmResolveSubsystem from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.java_library import JavaLibrary from pants.backend.jvm.targets.junit_tests import JUnitTests @@ -54,7 +55,7 @@ def setUp(self): 'version': 'custom' } } - init_subsystems([JUnit, ScalaPlatform], scala_options) + init_subsystems([JUnit, ScalaPlatform, ScoveragePlatform], scala_options) self.make_target( ':scala-library', From c427e219081854e32227604fba27aaf4244ab22d Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Sat, 20 Jul 2019 14:18:43 -0700 Subject: [PATCH 09/35] Scoverage (2/2) - adding report generator --- .../backend/jvm/tasks/coverage/manager.py | 19 +- .../backend/jvm/tasks/coverage/scoverage.py | 152 +++++++++++ .../org/pantsbuild/scoverage/report/BUILD | 16 ++ .../scoverage/report/ScoverageRep.scala | 252 ++++++++++++++++++ .../scoverage/report/Settings.scala | 79 ++++++ 5 files changed, 513 insertions(+), 5 deletions(-) create mode 100644 src/python/pants/backend/jvm/tasks/coverage/scoverage.py create mode 100644 src/scala/org/pantsbuild/scoverage/report/BUILD create mode 100644 src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala create mode 100644 src/scala/org/pantsbuild/scoverage/report/Settings.scala diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index fd88223d99e..97daa22cda4 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -4,7 +4,9 @@ import os import shutil +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.tasks.coverage.cobertura import Cobertura +from pants.backend.jvm.tasks.coverage.scoverage import Scoverage from pants.backend.jvm.tasks.coverage.engine import NoCoverage from pants.backend.jvm.tasks.coverage.jacoco import Jacoco from pants.subsystem.subsystem import Subsystem @@ -57,14 +59,14 @@ class CodeCoverage(Subsystem): @classmethod def subsystem_dependencies(cls): - return super().subsystem_dependencies() + (Cobertura.Factory, Jacoco.Factory) + return super().subsystem_dependencies() + (Cobertura.Factory, Jacoco.Factory, Scoverage.Factory) # TODO(jtrobec): move these to subsystem scope after deprecating @staticmethod def register_junit_options(register, register_jvm_tool): register('--coverage', type=bool, fingerprint=True, help='Collect code coverage data.') register('--coverage-processor', advanced=True, fingerprint=True, - choices=['cobertura', 'jacoco'], default=None, + choices=['cobertura', 'jacoco', 'scoverage'], default=None, help="Which coverage processor to use if --coverage is enabled. If this option is " "unset but coverage is enabled implicitly or explicitly, defaults to 'cobertura'." "If this option is explicitly set, implies --coverage.") @@ -83,18 +85,25 @@ def register_junit_options(register, register_jvm_tool): # register options for coverage engines # TODO(jtrobec): get rid of these calls when engines are dependent subsystems Cobertura.register_junit_options(register, register_jvm_tool) + Scoverage.register_junit_options(register, register_jvm_tool) class InvalidCoverageEngine(Exception): """Indicates an invalid coverage engine type was selected.""" def get_coverage_engine(self, task, output_dir, all_targets, execute_java): options = task.get_options() - if options.coverage or options.coverage_processor or options.is_flagged('coverage_open'): + processor = options.coverage_processor + if ScoveragePlatform.global_instance().get_options().enable_scoverage: + processor = 'scoverage' + + if options.coverage or processor or options.is_flagged('coverage_open'): settings = CodeCoverageSettings.from_task(task, workdir=output_dir) - if options.coverage_processor in ('cobertura', None): + if processor in ('cobertura', None): return Cobertura.Factory.global_instance().create(settings, all_targets, execute_java) - elif options.coverage_processor == 'jacoco': + elif processor == 'jacoco': return Jacoco.Factory.global_instance().create(settings, all_targets, execute_java) + elif processor == 'scoverage': + return Scoverage.Factory.global_instance().create(settings, all_targets, execute_java) else: # NB: We should never get here since the `--coverage-processor` is restricted by `choices`, # but for clarity. diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py new file mode 100644 index 00000000000..0866cb1482f --- /dev/null +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -0,0 +1,152 @@ +# coding=utf-8 +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import functools +import os +import subprocess +import re +from pants.java.jar.jar_dependency import JarDependency +from pants.backend.jvm.tasks.coverage.engine import CoverageEngine +from pants.base.exceptions import TaskError +from pants.subsystem.subsystem import Subsystem +from pants.util.dirutil import relativize_paths, safe_mkdir, safe_mkdir_for, safe_walk, touch + + +report_generator = 'src/scala/org/pantsbuild/scoverage/report:gen2' + + +class Scoverage(CoverageEngine): + """Class to run coverage tests with scoverage""" + + + class Factory(Subsystem): + options_scope = 'scoverage-runtime' + + @classmethod + def create(cls, settings, targets, execute_java_for_targets): + """ + :param settings: Generic code coverage settings. + :type settings: :class:`CodeCoverageSettings` + :param list targets: A list of targets to instrument and record code coverage for. + :param execute_java_for_targets: A function that accepts a list of targets whose JVM platform + constraints are used to pick a JVM `Distribution`. The + function should also accept `*args` and `**kwargs` compatible + with the remaining parameters accepted by + `pants.java.util.execute_java`. + """ + + return Scoverage(settings, targets, execute_java_for_targets) + + + @staticmethod + def register_junit_options(register, register_jvm_tool): + + def scoverage_runtime_jar(**kwargs): + return JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', + rev='1.0.1-twitter', **kwargs) + + register_jvm_tool(register, + 'scalac-scoverage-runtime', + classpath=[ + scoverage_runtime_jar() + ]) + + def __init__(self, settings, targets, execute_java_for_targets): + """ + :param settings: Generic code coverage settings. + :type settings: :class:`CodeCoverageSettings` + :param list targets: A list of targets to instrument and record code coverage for. + :param execute_java_for_targets: A function that accepts a list of targets whose JVM platform + constraints are used to pick a JVM `Distribution`. The function + should also accept `*args` and `**kwargs` compatible with the + remaining parameters accepted by + `pants.java.util.execute_java`. + """ + self._settings = settings + self._context = settings.context + self._targets = targets + self._target_filters = [] # TODO(sameera): add target filtering for scoverage + self._execute_java = functools.partial(execute_java_for_targets, targets) + + def _iter_datafiles(self, output_dir): + for root, _, files in safe_walk(output_dir): + for f in files: + if f.startswith("scoverage"): + yield os.path.join(root, f) + + def _iter_datadirs(self, output_dir): + for root, dirs, _ in safe_walk(output_dir): + for d in dirs: + if d.startswith("measurements"): + yield os.path.join(root, d) + break + + + def instrument(self, output_dir): + # Since scoverage does compile time instrumentation, we only need to clean-up existing runs. + for datafile in self._iter_datafiles(output_dir): + os.unlink(datafile) + + + def run_modifications(self, output_dir): + measurement_dir = os.path.join(output_dir, "scoverage", "measurements") + safe_mkdir(measurement_dir, clean=True) + data_dir_option = f'-Dscoverage_measurement_path={measurement_dir}' + + return self.RunModifications.create( + classpath_prepend=self._settings.tool_classpath('scalac-scoverage-runtime'), + extra_jvm_options=[data_dir_option]) + + + def report(self, output_dir, execution_failed_exception=None): + if execution_failed_exception: + self._settings.log.warn('Test failed: {}'.format(execution_failed_exception)) + return + + for md in self._iter_datadirs(output_dir): + parent_dir = os.path.dirname(md) + base_report_dir = os.path.join(parent_dir, 'reports') + safe_mkdir(base_report_dir, clean=True) + + # TODO(sameera): add target filtering for scoverage + filtered_targets = self.filter_scoverage_targets(md) + self._execute_scoverage_report_gen(measurements_dir=md, report_dir=base_report_dir, + target_filter=filtered_targets) + + # Opening the last generated report in case `--no-test-junit-fast` is specified. + if self._settings.coverage_open: + return os.path.join(base_report_dir, 'html', 'index.html') + + def _execute_scoverage_report_gen(self, measurements_dir, report_dir, target_filter): + cmd = [ + './pants', + '--scoverage-enable-scoverage=True', + 'run', + f'{report_generator}', + f'--jvm-run-jvm-program-args=["-measurementsDirPath","{measurements_dir}","-reportDirPath",' + f'"{report_dir}"]', + ] + + # if target_filter: + # cmd += ['--jvm-run-jvm-program-args="-neededTargets={}"'.format(','.join(target_filter))] + + with self._context.new_workunit(name='scoverage_report_generator') as workunit: + result = subprocess.call(cmd) + + if result != 0: + raise TaskError("scoverage ... exited non-zero ({0})" + " 'failed to generate report'".format(result)) + + def filter_scoverage_targets(self, measurements_dir): + return [d for d in os.listdir(measurements_dir) if self._include_dir(d)] + + def _include_dir(self, dir): + if len(self._target_filters) == 0: + return True + else: + for filter in self._target_filters: + filter = filter.replace("/", ".") + if re.search(filter, dir) is not None: + return True + return False diff --git a/src/scala/org/pantsbuild/scoverage/report/BUILD b/src/scala/org/pantsbuild/scoverage/report/BUILD new file mode 100644 index 00000000000..438520754ff --- /dev/null +++ b/src/scala/org/pantsbuild/scoverage/report/BUILD @@ -0,0 +1,16 @@ +scala_library( + dependencies = [ + '3rdparty/jvm/commons-io', + '3rdparty/jvm/org/scala-sbt:util-logging', + 'src/scala/org/pantsbuild/zinc/options', + ], +) + +jvm_binary( + name = "gen2", + basename = "gen2", + main = "org.pantsbuild.scoverage.report.ScoverageRep", + dependencies=[ + ":report", + ] +) diff --git a/src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala b/src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala new file mode 100644 index 00000000000..c23dc045fbd --- /dev/null +++ b/src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala @@ -0,0 +1,252 @@ +package org.pantsbuild.scoverage.report + +import java.io.File +import org.pantsbuild.zinc.options.Parsed +import org.apache.commons.io.FileUtils +import sbt.internal.util.ConsoleLogger +import scoverage.{Coverage, IOUtils, Serializer} +import scoverage.report.{ScoverageHtmlWriter, ScoverageXmlWriter} + +object ScoverageRep { + val Scoverage = "scoverage" + + // Copying the logger from org.pantsbuild.zinc.bootstrap + // As per https://github.com/pantsbuild/pants/issues/6160, this is a workaround + // so we can run zinc without $PATH (as needed in remoting). + System.setProperty("sbt.log.format", "true") + + val cl = ConsoleLogger.apply() + + case class ReportOptions( + loadDataDir: Boolean, + measurementsPath: String, + sourcePath: String, + reportPath: String, + dataPath: String, + writeHtml: Boolean, + writeXml: Boolean, + writeXmlDebug: Boolean, + cleanOld: Boolean, + targetFilters: Seq[String] + ) { + val writeReports: Boolean = { + val write = writeHtml || writeXml + if (!write) { + // we could end here, but we will still attempt to load the coverage and dump stats as info + // to the log, so maybe somebody would want to run this w/o generating reports? + cl.warn("No report output format specified, so no reports will be written.") + } + write + } + } + + object ReportOptions { + def apply(settings: Settings): ReportOptions = { + ReportOptions( + loadDataDir = settings.loadDataDir, + measurementsPath = settings.measurementsDirPath, + sourcePath = settings.sourceDirPath, + reportPath = settings.reportDirPath, + dataPath = settings.dataDirPath, + writeHtml = settings.writeHtmlReport, + writeXml = settings.writeXmlReport, + writeXmlDebug = settings.writeXmlDebug, + cleanOld = settings.cleanOldReports, + targetFilters = settings.targetFilters + ) + } + } + + /** + * + * @param dataDirs list of measurement directories for which coverage + * report has to be generated + * @return [Coverage] object for the [dataDirs] + */ + private def aggregatedCoverage(dataDirs: Seq[File]): Coverage = { + var id = 0 + val coverage = Coverage() + dataDirs foreach { dataDir => + val coverageFile: File = Serializer.coverageFile(dataDir) + if (coverageFile.exists) { + val subcoverage: Coverage = Serializer.deserialize(coverageFile) + val measurementFiles: Array[File] = IOUtils.findMeasurementFiles(dataDir) + val measurements = IOUtils.invoked(measurementFiles.toIndexedSeq) + subcoverage.apply(measurements) + subcoverage.statements foreach { stmt => + // need to ensure all the ids are unique otherwise the coverage object will have stmt collisions + id = id + 1 + coverage add stmt.copy(id = id) + } + } + } + coverage + } + + /** + * + * @param dataDirs list of measurement directories for which coverage + * report has to be generated + * @return Coverage object wrapped in Option + */ + private def aggregate(dataDirs: Seq[File]): Option[Coverage] = { + cl.success(s"Found ${dataDirs.size} subproject scoverage data directories [${dataDirs.mkString(",")}]") + if (dataDirs.nonEmpty) { + Some(aggregatedCoverage(dataDirs)) + } else { + None + } + } + + /** + * Select the appropriate directories for which the scoverage report has + * to be generated. If [targetFiles] is empty, report is generated for all + * measurements directories inside in [dataDir]. + */ + private def filterFiles(dataDir: File, options: ReportOptions): Seq[File] = { + val targetFiles = options.targetFilters + + if(targetFiles.nonEmpty) { + cl.info(s"Looking for targets: $targetFiles") + dataDir.listFiles.filter(_.isDirectory).toSeq.filter { + file => targetFiles.contains(file.getName()) + } + } + else { + dataDir.listFiles.filter(_.isDirectory).toSeq + } + } + + /** + * Aggregating coverage from all the coverage measurements. + */ + private def loadAggregatedCoverage(dataPath: String, options: ReportOptions): Option[Coverage] = { + val dataDir: File = new File(dataPath) + cl.info(s"Attempting to open scoverage data dir: [$dataDir]") + if(dataDir.exists){ + cl.info(s"Aggregating coverage.") + val dataDirs: Seq[File] = filterFiles(dataDir, options) + aggregate(dataDirs) + } + else { + cl.error("Coverage directory does not exists.") + None + } + } + + /** + * Loads coverage data from the specified data directory. + */ + private def loadCoverage(dataPath: String): Option[Coverage] = { + val dataDir: File = new File(dataPath) + cl.info(s"Attempting to open scoverage data dir [$dataDir]") + + if (dataDir.exists) { + val coverageFile = Serializer.coverageFile(dataDir) + cl.info(s"Reading scoverage instrumentation [$coverageFile]") + + coverageFile.exists match { + case true => + val coverage = Serializer.deserialize(coverageFile) + cl.info(s"Reading scoverage measurements...") + + val measurementFiles = IOUtils.findMeasurementFiles(dataDir) + val measurements = IOUtils.invoked(measurementFiles) + coverage.apply(measurements) + Some(coverage) + + case false => + cl.error("Coverage file did not exist") + None + + } + } else { + cl.error("Data dir did not exist!") + None + } + + } + + /** + * Writes coverage reports usign the specified source path to the specified report directory. + */ + private def writeReports(coverage: Coverage, options: ReportOptions): Unit = { + val sourceDir = new File(options.sourcePath) + val reportDir = new File(options.reportPath) + val reportDirHtml = new File(options.reportPath + "/html") + val reportDirXml = new File(options.reportPath + "/xml") + + if (sourceDir.exists) { + if (options.cleanOld && reportDir.exists) { + cl.info(s"Nuking old report directory [$reportDir].") + FileUtils.deleteDirectory(reportDir) + } + + if (!reportDir.exists) { + cl.info(s"Creating HTML report directory [$reportDirHtml]") + reportDirHtml.mkdirs + cl.info(s"Creating XML report directory [$reportDirXml]") + reportDirXml.mkdirs + } + + if (options.writeHtml) { + cl.info(s"Writing HTML scoverage reports to [$reportDirHtml]") + new ScoverageHtmlWriter(Seq(sourceDir), reportDirHtml, None).write(coverage) + } + + if (options.writeXml) { + cl.info(s"Writing XML scoverage reports to [$reportDirXml]") + new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, false).write(coverage) + if (options.writeXmlDebug) { + new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, true).write(coverage) + } + } + } else { + cl.error(s"Source dir [$sourceDir] does not exist") + } + + cl.success(s"Statement coverage: ${coverage.statementCoverageFormatted}%") + cl.success(s"Branch coverage: ${coverage.branchCoverageFormatted}%") + } + + + def main(args: Array[String]): Unit = { + val Parsed(settings, residual, errors) = Settings.parse(args) + val reportOptions = ReportOptions(settings) + + // bail out on any command-line option errors + if (errors.nonEmpty) { + for (error <- errors) System.err.println(error) + System.err.println("See %s -help for information about options" format Scoverage) + sys.exit(1) + } + + if (settings.help) { + Settings.printUsage(Scoverage) + return + } + + settings.loadDataDir match { + case false => + loadAggregatedCoverage(reportOptions.measurementsPath, reportOptions) match { + case Some(cov) => + cl.success("Coverage loaded successfully.\n") + if (reportOptions.writeReports) { + writeReports(cov, reportOptions) + } + + case None => cl.error("Failed to load coverage.") + } + case true => + loadCoverage(reportOptions.dataPath) match { + case Some(cov) => + cl.success("Coverage loaded successfully!") + if (reportOptions.writeReports) { + writeReports(cov, reportOptions) + } + case _ => cl.error("Failed to load coverage") + } + } + + } +} diff --git a/src/scala/org/pantsbuild/scoverage/report/Settings.scala b/src/scala/org/pantsbuild/scoverage/report/Settings.scala new file mode 100644 index 00000000000..f05af9d2841 --- /dev/null +++ b/src/scala/org/pantsbuild/scoverage/report/Settings.scala @@ -0,0 +1,79 @@ +package org.pantsbuild.scoverage.report + +import org.pantsbuild.zinc.options.OptionSet +import org.pantsbuild.zinc.options.ArgumentOption + +/** + * All parsed command-line options. + */ +case class Settings( + help: Boolean = false, + loadDataDir: Boolean = false, + measurementsDirPath: String = "", + reportDirPath: String = "", + sourceDirPath: String = ".", + dataDirPath: String = "", + writeHtmlReport: Boolean = true, + writeXmlReport: Boolean = true, + writeXmlDebug: Boolean = false, + cleanOldReports: Boolean = true, + targetFilters:Seq[String] = Seq() +) + +object Settings extends OptionSet2[Settings] { + override def empty = Settings() + + override val options = Seq( + boolean( ("-help", "-h"), "Print this usage message", + (s: Settings) => s.copy(help = true)), + + boolean( "-loadDataDir", "Load a single measurements directory instead of aggregating coverage reports. Must pass in `dataDirPath `", + (s: Settings) => s.copy(loadDataDir = true)), + + string( "-measurementsDirPath", "dir", "Directory where all scoverage measurements data is stored.", + (s: Settings, dir: String) => s.copy(measurementsDirPath = dir)), + + string( "-reportDirPath", "dir", "Target output directory to place the reports.", + (s: Settings, dir: String) => s.copy(reportDirPath = dir)), + + string( "-sourceDirPath", "dir", "Directory containing the project sources.", + (s: Settings, dir: String) => s.copy(sourceDirPath = dir)), + + string( "-dataDirPath", "dir", "Scoverage data file directory to be used in case report needed for single measurements " + + "directory. Must set `loadDataDir` to use this options.", + (s: Settings, dir: String) => s.copy(dataDirPath = dir)), + + boolean( "-writeHtmlReport", "Write the HTML version of the coverage report.", + (s: Settings) => s.copy(writeHtmlReport = true)), + + boolean( "-writeXmlReport", "Write the XML version of the coverage report.", + (s: Settings) => s.copy(writeXmlReport = true)), + + boolean( "-writeXmlDebug", "Write debug information to the XML version of the coverage report.", + (s: Settings) => s.copy(writeXmlDebug = true)), + + boolean( "-cleanOldReports", "Delete any existing reports directory prior to writing reports.", + (s: Settings) => s.copy(cleanOldReports = true)), + + stringList("-targetFilters", "filters", "Directory names for which report has to be generated.", + (s:Settings, filters: Seq[String]) => s.copy(targetFilters = filters)), + + ) +} + +trait OptionSet2[T] extends OptionSet[T]{ + def stringList(opt: String, arg: String, desc: String, action: (T, Seq[String]) => T) = new StringListOption[T](Seq(opt), arg, desc, action) +} + + +class StringListOption[Context]( + val options: Seq[String], + val argument: String, + val description: String, + val action: (Context, Seq[String]) => Context) + extends ArgumentOption[Seq[String], Context]{ + def parse(arg: String): Option[Seq[String]] = { + val expanded = arg.split(",") + Some(expanded) + } +} From 324f59daa6780709deaf8d4babae77a313214462 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Sat, 20 Jul 2019 14:35:37 -0700 Subject: [PATCH 10/35] changing report generator's name --- src/scala/org/pantsbuild/scoverage/report/BUILD | 2 +- .../report/{ScoverageRep.scala => ScoverageReport.scala} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/scala/org/pantsbuild/scoverage/report/{ScoverageRep.scala => ScoverageReport.scala} (99%) diff --git a/src/scala/org/pantsbuild/scoverage/report/BUILD b/src/scala/org/pantsbuild/scoverage/report/BUILD index 438520754ff..a5c9786f48f 100644 --- a/src/scala/org/pantsbuild/scoverage/report/BUILD +++ b/src/scala/org/pantsbuild/scoverage/report/BUILD @@ -9,7 +9,7 @@ scala_library( jvm_binary( name = "gen2", basename = "gen2", - main = "org.pantsbuild.scoverage.report.ScoverageRep", + main = "org.pantsbuild.scoverage.report.ScoverageReport", dependencies=[ ":report", ] diff --git a/src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala b/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala similarity index 99% rename from src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala rename to src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala index c23dc045fbd..a88ce3a8d98 100644 --- a/src/scala/org/pantsbuild/scoverage/report/ScoverageRep.scala +++ b/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala @@ -7,7 +7,7 @@ import sbt.internal.util.ConsoleLogger import scoverage.{Coverage, IOUtils, Serializer} import scoverage.report.{ScoverageHtmlWriter, ScoverageXmlWriter} -object ScoverageRep { +object ScoverageReport { val Scoverage = "scoverage" // Copying the logger from org.pantsbuild.zinc.bootstrap From 1a7e8fc81f700982d3d122971613b363b6ff0ca6 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 09:50:46 -0700 Subject: [PATCH 11/35] scoverage report generator --- 3rdparty/jvm/com/github/scopt/BUILD | 2 +- 3rdparty/jvm/com/twitter/BUILD | 6 + .../org/pantsbuild/scoverage/report/BUILD | 3 +- .../scoverage/report/ScoverageReport.scala | 149 +++++++++--------- .../scoverage/report/Settings.scala | 134 ++++++++-------- 5 files changed, 149 insertions(+), 145 deletions(-) diff --git a/3rdparty/jvm/com/github/scopt/BUILD b/3rdparty/jvm/com/github/scopt/BUILD index 23730e2b9c9..506e2064f3d 100644 --- a/3rdparty/jvm/com/github/scopt/BUILD +++ b/3rdparty/jvm/com/github/scopt/BUILD @@ -3,7 +3,7 @@ jar_library( jars=[ - scala_jar(org='com.github.scopt', name='scopt', rev='3.7.0'), + scala_jar(org='com.github.scopt', name='scopt', rev='4.0.0-RC2'), ], # Is used as a command line parser library ) diff --git a/3rdparty/jvm/com/twitter/BUILD b/3rdparty/jvm/com/twitter/BUILD index 095e12a7ae7..cc0e0a6d0e0 100644 --- a/3rdparty/jvm/com/twitter/BUILD +++ b/3rdparty/jvm/com/twitter/BUILD @@ -14,3 +14,9 @@ jar_library(name='scrooge-core', scala_jar(org='com.twitter', name='scrooge-core', rev=SCROOGE_REV), ], ) + +jar_library(name='scalac-scoverage-plugin', + jars=[ + scala_jar(org='com.twitter.scoverage', name='scalac-scoverage-plugin', rev='1.0.1-twitter'), + ], + ) diff --git a/src/scala/org/pantsbuild/scoverage/report/BUILD b/src/scala/org/pantsbuild/scoverage/report/BUILD index a5c9786f48f..276d90c6b6f 100644 --- a/src/scala/org/pantsbuild/scoverage/report/BUILD +++ b/src/scala/org/pantsbuild/scoverage/report/BUILD @@ -2,7 +2,8 @@ scala_library( dependencies = [ '3rdparty/jvm/commons-io', '3rdparty/jvm/org/scala-sbt:util-logging', - 'src/scala/org/pantsbuild/zinc/options', + '3rdparty/jvm/com/github/scopt', + '3rdparty/jvm/com/twitter:scalac-scoverage-plugin' ], ) diff --git a/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala b/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala index a88ce3a8d98..f370e18d08f 100644 --- a/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala +++ b/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala @@ -1,11 +1,12 @@ package org.pantsbuild.scoverage.report import java.io.File -import org.pantsbuild.zinc.options.Parsed -import org.apache.commons.io.FileUtils import sbt.internal.util.ConsoleLogger -import scoverage.{Coverage, IOUtils, Serializer} -import scoverage.report.{ScoverageHtmlWriter, ScoverageXmlWriter} +import org.apache.commons.io.FileUtils +import scopt.OParser + +import scoverage.{ Coverage, IOUtils, Serializer } +import scoverage.report.{ ScoverageHtmlWriter, ScoverageXmlWriter } object ScoverageReport { val Scoverage = "scoverage" @@ -15,7 +16,8 @@ object ScoverageReport { // so we can run zinc without $PATH (as needed in remoting). System.setProperty("sbt.log.format", "true") - val cl = ConsoleLogger.apply() + // Setting the logger + val logger = ConsoleLogger.apply() case class ReportOptions( loadDataDir: Boolean, @@ -27,14 +29,13 @@ object ScoverageReport { writeXml: Boolean, writeXmlDebug: Boolean, cleanOld: Boolean, - targetFilters: Seq[String] - ) { + targetFilters: Seq[String]) { val writeReports: Boolean = { val write = writeHtml || writeXml if (!write) { // we could end here, but we will still attempt to load the coverage and dump stats as info // to the log, so maybe somebody would want to run this w/o generating reports? - cl.warn("No report output format specified, so no reports will be written.") + logger.warn("No report output format specified, so no reports will be written.") } write } @@ -52,11 +53,9 @@ object ScoverageReport { writeXml = settings.writeXmlReport, writeXmlDebug = settings.writeXmlDebug, cleanOld = settings.cleanOldReports, - targetFilters = settings.targetFilters - ) + targetFilters = settings.targetFilters) } } - /** * * @param dataDirs list of measurement directories for which coverage @@ -90,7 +89,7 @@ object ScoverageReport { * @return Coverage object wrapped in Option */ private def aggregate(dataDirs: Seq[File]): Option[Coverage] = { - cl.success(s"Found ${dataDirs.size} subproject scoverage data directories [${dataDirs.mkString(",")}]") + logger.info(s"Found ${dataDirs.size} subproject scoverage data directories [${dataDirs.mkString(",")}]") if (dataDirs.nonEmpty) { Some(aggregatedCoverage(dataDirs)) } else { @@ -98,22 +97,37 @@ object ScoverageReport { } } + /** + * + * @param dataDir root directory to search under + * @return all the directories and subdirs containing scoverage files beginning at [dataDir] + */ + def getAllCoverageDirs(dataDir: File, acc: Seq[File]): Seq[File] = { + if (dataDir.listFiles.filter(_.isFile).toSeq.exists(_.getName contains "scoverage.coverage")) { + dataDir.listFiles.filter(_.isDirectory).toSeq + .foldRight(acc :+ dataDir) { (e, a) => getAllCoverageDirs(e, a) } + } else { + dataDir.listFiles.filter(_.isDirectory).toSeq + .foldRight(acc) { (e, a) => getAllCoverageDirs(e, a) } + } + } /** * Select the appropriate directories for which the scoverage report has * to be generated. If [targetFiles] is empty, report is generated for all * measurements directories inside in [dataDir]. */ - private def filterFiles(dataDir: File, options: ReportOptions): Seq[File] = { + def filterFiles(dataDir: File, options: ReportOptions): Seq[File] = { val targetFiles = options.targetFilters - if(targetFiles.nonEmpty) { - cl.info(s"Looking for targets: $targetFiles") - dataDir.listFiles.filter(_.isDirectory).toSeq.filter { - file => targetFiles.contains(file.getName()) + val coverareDirs = getAllCoverageDirs(dataDir, Seq()) + + if (targetFiles.nonEmpty) { + logger.info(s"Looking for targets: $targetFiles") + coverareDirs.filter { + file => targetFiles.exists(file.toString contains _) } - } - else { - dataDir.listFiles.filter(_.isDirectory).toSeq + } else { + coverareDirs } } @@ -122,14 +136,13 @@ object ScoverageReport { */ private def loadAggregatedCoverage(dataPath: String, options: ReportOptions): Option[Coverage] = { val dataDir: File = new File(dataPath) - cl.info(s"Attempting to open scoverage data dir: [$dataDir]") - if(dataDir.exists){ - cl.info(s"Aggregating coverage.") + logger.info(s"Attempting to open scoverage data dir: [$dataDir]") + if (dataDir.exists) { + logger.info(s"Aggregating coverage.") val dataDirs: Seq[File] = filterFiles(dataDir, options) aggregate(dataDirs) - } - else { - cl.error("Coverage directory does not exists.") + } else { + logger.error("Coverage directory does not exists.") None } } @@ -139,16 +152,16 @@ object ScoverageReport { */ private def loadCoverage(dataPath: String): Option[Coverage] = { val dataDir: File = new File(dataPath) - cl.info(s"Attempting to open scoverage data dir [$dataDir]") + logger.info(s"Attempting to open scoverage data dir [$dataDir]") if (dataDir.exists) { val coverageFile = Serializer.coverageFile(dataDir) - cl.info(s"Reading scoverage instrumentation [$coverageFile]") + logger.info(s"Reading scoverage instrumentation [$coverageFile]") coverageFile.exists match { case true => val coverage = Serializer.deserialize(coverageFile) - cl.info(s"Reading scoverage measurements...") + logger.info(s"Reading scoverage measurements...") val measurementFiles = IOUtils.findMeasurementFiles(dataDir) val measurements = IOUtils.invoked(measurementFiles) @@ -156,12 +169,12 @@ object ScoverageReport { Some(coverage) case false => - cl.error("Coverage file did not exist") + logger.error("Coverage file did not exist") None } } else { - cl.error("Data dir did not exist!") + logger.error("Data dir did not exist!") None } @@ -178,75 +191,65 @@ object ScoverageReport { if (sourceDir.exists) { if (options.cleanOld && reportDir.exists) { - cl.info(s"Nuking old report directory [$reportDir].") + logger.info(s"Nuking old report directory [$reportDir].") FileUtils.deleteDirectory(reportDir) } if (!reportDir.exists) { - cl.info(s"Creating HTML report directory [$reportDirHtml]") + logger.info(s"Creating HTML report directory [$reportDirHtml]") reportDirHtml.mkdirs - cl.info(s"Creating XML report directory [$reportDirXml]") + logger.info(s"Creating XML report directory [$reportDirXml]") reportDirXml.mkdirs } if (options.writeHtml) { - cl.info(s"Writing HTML scoverage reports to [$reportDirHtml]") + logger.info(s"Writing HTML scoverage reports to [$reportDirHtml]") new ScoverageHtmlWriter(Seq(sourceDir), reportDirHtml, None).write(coverage) } if (options.writeXml) { - cl.info(s"Writing XML scoverage reports to [$reportDirXml]") + logger.info(s"Writing XML scoverage reports to [$reportDirXml]") new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, false).write(coverage) if (options.writeXmlDebug) { new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, true).write(coverage) } } } else { - cl.error(s"Source dir [$sourceDir] does not exist") + logger.error(s"Source dir [$sourceDir] does not exist") } - cl.success(s"Statement coverage: ${coverage.statementCoverageFormatted}%") - cl.success(s"Branch coverage: ${coverage.branchCoverageFormatted}%") + logger.success(s"Statement coverage: ${coverage.statementCoverageFormatted}%") + logger.success(s"Branch coverage: ${coverage.branchCoverageFormatted}%") } - def main(args: Array[String]): Unit = { - val Parsed(settings, residual, errors) = Settings.parse(args) - val reportOptions = ReportOptions(settings) - - // bail out on any command-line option errors - if (errors.nonEmpty) { - for (error <- errors) System.err.println(error) - System.err.println("See %s -help for information about options" format Scoverage) - sys.exit(1) - } - - if (settings.help) { - Settings.printUsage(Scoverage) - return - } - - settings.loadDataDir match { - case false => - loadAggregatedCoverage(reportOptions.measurementsPath, reportOptions) match { - case Some(cov) => - cl.success("Coverage loaded successfully.\n") - if (reportOptions.writeReports) { - writeReports(cov, reportOptions) + OParser.parse(Settings.parser1, args, Settings()) match { + case Some(settings) => + val reportOptions = ReportOptions(settings) + + reportOptions.loadDataDir match { + case false => + loadAggregatedCoverage(reportOptions.measurementsPath, reportOptions) match { + case Some(cov) => + logger.success("Coverage loaded successfully.") + if (reportOptions.writeReports) { + writeReports(cov, reportOptions) + } + + case None => logger.error("Failed to load coverage.") } - - case None => cl.error("Failed to load coverage.") - } - case true => - loadCoverage(reportOptions.dataPath) match { - case Some(cov) => - cl.success("Coverage loaded successfully!") - if (reportOptions.writeReports) { - writeReports(cov, reportOptions) + case true => + loadCoverage(reportOptions.dataPath) match { + case Some(cov) => + logger.success("Coverage loaded successfully!") + if (reportOptions.writeReports) { + writeReports(cov, reportOptions) + } + case _ => logger.error("Failed to load coverage") } - case _ => cl.error("Failed to load coverage") } - } + case None => logger.error("Incorrect options supplied") + } } } diff --git a/src/scala/org/pantsbuild/scoverage/report/Settings.scala b/src/scala/org/pantsbuild/scoverage/report/Settings.scala index f05af9d2841..ad22b842daa 100644 --- a/src/scala/org/pantsbuild/scoverage/report/Settings.scala +++ b/src/scala/org/pantsbuild/scoverage/report/Settings.scala @@ -1,79 +1,73 @@ package org.pantsbuild.scoverage.report -import org.pantsbuild.zinc.options.OptionSet -import org.pantsbuild.zinc.options.ArgumentOption +import scopt.OParser /** * All parsed command-line options. */ case class Settings( - help: Boolean = false, - loadDataDir: Boolean = false, - measurementsDirPath: String = "", - reportDirPath: String = "", - sourceDirPath: String = ".", - dataDirPath: String = "", - writeHtmlReport: Boolean = true, - writeXmlReport: Boolean = true, - writeXmlDebug: Boolean = false, - cleanOldReports: Boolean = true, - targetFilters:Seq[String] = Seq() -) - -object Settings extends OptionSet2[Settings] { - override def empty = Settings() - - override val options = Seq( - boolean( ("-help", "-h"), "Print this usage message", - (s: Settings) => s.copy(help = true)), - - boolean( "-loadDataDir", "Load a single measurements directory instead of aggregating coverage reports. Must pass in `dataDirPath `", - (s: Settings) => s.copy(loadDataDir = true)), - - string( "-measurementsDirPath", "dir", "Directory where all scoverage measurements data is stored.", - (s: Settings, dir: String) => s.copy(measurementsDirPath = dir)), - - string( "-reportDirPath", "dir", "Target output directory to place the reports.", - (s: Settings, dir: String) => s.copy(reportDirPath = dir)), - - string( "-sourceDirPath", "dir", "Directory containing the project sources.", - (s: Settings, dir: String) => s.copy(sourceDirPath = dir)), - - string( "-dataDirPath", "dir", "Scoverage data file directory to be used in case report needed for single measurements " + - "directory. Must set `loadDataDir` to use this options.", - (s: Settings, dir: String) => s.copy(dataDirPath = dir)), - - boolean( "-writeHtmlReport", "Write the HTML version of the coverage report.", - (s: Settings) => s.copy(writeHtmlReport = true)), - - boolean( "-writeXmlReport", "Write the XML version of the coverage report.", - (s: Settings) => s.copy(writeXmlReport = true)), - - boolean( "-writeXmlDebug", "Write debug information to the XML version of the coverage report.", - (s: Settings) => s.copy(writeXmlDebug = true)), - - boolean( "-cleanOldReports", "Delete any existing reports directory prior to writing reports.", - (s: Settings) => s.copy(cleanOldReports = true)), - - stringList("-targetFilters", "filters", "Directory names for which report has to be generated.", - (s:Settings, filters: Seq[String]) => s.copy(targetFilters = filters)), - - ) -} - -trait OptionSet2[T] extends OptionSet[T]{ - def stringList(opt: String, arg: String, desc: String, action: (T, Seq[String]) => T) = new StringListOption[T](Seq(opt), arg, desc, action) -} - - -class StringListOption[Context]( - val options: Seq[String], - val argument: String, - val description: String, - val action: (Context, Seq[String]) => Context) - extends ArgumentOption[Seq[String], Context]{ - def parse(arg: String): Option[Seq[String]] = { - val expanded = arg.split(",") - Some(expanded) + loadDataDir: Boolean = false, + measurementsDirPath: String = "", + reportDirPath: String = "", + sourceDirPath: String = ".", + dataDirPath: String = "", + writeHtmlReport: Boolean = true, + writeXmlReport: Boolean = true, + writeXmlDebug: Boolean = false, + cleanOldReports: Boolean = true, + targetFilters: Seq[String] = Seq()) + +object Settings { + + val builder = OParser.builder[Settings] + val parser1 = { + import builder._ + OParser.sequence( + programName("scoverageReportGenerator"), + head("scoverageReportGenerator"), + + help('h', "help") + .text("Print this usage message."), + + opt[Unit]("loadDataDir") + .action((_, s: Settings) => s.copy(loadDataDir = true)) + .text("Load a single measurements directory instead of aggregating coverage reports. Must pass in `dataDirPath `"), + + opt[String]("measurementsDirPath") + .action((dir: String, s: Settings) => s.copy(measurementsDirPath = dir)) + .text("Directory where all scoverage measurements data is stored."), + + opt[String]("reportDirPath") + .action((dir: String, s: Settings) => s.copy(reportDirPath = dir)) + .text("Target output directory to place the reports."), + + opt[String]("sourceDirPath") + .action((dir: String, s: Settings) => s.copy(sourceDirPath = dir)) + .text("Directory containing the project sources."), + + opt[String]("dataDirPath") + .action((dir: String, s: Settings) => s.copy(dataDirPath = dir)) + .text("Scoverage data file directory to be used in case report needed for single measurements " + + "directory. Must set `loadDataDir` to use this options."), + + opt[Unit]("writeHtmlReport") + .action((_, s: Settings) => s.copy(writeHtmlReport = true)) + .text("Write the HTML version of the coverage report."), + + opt[Unit]("writeXmlReport") + .action((_, s: Settings) => s.copy(writeXmlReport = true)) + .text("Write the XML version of the coverage report."), + + opt[Unit]("writeXmlDebug") + .action((_, s: Settings) => s.copy(writeXmlDebug = true)) + .text("Write debug information to the XML version of the coverage report."), + + opt[Unit]("cleanOldReports") + .action((_, s: Settings) => s.copy(cleanOldReports = true)) + .text("Delete any existing reports directory prior to writing reports."), + + opt[Seq[String]]("targetFilters") + .action((f: Seq[String], s: Settings) => s.copy(targetFilters = f)) + .text("Directory names for which report has to be generated.")) } } From 739032a0029df27dcc369ea4c92a1a2b14c2ec59 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 09:55:46 -0700 Subject: [PATCH 12/35] adding scoverage report gen --- .../jvm/subsystems/scoverage_platform.py | 3 +- .../backend/jvm/tasks/coverage/manager.py | 24 ++++++++++-- .../backend/jvm/tasks/coverage/scoverage.py | 37 ++++++++++--------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 7c4caebaf3a..cb48803d729 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -26,7 +26,8 @@ def register_options(cls, register): default=False, type=bool, help='Specifies whether to generate scoverage reports for scala test targets.' - 'Default value is False') + 'Default value is False.If True,' + 'implies --test-junit-coverage-processor=scoverage.') register('--blacklist-file', default=blacklist_file, diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index 97daa22cda4..f1a483b01a7 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -1,6 +1,7 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging import os import shutil @@ -13,6 +14,7 @@ from pants.util.dirutil import safe_mkdir from pants.util.strutil import safe_shlex_split +logger = logging.getLogger(__name__) class CodeCoverageSettings: """A class containing settings for code coverage tasks.""" @@ -68,12 +70,15 @@ def register_junit_options(register, register_jvm_tool): register('--coverage-processor', advanced=True, fingerprint=True, choices=['cobertura', 'jacoco', 'scoverage'], default=None, help="Which coverage processor to use if --coverage is enabled. If this option is " - "unset but coverage is enabled implicitly or explicitly, defaults to 'cobertura'." - "If this option is explicitly set, implies --coverage.") + "unset but coverage is enabled implicitly or explicitly, defaults to 'cobertura'. " + "If this option is explicitly set, implies --coverage. If this option is set to " + "scoverage, then first scoverage MUST be enabled by passing option " + "--scoverage-enable-scoverage.") # We need to fingerprint this even though it nominally UI-only affecting option since the # presence of this option alone can implicitly flag on `--coverage`. register('--coverage-open', type=bool, fingerprint=True, - help='Open the generated HTML coverage report in a browser. Implies --coverage.') + help='Open the generated HTML coverage report in a browser. Implies --coverage ' + 'with cobertura as the engine.') register('--coverage-jvm-options', advanced=True, type=list, fingerprint=True, help='JVM flags to be added when running the coverage processor. For example: ' @@ -92,8 +97,19 @@ class InvalidCoverageEngine(Exception): def get_coverage_engine(self, task, output_dir, all_targets, execute_java): options = task.get_options() + enable_scoverage = ScoveragePlatform.global_instance().get_options().enable_scoverage processor = options.coverage_processor - if ScoveragePlatform.global_instance().get_options().enable_scoverage: + + if (processor == 'scoverage' and not enable_scoverage): + raise self.InvalidCoverageEngine("Cannot set processor to scoverage without first enabling " + "scoverage (by passing --scoverage-enable-scoverage option)") + + if enable_scoverage: + if processor is None: + logger.info("Scoverage is enabled. Setting coverage engine to scoverage.") + elif processor != 'scoverage': + logger.warning(f"Scoverage is enabled. Cannot use {processor} as the engine. Setting " + f"coverage engine to scoverage.") processor = 'scoverage' if options.coverage or processor or options.is_flagged('coverage_open'): diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 0866cb1482f..9f17a99a0a2 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -46,12 +46,25 @@ def scoverage_runtime_jar(**kwargs): return JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', rev='1.0.1-twitter', **kwargs) + def scoverage_report_jar(**kwargs): + return [JarDependency(org='org.scoverage', name='scoverage-report_2.12', + rev='3.4.9-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/scoverage/scoverage-report_2.12/3.4.9-SNAPSHOT/scoverage-report_2.12-3.4.9-SNAPSHOT.jar', **kwargs), + JarDependency(org='org.apache.directory.studio', name='org.apache.commons.io', rev='2.4'), + JarDependency(org='com.github.scopt', name='scopt_2.12', rev='4.0.0-RC2'), + JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter')] + register_jvm_tool(register, 'scalac-scoverage-runtime', classpath=[ scoverage_runtime_jar() ]) + register_jvm_tool(register, + 'scoverage-report', + classpath= + scoverage_report_jar() + ) + def __init__(self, settings, targets, execute_java_for_targets): """ :param settings: Generic code coverage settings. @@ -100,23 +113,13 @@ def run_modifications(self, output_dir): def report(self, output_dir, execution_failed_exception=None): - if execution_failed_exception: - self._settings.log.warn('Test failed: {}'.format(execution_failed_exception)) - return - - for md in self._iter_datadirs(output_dir): - parent_dir = os.path.dirname(md) - base_report_dir = os.path.join(parent_dir, 'reports') - safe_mkdir(base_report_dir, clean=True) - - # TODO(sameera): add target filtering for scoverage - filtered_targets = self.filter_scoverage_targets(md) - self._execute_scoverage_report_gen(measurements_dir=md, report_dir=base_report_dir, - target_filter=filtered_targets) - - # Opening the last generated report in case `--no-test-junit-fast` is specified. - if self._settings.coverage_open: - return os.path.join(base_report_dir, 'html', 'index.html') + cobertura_cp = self._settings.tool_classpath('scoverage-report') + result = self._execute_java(classpath=cobertura_cp, + main='scoverageReport.ScoverageReport', + jvm_options=self._settings.coverage_jvm_options, + args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", "--reportDirPath",f"{output_dir}/ScoverageReports"], + ) + def _execute_scoverage_report_gen(self, measurements_dir, report_dir, target_filter): cmd = [ From fc4ed94de97c4de073b65e31509ddb6074ab7954 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 11:56:15 -0700 Subject: [PATCH 13/35] testing with new report gen local publish --- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 9f17a99a0a2..265dd51be2b 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -47,10 +47,11 @@ def scoverage_runtime_jar(**kwargs): rev='1.0.1-twitter', **kwargs) def scoverage_report_jar(**kwargs): - return [JarDependency(org='org.scoverage', name='scoverage-report_2.12', - rev='3.4.9-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/scoverage/scoverage-report_2.12/3.4.9-SNAPSHOT/scoverage-report_2.12-3.4.9-SNAPSHOT.jar', **kwargs), + return [JarDependency(org='org.pantsbuild', name='scoverageReport_2.12', + rev='0.0.1-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/pantsbuild/scoverageReport_2.12/0.0.1-SNAPSHOT/scoverageReport_2.12-0.0.1-SNAPSHOT.jar', **kwargs), JarDependency(org='org.apache.directory.studio', name='org.apache.commons.io', rev='2.4'), - JarDependency(org='com.github.scopt', name='scopt_2.12', rev='4.0.0-RC2'), + JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), + JarDependency(org='org.scala-sbt', name='util-logging_2.12', rev='1.3.0-M8'), JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter')] register_jvm_tool(register, @@ -115,7 +116,7 @@ def run_modifications(self, output_dir): def report(self, output_dir, execution_failed_exception=None): cobertura_cp = self._settings.tool_classpath('scoverage-report') result = self._execute_java(classpath=cobertura_cp, - main='scoverageReport.ScoverageReport', + main='org.pantsbuild.scoverage.report.ScoverageReport', jvm_options=self._settings.coverage_jvm_options, args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", "--reportDirPath",f"{output_dir}/ScoverageReports"], ) From 3212da709ab3f6a4904cd34cbcb4b5449dc22fa4 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 15:00:01 -0700 Subject: [PATCH 14/35] changing scoverage to work with new reportGen --- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 265dd51be2b..d7f3ca3b65e 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -47,12 +47,13 @@ def scoverage_runtime_jar(**kwargs): rev='1.0.1-twitter', **kwargs) def scoverage_report_jar(**kwargs): - return [JarDependency(org='org.pantsbuild', name='scoverageReport_2.12', - rev='0.0.1-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/pantsbuild/scoverageReport_2.12/0.0.1-SNAPSHOT/scoverageReport_2.12-0.0.1-SNAPSHOT.jar', **kwargs), + return [JarDependency(org='org.pantsbuild', name='scoverage-report-generator_2.12', + rev='0.0.1-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/pantsbuild/scoverage-report-generator_2.12/0.0.1-SNAPSHOT/scoverage-report-generator_2.12-0.0.1-SNAPSHOT.jar', **kwargs), JarDependency(org='org.apache.directory.studio', name='org.apache.commons.io', rev='2.4'), JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), - JarDependency(org='org.scala-sbt', name='util-logging_2.12', rev='1.3.0-M8'), - JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter')] + JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), + JarDependency(org='org.slf4j', name='slf4j-simple', rev='1.7.26'), + JarDependency(org='org.slf4j', name='slf4j-api', rev='1.7.26')] register_jvm_tool(register, 'scalac-scoverage-runtime', From f9561dc5f2a65f69743b1160d9584b75f9e1d9f7 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 16:36:38 -0700 Subject: [PATCH 15/35] relocatiing modifications to scala library --- .../jvm/subsystems/scoverage_platform.py | 45 +------------------ .../backend/jvm/targets/scala_library.py | 39 +++++++++++++--- .../backend/jvm/tasks/coverage/scoverage.py | 4 ++ 3 files changed, 37 insertions(+), 51 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index cb48803d729..4ddcdb154fd 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -26,7 +26,7 @@ def register_options(cls, register): default=False, type=bool, help='Specifies whether to generate scoverage reports for scala test targets.' - 'Default value is False.If True,' + 'Default value is False. If True,' 'implies --test-junit-coverage-processor=scoverage.') register('--blacklist-file', @@ -69,49 +69,6 @@ def injectables_spec_mapping(self): 'scoverage': [f"{self.get_options().scoverage_target_path}"], } - def get_scalac_plugins(self, target) -> List[str]: - """ - Adds 'scoverage' to scalac_plugins in case scoverage is enabled for that [target]. - :return: modified scalac_plugins - """ - - # Prevent instrumenting generated targets and targets in blacklist. - if target.identifier.startswith(".pants.d.gen") or self.is_blacklisted(target): - return target.payload.scalac_plugins - - scalac_plugins = target.payload.scalac_plugins - if scalac_plugins: - scalac_plugins.append(SCOVERAGE) - else: - scalac_plugins = [SCOVERAGE] - return scalac_plugins - - def get_scalac_plugin_args(self, target) -> Dict[str, List[str]]: - """ - Adds 'scoverage' to scalac_plugins_args in case scoverage is enabled for that [target]. - :return: modified scalac_plugins_args - """ - scalac_plugin_args = target.payload.scalac_plugin_args - if scalac_plugin_args: - scalac_plugin_args.update( - {"scoverage": ["writeToClasspath:true", f"dataDir:{target.identifier}"]}) - else: - scalac_plugin_args = { - "scoverage": ["writeToClasspath:true", f"dataDir:{target.identifier}"] - } - return scalac_plugin_args - - def get_compiler_option_sets(self, target): - """ - Adds 'scoverage' to compiler_options_sets in case scoverage is enabled for that [target]. - :return: modified compiler_option_sets - """ - compiler_option_sets = target.payload.compiler_option_sets - if compiler_option_sets: - list(compiler_option_sets).append(SCOVERAGE) - else: - compiler_option_sets = [SCOVERAGE] - return tuple(compiler_option_sets) def is_blacklisted(self, target) -> bool: """ diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index dc6d751a3ee..98c752266a6 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -10,6 +10,7 @@ from pants.base.payload_field import PrimitiveField from pants.build_graph.address import Address +SCOVERAGE = "scoverage" class ScalaLibrary(ExportableJvmLibrary): """A Scala library. @@ -26,6 +27,8 @@ class ScalaLibrary(ExportableJvmLibrary): default_sources_globs = '*.scala' default_sources_exclude_globs = JUnitTests.scala_test_globs + + @classmethod def subsystems(cls): return super().subsystems() + (ScalaPlatform, ScoveragePlatform,) @@ -48,7 +51,7 @@ def __init__(self, java_sources=None, payload=None, **kwargs): }) super().__init__(payload=payload, **kwargs) - self._scoverage = ScoveragePlatform.global_instance().get_options().enable_scoverage + self._scoverage = ScoveragePlatform.global_instance() @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): @@ -96,8 +99,17 @@ def scalac_plugins(self): :return: See constructor. :rtype: list of strings. """ - if self._scoverage: - return ScoveragePlatform.global_instance().get_scalac_plugins(self) + if self._scoverage.get_options().enable_scoverage: + # Prevent instrumenting generated targets and targets in blacklist. + if self.identifier.startswith(".pants.d.gen") or self._scoverage.is_blacklisted(self): + return self.payload.scalac_plugins + + scalac_plugins = self.payload.scalac_plugins + if scalac_plugins: + scalac_plugins.append(SCOVERAGE) + else: + scalac_plugins = [SCOVERAGE] + return scalac_plugins return self.payload.scalac_plugins @@ -107,8 +119,16 @@ def scalac_plugin_args(self): :return: See constructor. :rtype: map from string to list of strings. """ - if self._scoverage: - return ScoveragePlatform.global_instance().get_scalac_plugin_args(self) + if self._scoverage.get_options().enable_scoverage: + scalac_plugin_args = self.payload.scalac_plugin_args + if scalac_plugin_args: + scalac_plugin_args.update( + {"scoverage": ["writeToClasspath:true", f"dataDir:{self.identifier}"]}) + else: + scalac_plugin_args = { + "scoverage": ["writeToClasspath:true", f"dataDir:{self.identifier}"] + } + return scalac_plugin_args return self.payload.scalac_plugin_args @@ -119,7 +139,12 @@ def compiler_option_sets(self): :return: See constructor. :rtype: list """ - if self._scoverage: - return ScoveragePlatform.global_instance().get_compiler_option_sets(self) + if self._scoverage.get_options().enable_scoverage: + compiler_option_sets = self.payload.compiler_option_sets + if compiler_option_sets: + list(compiler_option_sets).append(SCOVERAGE) + else: + compiler_option_sets = [SCOVERAGE] + return tuple(compiler_option_sets) return self.payload.compiler_option_sets diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index d7f3ca3b65e..0143eda42e2 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -122,6 +122,10 @@ def report(self, output_dir, execution_failed_exception=None): args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", "--reportDirPath",f"{output_dir}/ScoverageReports"], ) + if result != 0: + raise TaskError('java {} ... exited non-zero ({}) - failed to {}' + .format('org.pantsbuild.scoverage.report.ScoverageReport', result, 'scoverage-report-generator')) + def _execute_scoverage_report_gen(self, measurements_dir, report_dir, target_filter): cmd = [ From 3d913e318b416fcd9d54e9fcc6da700efc985fac Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Tue, 23 Jul 2019 23:41:59 -0700 Subject: [PATCH 16/35] playing with new report gen --- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 0143eda42e2..38edc9c7f6a 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -119,7 +119,10 @@ def report(self, output_dir, execution_failed_exception=None): result = self._execute_java(classpath=cobertura_cp, main='org.pantsbuild.scoverage.report.ScoverageReport', jvm_options=self._settings.coverage_jvm_options, - args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", "--reportDirPath",f"{output_dir}/ScoverageReports"], + args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", + "--htmlDirPath", f"{output_dir}/scoverage/reports/html", + "--xmlDirPath",f"{output_dir}/scoverage/reports/xml", + "--cleanOldReports"], ) if result != 0: From 3209a9f76eb5322b115d6b1bb6e7b7ff80b8a9e0 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 14:42:49 -0700 Subject: [PATCH 17/35] adding scoverage runtime environment along with target filtering and blacklisted files --- .../jvm/subsystems/scoverage_platform.py | 23 ++- .../backend/jvm/targets/scala_library.py | 10 +- .../backend/jvm/tasks/coverage/manager.py | 1 - .../backend/jvm/tasks/coverage/scoverage.py | 167 +++++++++++------- 4 files changed, 121 insertions(+), 80 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 4ddcdb154fd..cb54eddd2aa 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -7,15 +7,12 @@ from pants.java.jar.jar_dependency import JarDependency from pants.backend.jvm.targets.jar_library import JarLibrary from pants.build_graph.address import Address -from typing import Any, Dict, List, Optional, Union SCOVERAGE = "scoverage" -blacklist_file = 'new_blacklist_scoverage' - class ScoveragePlatform(InjectablesMixin, Subsystem): - """The scala coverage platform.""" + """The scoverage platform.""" options_scope = 'scoverage' @@ -30,7 +27,6 @@ def register_options(cls, register): 'implies --test-junit-coverage-processor=scoverage.') register('--blacklist-file', - default=blacklist_file, type=str, help='Path to files containing targets not to be instrumented.') @@ -39,6 +35,17 @@ def register_options(cls, register): type=str, help='Path to the scoverage dependency.') + def __init__(self, *args, **kwargs): + super(ScoveragePlatform, self).__init__(*args, **kwargs) + + # Setting up the scoverage blacklist files which contains targets + # not to be instrumented. + if (self.get_options().blacklist_file and + os.path.exists(self.get_options().blacklist_file)): + self._blacklist_file_contents = open(self.get_options().blacklist_file).read() + else: + self._blacklist_file_contents = None + def scoverage_jar(self): return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), @@ -74,10 +81,12 @@ def is_blacklisted(self, target) -> bool: """ Checks if the [target] is blacklisted or not. """ - if not os.path.exists(self.get_options().blacklist_file): + + # File not specified + if not self._blacklist_file_contents: return False - if target.address.spec in open(self.get_options().blacklist_file).read(): + if target.address.spec in self._blacklist_file_contents: return True else: return False diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index 98c752266a6..5cbb5a3fac6 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -51,7 +51,7 @@ def __init__(self, java_sources=None, payload=None, **kwargs): }) super().__init__(payload=payload, **kwargs) - self._scoverage = ScoveragePlatform.global_instance() + self._scoverage_instance = ScoveragePlatform.global_instance() @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): @@ -99,9 +99,9 @@ def scalac_plugins(self): :return: See constructor. :rtype: list of strings. """ - if self._scoverage.get_options().enable_scoverage: + if self._scoverage_instance.get_options().enable_scoverage: # Prevent instrumenting generated targets and targets in blacklist. - if self.identifier.startswith(".pants.d.gen") or self._scoverage.is_blacklisted(self): + if self.identifier.startswith(".pants.d.gen") or self._scoverage_instance.is_blacklisted(self): return self.payload.scalac_plugins scalac_plugins = self.payload.scalac_plugins @@ -119,7 +119,7 @@ def scalac_plugin_args(self): :return: See constructor. :rtype: map from string to list of strings. """ - if self._scoverage.get_options().enable_scoverage: + if self._scoverage_instance.get_options().enable_scoverage: scalac_plugin_args = self.payload.scalac_plugin_args if scalac_plugin_args: scalac_plugin_args.update( @@ -139,7 +139,7 @@ def compiler_option_sets(self): :return: See constructor. :rtype: list """ - if self._scoverage.get_options().enable_scoverage: + if self._scoverage_instance.get_options().enable_scoverage: compiler_option_sets = self.payload.compiler_option_sets if compiler_option_sets: list(compiler_option_sets).append(SCOVERAGE) diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index f1a483b01a7..1d6ee6037c7 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -90,7 +90,6 @@ def register_junit_options(register, register_jvm_tool): # register options for coverage engines # TODO(jtrobec): get rid of these calls when engines are dependent subsystems Cobertura.register_junit_options(register, register_jvm_tool) - Scoverage.register_junit_options(register, register_jvm_tool) class InvalidCoverageEngine(Exception): """Indicates an invalid coverage engine type was selected.""" diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 38edc9c7f6a..461f2e41b5d 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -6,6 +6,8 @@ import os import subprocess import re + +from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin from pants.java.jar.jar_dependency import JarDependency from pants.backend.jvm.tasks.coverage.engine import CoverageEngine from pants.base.exceptions import TaskError @@ -13,18 +15,60 @@ from pants.util.dirutil import relativize_paths, safe_mkdir, safe_mkdir_for, safe_walk, touch -report_generator = 'src/scala/org/pantsbuild/scoverage/report:gen2' - class Scoverage(CoverageEngine): """Class to run coverage tests with scoverage""" - class Factory(Subsystem): + class Factory(Subsystem, JvmToolMixin): + + # Cannot have the same scope as ScoveragePlatform, i.e they + # both cannot share the scope `scoverage`. options_scope = 'scoverage-runtime' @classmethod - def create(cls, settings, targets, execute_java_for_targets): + def register_options(cls, register): + super(Scoverage.Factory, cls).register_options(register) + + + def scoverage_jar(name, **kwargs): + return JarDependency(org='com.twitter.scoverage', name=name, rev='1.0.1-twitter', **kwargs) + + def slf4j_jar(name): + return JarDependency(org='org.slf4j', name=name, rev='1.7.5') + + def scopt_jar(): + return JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0') + + def commons_jar(): + return JarDependency(org='commons-io', name='commons-io', rev='2.5') + + def scoverage_report_jar(**kwargs): + return JarDependency(org='org.pantsbuild', name='scoverage-report-generator_2.12', + rev='0.0.1', **kwargs) + + + # We need to inject report generator at runtime. + cls.register_jvm_tool(register, + 'scoverage-report', + classpath=[ + scoverage_report_jar(), + commons_jar(), + scopt_jar(), + slf4j_jar('slf4j-simple'), slf4j_jar('slf4j-api'), + scoverage_jar('scalac-scoverage-plugin_2.12') + ] + ) + + register('--target-filters', type=list, default=[], + help='Regex patterns passed to scoverage, specifying which targets should be ' + 'included in reports. All targets matching any of the patterns will be ' + 'included when generating reports. If no targets are specified, all ' + 'targets are included, which would be the same as specifying ".*" as a ' + 'filter.') + + + def create(self, settings, targets, execute_java_for_targets): """ :param settings: Generic code coverage settings. :type settings: :class:`CodeCoverageSettings` @@ -36,38 +80,15 @@ def create(cls, settings, targets, execute_java_for_targets): `pants.java.util.execute_java`. """ - return Scoverage(settings, targets, execute_java_for_targets) - + report_path = self.tool_classpath_from_products(settings.context.products, 'scoverage-report', + scope='scoverage-runtime') - @staticmethod - def register_junit_options(register, register_jvm_tool): + target_filters = Scoverage.Factory.global_instance().get_options().target_filters - def scoverage_runtime_jar(**kwargs): - return JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-runtime_2.12', - rev='1.0.1-twitter', **kwargs) + return Scoverage(report_path, target_filters, settings, targets, execute_java_for_targets) - def scoverage_report_jar(**kwargs): - return [JarDependency(org='org.pantsbuild', name='scoverage-report-generator_2.12', - rev='0.0.1-SNAPSHOT', url='file:/Users/sameera/.m2/repository/org/pantsbuild/scoverage-report-generator_2.12/0.0.1-SNAPSHOT/scoverage-report-generator_2.12-0.0.1-SNAPSHOT.jar', **kwargs), - JarDependency(org='org.apache.directory.studio', name='org.apache.commons.io', rev='2.4'), - JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), - JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), - JarDependency(org='org.slf4j', name='slf4j-simple', rev='1.7.26'), - JarDependency(org='org.slf4j', name='slf4j-api', rev='1.7.26')] - - register_jvm_tool(register, - 'scalac-scoverage-runtime', - classpath=[ - scoverage_runtime_jar() - ]) - - register_jvm_tool(register, - 'scoverage-report', - classpath= - scoverage_report_jar() - ) - def __init__(self, settings, targets, execute_java_for_targets): + def __init__(self, report_path, target_filters, settings, targets, execute_java_for_targets): """ :param settings: Generic code coverage settings. :type settings: :class:`CodeCoverageSettings` @@ -81,15 +102,26 @@ def __init__(self, settings, targets, execute_java_for_targets): self._settings = settings self._context = settings.context self._targets = targets - self._target_filters = [] # TODO(sameera): add target filtering for scoverage + self._target_filters = target_filters self._execute_java = functools.partial(execute_java_for_targets, targets) + self._coverage_force = settings.options.coverage_force + self._report_path = report_path + + # All scoverage instrument files have the name "scoverage.coverage" and + # all measurement files are called "scoverage.measurements.". + # This function is used in [indtrument(output_dir)] function below to clean up + # all pre-existing scoverage files before generating new ones. def _iter_datafiles(self, output_dir): for root, _, files in safe_walk(output_dir): for f in files: if f.startswith("scoverage"): yield os.path.join(root, f) + # Used below for target filtering. Returns the parent directories + # under which all the scoverage data (for all targets) is stored. + # Currently, since all scoverage data for a test target is stored under + # `scoverage/measurements`, path to `scoverage/measurements` is returned. def _iter_datadirs(self, output_dir): for root, dirs, _ in safe_walk(output_dir): for d in dirs: @@ -109,47 +141,45 @@ def run_modifications(self, output_dir): safe_mkdir(measurement_dir, clean=True) data_dir_option = f'-Dscoverage_measurement_path={measurement_dir}' - return self.RunModifications.create( - classpath_prepend=self._settings.tool_classpath('scalac-scoverage-runtime'), - extra_jvm_options=[data_dir_option]) + return self.RunModifications.create(extra_jvm_options=[data_dir_option]) def report(self, output_dir, execution_failed_exception=None): - cobertura_cp = self._settings.tool_classpath('scoverage-report') - result = self._execute_java(classpath=cobertura_cp, - main='org.pantsbuild.scoverage.report.ScoverageReport', + if execution_failed_exception: + self._settings.log.warn('Test failed: {}'.format(execution_failed_exception)) + if self._coverage_force: + self._settings.log.warn('Generating report even though tests failed, because the' + 'coverage-force flag is set.') + else: + return + + main = 'org.pantsbuild.scoverage.report.ScoverageReport' + scoverage_cp = self._report_path + + final_target_dirs = [] + for parent_measurements_dir in self._iter_datadirs(output_dir): + final_target_dirs += self.filter_scoverage_targets(parent_measurements_dir) + + args = ["--measurementsDirPath",f"{output_dir}", + "--htmlDirPath", f"{output_dir}/scoverage/reports/html", + "--xmlDirPath", f"{output_dir}/scoverage/reports/xml", + "--targetFilters", f"{','.join(final_target_dirs)}", + "--cleanOldReports"] + + + result = self._execute_java(classpath=scoverage_cp, + main=main, jvm_options=self._settings.coverage_jvm_options, - args=["--measurementsDirPath",f"{output_dir}/scoverage/measurements", - "--htmlDirPath", f"{output_dir}/scoverage/reports/html", - "--xmlDirPath",f"{output_dir}/scoverage/reports/xml", - "--cleanOldReports"], - ) + args=args, + workunit_factory=self._context.new_workunit, + workunit_name='scoverage-report-generator') if result != 0: - raise TaskError('java {} ... exited non-zero ({}) - failed to {}' - .format('org.pantsbuild.scoverage.report.ScoverageReport', result, 'scoverage-report-generator')) - - - def _execute_scoverage_report_gen(self, measurements_dir, report_dir, target_filter): - cmd = [ - './pants', - '--scoverage-enable-scoverage=True', - 'run', - f'{report_generator}', - f'--jvm-run-jvm-program-args=["-measurementsDirPath","{measurements_dir}","-reportDirPath",' - f'"{report_dir}"]', - ] + raise TaskError(f"java {main} ... exited non-zero ({result}) - failed to scoverage-report-generator") - # if target_filter: - # cmd += ['--jvm-run-jvm-program-args="-neededTargets={}"'.format(','.join(target_filter))] - - with self._context.new_workunit(name='scoverage_report_generator') as workunit: - result = subprocess.call(cmd) - - if result != 0: - raise TaskError("scoverage ... exited non-zero ({0})" - " 'failed to generate report'".format(result)) + # Returns the directories under [measurements_dir] which need to + # be passed to the report generator. def filter_scoverage_targets(self, measurements_dir): return [d for d in os.listdir(measurements_dir) if self._include_dir(d)] @@ -158,7 +188,10 @@ def _include_dir(self, dir): return True else: for filter in self._target_filters: - filter = filter.replace("/", ".") + + # If target filter is specified as an address spec, turn it into + # target identifier as the scoverage directory names are made out of target identifiers. + filter = filter.replace("/", ".").replace(":", ".") if re.search(filter, dir) is not None: return True return False From 3cd1b8bf40c9dc0bab776dfa3dc4c8ff2105494e Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 14:45:15 -0700 Subject: [PATCH 18/35] removing report generator from here --- .../org/pantsbuild/scoverage/report/BUILD | 17 -- .../scoverage/report/ScoverageReport.scala | 255 ------------------ .../scoverage/report/Settings.scala | 73 ----- 3 files changed, 345 deletions(-) delete mode 100644 src/scala/org/pantsbuild/scoverage/report/BUILD delete mode 100644 src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala delete mode 100644 src/scala/org/pantsbuild/scoverage/report/Settings.scala diff --git a/src/scala/org/pantsbuild/scoverage/report/BUILD b/src/scala/org/pantsbuild/scoverage/report/BUILD deleted file mode 100644 index 276d90c6b6f..00000000000 --- a/src/scala/org/pantsbuild/scoverage/report/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - dependencies = [ - '3rdparty/jvm/commons-io', - '3rdparty/jvm/org/scala-sbt:util-logging', - '3rdparty/jvm/com/github/scopt', - '3rdparty/jvm/com/twitter:scalac-scoverage-plugin' - ], -) - -jvm_binary( - name = "gen2", - basename = "gen2", - main = "org.pantsbuild.scoverage.report.ScoverageReport", - dependencies=[ - ":report", - ] -) diff --git a/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala b/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala deleted file mode 100644 index f370e18d08f..00000000000 --- a/src/scala/org/pantsbuild/scoverage/report/ScoverageReport.scala +++ /dev/null @@ -1,255 +0,0 @@ -package org.pantsbuild.scoverage.report - -import java.io.File -import sbt.internal.util.ConsoleLogger -import org.apache.commons.io.FileUtils -import scopt.OParser - -import scoverage.{ Coverage, IOUtils, Serializer } -import scoverage.report.{ ScoverageHtmlWriter, ScoverageXmlWriter } - -object ScoverageReport { - val Scoverage = "scoverage" - - // Copying the logger from org.pantsbuild.zinc.bootstrap - // As per https://github.com/pantsbuild/pants/issues/6160, this is a workaround - // so we can run zinc without $PATH (as needed in remoting). - System.setProperty("sbt.log.format", "true") - - // Setting the logger - val logger = ConsoleLogger.apply() - - case class ReportOptions( - loadDataDir: Boolean, - measurementsPath: String, - sourcePath: String, - reportPath: String, - dataPath: String, - writeHtml: Boolean, - writeXml: Boolean, - writeXmlDebug: Boolean, - cleanOld: Boolean, - targetFilters: Seq[String]) { - val writeReports: Boolean = { - val write = writeHtml || writeXml - if (!write) { - // we could end here, but we will still attempt to load the coverage and dump stats as info - // to the log, so maybe somebody would want to run this w/o generating reports? - logger.warn("No report output format specified, so no reports will be written.") - } - write - } - } - - object ReportOptions { - def apply(settings: Settings): ReportOptions = { - ReportOptions( - loadDataDir = settings.loadDataDir, - measurementsPath = settings.measurementsDirPath, - sourcePath = settings.sourceDirPath, - reportPath = settings.reportDirPath, - dataPath = settings.dataDirPath, - writeHtml = settings.writeHtmlReport, - writeXml = settings.writeXmlReport, - writeXmlDebug = settings.writeXmlDebug, - cleanOld = settings.cleanOldReports, - targetFilters = settings.targetFilters) - } - } - /** - * - * @param dataDirs list of measurement directories for which coverage - * report has to be generated - * @return [Coverage] object for the [dataDirs] - */ - private def aggregatedCoverage(dataDirs: Seq[File]): Coverage = { - var id = 0 - val coverage = Coverage() - dataDirs foreach { dataDir => - val coverageFile: File = Serializer.coverageFile(dataDir) - if (coverageFile.exists) { - val subcoverage: Coverage = Serializer.deserialize(coverageFile) - val measurementFiles: Array[File] = IOUtils.findMeasurementFiles(dataDir) - val measurements = IOUtils.invoked(measurementFiles.toIndexedSeq) - subcoverage.apply(measurements) - subcoverage.statements foreach { stmt => - // need to ensure all the ids are unique otherwise the coverage object will have stmt collisions - id = id + 1 - coverage add stmt.copy(id = id) - } - } - } - coverage - } - - /** - * - * @param dataDirs list of measurement directories for which coverage - * report has to be generated - * @return Coverage object wrapped in Option - */ - private def aggregate(dataDirs: Seq[File]): Option[Coverage] = { - logger.info(s"Found ${dataDirs.size} subproject scoverage data directories [${dataDirs.mkString(",")}]") - if (dataDirs.nonEmpty) { - Some(aggregatedCoverage(dataDirs)) - } else { - None - } - } - - /** - * - * @param dataDir root directory to search under - * @return all the directories and subdirs containing scoverage files beginning at [dataDir] - */ - def getAllCoverageDirs(dataDir: File, acc: Seq[File]): Seq[File] = { - if (dataDir.listFiles.filter(_.isFile).toSeq.exists(_.getName contains "scoverage.coverage")) { - dataDir.listFiles.filter(_.isDirectory).toSeq - .foldRight(acc :+ dataDir) { (e, a) => getAllCoverageDirs(e, a) } - } else { - dataDir.listFiles.filter(_.isDirectory).toSeq - .foldRight(acc) { (e, a) => getAllCoverageDirs(e, a) } - } - } - /** - * Select the appropriate directories for which the scoverage report has - * to be generated. If [targetFiles] is empty, report is generated for all - * measurements directories inside in [dataDir]. - */ - def filterFiles(dataDir: File, options: ReportOptions): Seq[File] = { - val targetFiles = options.targetFilters - - val coverareDirs = getAllCoverageDirs(dataDir, Seq()) - - if (targetFiles.nonEmpty) { - logger.info(s"Looking for targets: $targetFiles") - coverareDirs.filter { - file => targetFiles.exists(file.toString contains _) - } - } else { - coverareDirs - } - } - - /** - * Aggregating coverage from all the coverage measurements. - */ - private def loadAggregatedCoverage(dataPath: String, options: ReportOptions): Option[Coverage] = { - val dataDir: File = new File(dataPath) - logger.info(s"Attempting to open scoverage data dir: [$dataDir]") - if (dataDir.exists) { - logger.info(s"Aggregating coverage.") - val dataDirs: Seq[File] = filterFiles(dataDir, options) - aggregate(dataDirs) - } else { - logger.error("Coverage directory does not exists.") - None - } - } - - /** - * Loads coverage data from the specified data directory. - */ - private def loadCoverage(dataPath: String): Option[Coverage] = { - val dataDir: File = new File(dataPath) - logger.info(s"Attempting to open scoverage data dir [$dataDir]") - - if (dataDir.exists) { - val coverageFile = Serializer.coverageFile(dataDir) - logger.info(s"Reading scoverage instrumentation [$coverageFile]") - - coverageFile.exists match { - case true => - val coverage = Serializer.deserialize(coverageFile) - logger.info(s"Reading scoverage measurements...") - - val measurementFiles = IOUtils.findMeasurementFiles(dataDir) - val measurements = IOUtils.invoked(measurementFiles) - coverage.apply(measurements) - Some(coverage) - - case false => - logger.error("Coverage file did not exist") - None - - } - } else { - logger.error("Data dir did not exist!") - None - } - - } - - /** - * Writes coverage reports usign the specified source path to the specified report directory. - */ - private def writeReports(coverage: Coverage, options: ReportOptions): Unit = { - val sourceDir = new File(options.sourcePath) - val reportDir = new File(options.reportPath) - val reportDirHtml = new File(options.reportPath + "/html") - val reportDirXml = new File(options.reportPath + "/xml") - - if (sourceDir.exists) { - if (options.cleanOld && reportDir.exists) { - logger.info(s"Nuking old report directory [$reportDir].") - FileUtils.deleteDirectory(reportDir) - } - - if (!reportDir.exists) { - logger.info(s"Creating HTML report directory [$reportDirHtml]") - reportDirHtml.mkdirs - logger.info(s"Creating XML report directory [$reportDirXml]") - reportDirXml.mkdirs - } - - if (options.writeHtml) { - logger.info(s"Writing HTML scoverage reports to [$reportDirHtml]") - new ScoverageHtmlWriter(Seq(sourceDir), reportDirHtml, None).write(coverage) - } - - if (options.writeXml) { - logger.info(s"Writing XML scoverage reports to [$reportDirXml]") - new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, false).write(coverage) - if (options.writeXmlDebug) { - new ScoverageXmlWriter(Seq(sourceDir), reportDirXml, true).write(coverage) - } - } - } else { - logger.error(s"Source dir [$sourceDir] does not exist") - } - - logger.success(s"Statement coverage: ${coverage.statementCoverageFormatted}%") - logger.success(s"Branch coverage: ${coverage.branchCoverageFormatted}%") - } - - def main(args: Array[String]): Unit = { - OParser.parse(Settings.parser1, args, Settings()) match { - case Some(settings) => - val reportOptions = ReportOptions(settings) - - reportOptions.loadDataDir match { - case false => - loadAggregatedCoverage(reportOptions.measurementsPath, reportOptions) match { - case Some(cov) => - logger.success("Coverage loaded successfully.") - if (reportOptions.writeReports) { - writeReports(cov, reportOptions) - } - - case None => logger.error("Failed to load coverage.") - } - case true => - loadCoverage(reportOptions.dataPath) match { - case Some(cov) => - logger.success("Coverage loaded successfully!") - if (reportOptions.writeReports) { - writeReports(cov, reportOptions) - } - case _ => logger.error("Failed to load coverage") - } - } - - case None => logger.error("Incorrect options supplied") - } - } -} diff --git a/src/scala/org/pantsbuild/scoverage/report/Settings.scala b/src/scala/org/pantsbuild/scoverage/report/Settings.scala deleted file mode 100644 index ad22b842daa..00000000000 --- a/src/scala/org/pantsbuild/scoverage/report/Settings.scala +++ /dev/null @@ -1,73 +0,0 @@ -package org.pantsbuild.scoverage.report - -import scopt.OParser - -/** - * All parsed command-line options. - */ -case class Settings( - loadDataDir: Boolean = false, - measurementsDirPath: String = "", - reportDirPath: String = "", - sourceDirPath: String = ".", - dataDirPath: String = "", - writeHtmlReport: Boolean = true, - writeXmlReport: Boolean = true, - writeXmlDebug: Boolean = false, - cleanOldReports: Boolean = true, - targetFilters: Seq[String] = Seq()) - -object Settings { - - val builder = OParser.builder[Settings] - val parser1 = { - import builder._ - OParser.sequence( - programName("scoverageReportGenerator"), - head("scoverageReportGenerator"), - - help('h', "help") - .text("Print this usage message."), - - opt[Unit]("loadDataDir") - .action((_, s: Settings) => s.copy(loadDataDir = true)) - .text("Load a single measurements directory instead of aggregating coverage reports. Must pass in `dataDirPath `"), - - opt[String]("measurementsDirPath") - .action((dir: String, s: Settings) => s.copy(measurementsDirPath = dir)) - .text("Directory where all scoverage measurements data is stored."), - - opt[String]("reportDirPath") - .action((dir: String, s: Settings) => s.copy(reportDirPath = dir)) - .text("Target output directory to place the reports."), - - opt[String]("sourceDirPath") - .action((dir: String, s: Settings) => s.copy(sourceDirPath = dir)) - .text("Directory containing the project sources."), - - opt[String]("dataDirPath") - .action((dir: String, s: Settings) => s.copy(dataDirPath = dir)) - .text("Scoverage data file directory to be used in case report needed for single measurements " + - "directory. Must set `loadDataDir` to use this options."), - - opt[Unit]("writeHtmlReport") - .action((_, s: Settings) => s.copy(writeHtmlReport = true)) - .text("Write the HTML version of the coverage report."), - - opt[Unit]("writeXmlReport") - .action((_, s: Settings) => s.copy(writeXmlReport = true)) - .text("Write the XML version of the coverage report."), - - opt[Unit]("writeXmlDebug") - .action((_, s: Settings) => s.copy(writeXmlDebug = true)) - .text("Write debug information to the XML version of the coverage report."), - - opt[Unit]("cleanOldReports") - .action((_, s: Settings) => s.copy(cleanOldReports = true)) - .text("Delete any existing reports directory prior to writing reports."), - - opt[Seq[String]]("targetFilters") - .action((f: Seq[String], s: Settings) => s.copy(targetFilters = f)) - .text("Directory names for which report has to be generated.")) - } -} From 66ace610f22f51262115cd3e911eb7d829d8ed74 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 14:57:46 -0700 Subject: [PATCH 19/35] resolving unwanted file changes --- 3rdparty/jvm/com/github/scopt/BUILD | 2 +- 3rdparty/jvm/com/twitter/BUILD | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/3rdparty/jvm/com/github/scopt/BUILD b/3rdparty/jvm/com/github/scopt/BUILD index 506e2064f3d..23730e2b9c9 100644 --- a/3rdparty/jvm/com/github/scopt/BUILD +++ b/3rdparty/jvm/com/github/scopt/BUILD @@ -3,7 +3,7 @@ jar_library( jars=[ - scala_jar(org='com.github.scopt', name='scopt', rev='4.0.0-RC2'), + scala_jar(org='com.github.scopt', name='scopt', rev='3.7.0'), ], # Is used as a command line parser library ) diff --git a/3rdparty/jvm/com/twitter/BUILD b/3rdparty/jvm/com/twitter/BUILD index cc0e0a6d0e0..095e12a7ae7 100644 --- a/3rdparty/jvm/com/twitter/BUILD +++ b/3rdparty/jvm/com/twitter/BUILD @@ -14,9 +14,3 @@ jar_library(name='scrooge-core', scala_jar(org='com.twitter', name='scrooge-core', rev=SCROOGE_REV), ], ) - -jar_library(name='scalac-scoverage-plugin', - jars=[ - scala_jar(org='com.twitter.scoverage', name='scalac-scoverage-plugin', rev='1.0.1-twitter'), - ], - ) From a47ef75639d6eac667f4ee7d4338fafb53b6c36b Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 15:06:31 -0700 Subject: [PATCH 20/35] minor changes to alignment --- src/python/pants/backend/jvm/subsystems/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/jvm/subsystems/BUILD b/src/python/pants/backend/jvm/subsystems/BUILD index 02439a53435..5a4bd937ad6 100644 --- a/src/python/pants/backend/jvm/subsystems/BUILD +++ b/src/python/pants/backend/jvm/subsystems/BUILD @@ -117,7 +117,7 @@ python_library( dependencies = [ 'src/python/pants/option', 'src/python/pants/subsystem', - 'src/python/pants/java/jar', + 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', ], ) From 7bacd7b977cb2b8d342b01c4975a5f64c65f455e Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 15:26:22 -0700 Subject: [PATCH 21/35] minor changes to alignment --- 3rdparty/jvm/com/twitter/BUILD | 1 + BUILD.tools | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/jvm/com/twitter/BUILD b/3rdparty/jvm/com/twitter/BUILD index 095e12a7ae7..c4858d5ee12 100644 --- a/3rdparty/jvm/com/twitter/BUILD +++ b/3rdparty/jvm/com/twitter/BUILD @@ -14,3 +14,4 @@ jar_library(name='scrooge-core', scala_jar(org='com.twitter', name='scrooge-core', rev=SCROOGE_REV), ], ) + diff --git a/BUILD.tools b/BUILD.tools index 1e0acbce5de..1e240dd0602 100644 --- a/BUILD.tools +++ b/BUILD.tools @@ -27,7 +27,6 @@ jar_library(name = 'scrooge-linter', jar(org='com.twitter', name='scrooge-linter_2.11', rev=SCROOGE_REV) ]) - # Google doesn't publish Kythe jars (yet?). So we publish them to a custom repo # (https://github.com/toolchainlabs/binhost) for now. See build-support/ivy/ivysettings.xml # for more info. From d9a94e174848861b462b3d4e08d4141375ba9f55 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 15:27:01 -0700 Subject: [PATCH 22/35] minor changes to alignment --- 3rdparty/jvm/com/twitter/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/3rdparty/jvm/com/twitter/BUILD b/3rdparty/jvm/com/twitter/BUILD index c4858d5ee12..00b766b6d02 100644 --- a/3rdparty/jvm/com/twitter/BUILD +++ b/3rdparty/jvm/com/twitter/BUILD @@ -15,3 +15,4 @@ jar_library(name='scrooge-core', ], ) + From 37295d609df65943e71e6e8b53d48607af2c9828 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 20:35:13 -0700 Subject: [PATCH 23/35] adding log statements --- .../pants/backend/jvm/subsystems/scoverage_platform.py | 7 +++++-- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index cb54eddd2aa..67cc7357212 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -1,6 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging import os from pants.build_graph.injectables_mixin import InjectablesMixin from pants.subsystem.subsystem import Subsystem @@ -8,6 +9,7 @@ from pants.backend.jvm.targets.jar_library import JarLibrary from pants.build_graph.address import Address +logger = logging.getLogger(__name__) SCOVERAGE = "scoverage" @@ -39,7 +41,8 @@ def __init__(self, *args, **kwargs): super(ScoveragePlatform, self).__init__(*args, **kwargs) # Setting up the scoverage blacklist files which contains targets - # not to be instrumented. + # not to be instrumented. Since the file is not expected to be really big, + # would it be ok to store it in memory? if (self.get_options().blacklist_file and os.path.exists(self.get_options().blacklist_file)): self._blacklist_file_contents = open(self.get_options().blacklist_file).read() @@ -81,12 +84,12 @@ def is_blacklisted(self, target) -> bool: """ Checks if the [target] is blacklisted or not. """ - # File not specified if not self._blacklist_file_contents: return False if target.address.spec in self._blacklist_file_contents: + logger.warning(f"{target.address.spec} found in blacklist, not instrumented.") return True else: return False diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 461f2e41b5d..b33c205ad39 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -4,7 +4,6 @@ import functools import os -import subprocess import re from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin @@ -177,6 +176,11 @@ def report(self, output_dir, execution_failed_exception=None): if result != 0: raise TaskError(f"java {main} ... exited non-zero ({result}) - failed to scoverage-report-generator") + self._settings.log.info(f"Scoverage html reports available at {output_dir}/scoverage/reports/html") + self._settings.log.info(f"Scoverage xml reports available at {output_dir}/scoverage/reports/xml") + if self._settings.coverage_open: + return os.path.join(output_dir, 'scoverage', 'reports', 'html', 'index.html') + # Returns the directories under [measurements_dir] which need to # be passed to the report generator. From f390ee492ba5d1013177a4f9a6da20e78903f5f9 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Wed, 24 Jul 2019 20:54:20 -0700 Subject: [PATCH 24/35] improving test file for blacklisted targets --- .../jvm/subsystems/test_scoverage_platform.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index ccc678d2ed5..203b98c2870 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -17,7 +17,14 @@ class ScoveragePlatformTest(TestBase): scoverage_path = '//:scoverage' blacklist_file_path = 'my/file/new_blacklist_scoverage_test' - def setup_scoverage_platform(self): + def setup_scoverage_platform(self, create_file=False): + tmp = self.create_file( + relpath=self.blacklist_file_path, + contents=dedent(""" + a/scala:blacked + """) + ) + options = { ScalaPlatform.options_scope: { 'version': 'custom', @@ -31,8 +38,18 @@ def setup_scoverage_platform(self): } } + options3 = { + ScoveragePlatform.options_scope: { + 'enable_scoverage' : 'False', + 'blacklist_file' : f"{tmp}" + } + } + init_subsystem(ScalaPlatform, options) - init_subsystem(ScoveragePlatform, options2) + if not create_file: + init_subsystem(ScoveragePlatform, options2) + else: + init_subsystem(ScoveragePlatform, options3) self.make_target('//:scalastyle', JarLibrary, @@ -135,17 +152,9 @@ def test_blacklist(self): instrumented as long as `scalac_plugins` do not contain `scoverage`. :return: """ - self.setup_scoverage_platform() + self.setup_scoverage_platform(create_file=True) ScoveragePlatform.global_instance().get_options().enable_scoverage = True - tmp = self.create_file( - relpath=self.blacklist_file_path, - contents=dedent(""" - a/scala:blacked - """) - ) - ScoveragePlatform.global_instance().get_options().blacklist_file = tmp - self.create_file( relpath='a/scala/pass.scala', contents=dedent(""" From b15f45d3a71780a5f127caad10ae3c2fa4424954 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 25 Jul 2019 09:42:25 -0700 Subject: [PATCH 25/35] adding scoverage dependency to test_rsc_compile to resolve unit test error --- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 1 - .../backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index b33c205ad39..2aa2e3e970c 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -1,4 +1,3 @@ -# coding=utf-8 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py index 83c843b50be..75623f49203 100644 --- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py +++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py @@ -6,6 +6,7 @@ from pants.backend.jvm.subsystems.junit import JUnit from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.java_library import JavaLibrary from pants.backend.jvm.targets.junit_tests import JUnitTests @@ -294,6 +295,7 @@ def init_dependencies_for_scala_libraries(self): init_subsystem( JUnit, ) + init_subsystem(ScoveragePlatform) self.make_target( '//:scala-library', target_type=JarLibrary, From ae22fed4ade5f87fe42a72082093cdef3ad5853c Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 25 Jul 2019 16:16:44 -0700 Subject: [PATCH 26/35] adding extra args in constructor rather than as a property --- .../jvm/subsystems/scoverage_platform.py | 41 +++---- .../backend/jvm/targets/scala_library.py | 104 ++++++++---------- .../backend/jvm/tasks/coverage/manager.py | 9 +- .../backend/jvm/tasks/coverage/scoverage.py | 32 +++--- 4 files changed, 81 insertions(+), 105 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 67cc7357212..d17c8a851f3 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -2,12 +2,13 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -import os +import re from pants.build_graph.injectables_mixin import InjectablesMixin from pants.subsystem.subsystem import Subsystem from pants.java.jar.jar_dependency import JarDependency from pants.backend.jvm.targets.jar_library import JarLibrary from pants.build_graph.address import Address +from pants.option.custom_types import target_option logger = logging.getLogger(__name__) @@ -24,30 +25,22 @@ def register_options(cls, register): register('--enable-scoverage', default=False, type=bool, - help='Specifies whether to generate scoverage reports for scala test targets.' - 'Default value is False. If True,' + help='Specifies whether to generate scoverage reports for scala test targets. ' + 'Default value is False. If True, ' 'implies --test-junit-coverage-processor=scoverage.') - register('--blacklist-file', - type=str, - help='Path to files containing targets not to be instrumented.') + register('--blacklist-targets', + type=list, + member_type=target_option, + help='List of targets not to be instrumented. Accepts Regex patterns. All ' + 'targets matching any of the patterns will not be instrumented. If no targets ' + 'are specified, all targets will be instrumented.') register('--scoverage-target-path', default='//:scoverage', type=str, help='Path to the scoverage dependency.') - def __init__(self, *args, **kwargs): - super(ScoveragePlatform, self).__init__(*args, **kwargs) - - # Setting up the scoverage blacklist files which contains targets - # not to be instrumented. Since the file is not expected to be really big, - # would it be ok to store it in memory? - if (self.get_options().blacklist_file and - os.path.exists(self.get_options().blacklist_file)): - self._blacklist_file_contents = open(self.get_options().blacklist_file).read() - else: - self._blacklist_file_contents = None def scoverage_jar(self): return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', @@ -84,12 +77,12 @@ def is_blacklisted(self, target) -> bool: """ Checks if the [target] is blacklisted or not. """ - # File not specified - if not self._blacklist_file_contents: + # No blacklisted targets specified. + if not self.get_options().blacklist_targets: return False - if target.address.spec in self._blacklist_file_contents: - logger.warning(f"{target.address.spec} found in blacklist, not instrumented.") - return True - else: - return False + for filter in self.get_options().blacklist_targets: + if re.search(filter, target.address.spec) is not None: + logger.info(f"{target.address.spec} found in blacklist, not instrumented.") + return True + return False diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index 5cbb5a3fac6..6c78bb07097 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -9,9 +9,11 @@ from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField from pants.build_graph.address import Address +from pants.build_graph.target import Target SCOVERAGE = "scoverage" + class ScalaLibrary(ExportableJvmLibrary): """A Scala library. @@ -27,13 +29,12 @@ class ScalaLibrary(ExportableJvmLibrary): default_sources_globs = '*.scala' default_sources_exclude_globs = JUnitTests.scala_test_globs - - @classmethod def subsystems(cls): return super().subsystems() + (ScalaPlatform, ScoveragePlatform,) - def __init__(self, java_sources=None, payload=None, **kwargs): + def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=None, + compiler_option_sets=None, payload=None, **kwargs): """ :param java_sources: Java libraries this library has a *circular* dependency on. @@ -49,7 +50,46 @@ def __init__(self, java_sources=None, payload=None, **kwargs): payload.add_fields({ 'java_sources': PrimitiveField(self.assert_list(java_sources, key_arg='java_sources')), }) - super().__init__(payload=payload, **kwargs) + + if ScoveragePlatform.global_instance().get_options().enable_scoverage: + # Settings scalac_plugins + if not (Target.compute_target_id(kwargs['address']).startswith(".pants.d.gen") or + ScoveragePlatform.global_instance().is_blacklisted(self)): + if scalac_plugins: + scalac_plugins.append(SCOVERAGE) + else: + scalac_plugins = [SCOVERAGE] + + # Setting scalac_plugin_args + if scalac_plugin_args: + scalac_plugin_args.update( + { + "scoverage": ["writeToClasspath:true", + f"dataDir:{Target.compute_target_id(kwargs['address'])}"] + }) + else: + scalac_plugin_args = { + "scoverage": ["writeToClasspath:true", + f"dataDir:{Target.compute_target_id(kwargs['address'])}"] + } + + # Setting compiler_option_sets + if compiler_option_sets: + list(compiler_option_sets).append(SCOVERAGE) + else: + compiler_option_sets = [SCOVERAGE] + + super().__init__(payload=payload, + scalac_plugins=scalac_plugins, + scalac_plugin_args=scalac_plugin_args, + compiler_option_sets=tuple(compiler_option_sets), + **kwargs) + else: + super().__init__(payload=payload, + scalac_plugins=scalac_plugins, + scalac_plugin_args=scalac_plugin_args, + compiler_option_sets=compiler_option_sets, + **kwargs) self._scoverage_instance = ScoveragePlatform.global_instance() @@ -92,59 +132,3 @@ def java_sources(self): raise TargetDefinitionException(self, 'No such java target: {}'.format(spec)) targets.append(target) return targets - - @property - def scalac_plugins(self): - """The names of compiler plugins to use when compiling this target with scalac. - :return: See constructor. - :rtype: list of strings. - """ - if self._scoverage_instance.get_options().enable_scoverage: - # Prevent instrumenting generated targets and targets in blacklist. - if self.identifier.startswith(".pants.d.gen") or self._scoverage_instance.is_blacklisted(self): - return self.payload.scalac_plugins - - scalac_plugins = self.payload.scalac_plugins - if scalac_plugins: - scalac_plugins.append(SCOVERAGE) - else: - scalac_plugins = [SCOVERAGE] - return scalac_plugins - - return self.payload.scalac_plugins - - @property - def scalac_plugin_args(self): - """Map from scalac plugin name to list of args for that plugin. - :return: See constructor. - :rtype: map from string to list of strings. - """ - if self._scoverage_instance.get_options().enable_scoverage: - scalac_plugin_args = self.payload.scalac_plugin_args - if scalac_plugin_args: - scalac_plugin_args.update( - {"scoverage": ["writeToClasspath:true", f"dataDir:{self.identifier}"]}) - else: - scalac_plugin_args = { - "scoverage": ["writeToClasspath:true", f"dataDir:{self.identifier}"] - } - return scalac_plugin_args - - return self.payload.scalac_plugin_args - - @property - def compiler_option_sets(self): - """For every element in this list, enable the corresponding flags on compilation - of targets. - :return: See constructor. - :rtype: list - """ - if self._scoverage_instance.get_options().enable_scoverage: - compiler_option_sets = self.payload.compiler_option_sets - if compiler_option_sets: - list(compiler_option_sets).append(SCOVERAGE) - else: - compiler_option_sets = [SCOVERAGE] - return tuple(compiler_option_sets) - - return self.payload.compiler_option_sets diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index 1d6ee6037c7..b10e309d7ca 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -104,11 +104,10 @@ def get_coverage_engine(self, task, output_dir, all_targets, execute_java): "scoverage (by passing --scoverage-enable-scoverage option)") if enable_scoverage: - if processor is None: - logger.info("Scoverage is enabled. Setting coverage engine to scoverage.") - elif processor != 'scoverage': - logger.warning(f"Scoverage is enabled. Cannot use {processor} as the engine. Setting " - f"coverage engine to scoverage.") + if processor not in (None, 'scoverage'): + raise self.InvalidCoverageEngine(f"Scoverage is enabled. " + f"Cannot use {processor} as the engine. Set engine to scoverage " + f"(--test-junit-coverage-processor=scoverage)") processor = 'scoverage' if options.coverage or processor or options.is_flagged('coverage_open'): diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 2aa2e3e970c..c21dcecdac7 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -35,12 +35,6 @@ def scoverage_jar(name, **kwargs): def slf4j_jar(name): return JarDependency(org='org.slf4j', name=name, rev='1.7.5') - def scopt_jar(): - return JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0') - - def commons_jar(): - return JarDependency(org='commons-io', name='commons-io', rev='2.5') - def scoverage_report_jar(**kwargs): return JarDependency(org='org.pantsbuild', name='scoverage-report-generator_2.12', rev='0.0.1', **kwargs) @@ -51,8 +45,8 @@ def scoverage_report_jar(**kwargs): 'scoverage-report', classpath=[ scoverage_report_jar(), - commons_jar(), - scopt_jar(), + JarDependency(org='commons-io', name='commons-io', rev='2.5'), + JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), slf4j_jar('slf4j-simple'), slf4j_jar('slf4j-api'), scoverage_jar('scalac-scoverage-plugin_2.12') ] @@ -106,21 +100,27 @@ def __init__(self, report_path, target_filters, settings, targets, execute_java_ self._report_path = report_path - # All scoverage instrument files have the name "scoverage.coverage" and - # all measurement files are called "scoverage.measurements.". - # This function is used in [indtrument(output_dir)] function below to clean up - # all pre-existing scoverage files before generating new ones. + # def _iter_datafiles(self, output_dir): + """ + All scoverage instrument files have the name "scoverage.coverage" and + all measurement files are called "scoverage.measurements.". + This function is used in [indtrument(output_dir)] function below to clean up + all pre-existing scoverage files before generating new ones. + """ for root, _, files in safe_walk(output_dir): for f in files: if f.startswith("scoverage"): yield os.path.join(root, f) - # Used below for target filtering. Returns the parent directories - # under which all the scoverage data (for all targets) is stored. - # Currently, since all scoverage data for a test target is stored under - # `scoverage/measurements`, path to `scoverage/measurements` is returned. + # def _iter_datadirs(self, output_dir): + """ + Used below for target filtering. Returns the parent directories + under which all the scoverage data (for all targets) is stored. + Currently, since all scoverage data for a test target is stored under + `scoverage/measurements`, path to `scoverage/measurements` is returned. + """ for root, dirs, _ in safe_walk(output_dir): for d in dirs: if d.startswith("measurements"): From e5b0a564b7ff35ffc398d8cca55f993c6f5f18c3 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 25 Jul 2019 16:22:23 -0700 Subject: [PATCH 27/35] removing target's scoverage instance --- src/python/pants/backend/jvm/targets/scala_library.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index 6c78bb07097..a389f843270 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -91,7 +91,6 @@ def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=No compiler_option_sets=compiler_option_sets, **kwargs) - self._scoverage_instance = ScoveragePlatform.global_instance() @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): From 12430dd3f38edba56dbfe546f010e957defedf7e Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 25 Jul 2019 17:47:05 -0700 Subject: [PATCH 28/35] fixing blacklisted target specification --- .../pants/backend/jvm/subsystems/scoverage_platform.py | 6 +++--- src/python/pants/backend/jvm/targets/scala_library.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index d17c8a851f3..34d5dd348a9 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -73,7 +73,7 @@ def injectables_spec_mapping(self): } - def is_blacklisted(self, target) -> bool: + def is_blacklisted(self, target_address_spec) -> bool: """ Checks if the [target] is blacklisted or not. """ @@ -82,7 +82,7 @@ def is_blacklisted(self, target) -> bool: return False for filter in self.get_options().blacklist_targets: - if re.search(filter, target.address.spec) is not None: - logger.info(f"{target.address.spec} found in blacklist, not instrumented.") + if re.search(filter, target_address_spec) is not None: + logger.info(f"{target_address_spec} found in blacklist, not instrumented.") return True return False diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index a389f843270..b61daa0ca68 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -54,7 +54,7 @@ def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=No if ScoveragePlatform.global_instance().get_options().enable_scoverage: # Settings scalac_plugins if not (Target.compute_target_id(kwargs['address']).startswith(".pants.d.gen") or - ScoveragePlatform.global_instance().is_blacklisted(self)): + ScoveragePlatform.global_instance().is_blacklisted(kwargs['address'].spec)): if scalac_plugins: scalac_plugins.append(SCOVERAGE) else: From 04065acc3f6f01e15ed74fc90119a33556215625 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Thu, 25 Jul 2019 23:29:27 -0700 Subject: [PATCH 29/35] cleaning checking of blacklisted targets; fixing scoverage test --- .../jvm/subsystems/scoverage_platform.py | 2 +- .../backend/jvm/targets/scala_library.py | 11 ++++++--- .../backend/jvm/tasks/coverage/manager.py | 3 +-- .../backend/jvm/tasks/coverage/scoverage.py | 8 +++---- .../jvm/subsystems/test_scoverage_platform.py | 23 ++++--------------- 5 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 34d5dd348a9..5b47f3c630a 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -83,6 +83,6 @@ def is_blacklisted(self, target_address_spec) -> bool: for filter in self.get_options().blacklist_targets: if re.search(filter, target_address_spec) is not None: - logger.info(f"{target_address_spec} found in blacklist, not instrumented.") + logger.debug(f"{target_address_spec} found in blacklist, not instrumented.") return True return False diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index b61daa0ca68..1c1d057bd7a 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -11,6 +11,7 @@ from pants.build_graph.address import Address from pants.build_graph.target import Target + SCOVERAGE = "scoverage" @@ -33,6 +34,11 @@ class ScalaLibrary(ExportableJvmLibrary): def subsystems(cls): return super().subsystems() + (ScalaPlatform, ScoveragePlatform,) + @staticmethod + def skip_instrumentation(**kwargs): + return (Target.compute_target_id(kwargs['address']).startswith(".pants.d.gen") or + ScoveragePlatform.global_instance().is_blacklisted(kwargs['address'].spec)) + def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=None, compiler_option_sets=None, payload=None, **kwargs): """ @@ -53,8 +59,8 @@ def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=No if ScoveragePlatform.global_instance().get_options().enable_scoverage: # Settings scalac_plugins - if not (Target.compute_target_id(kwargs['address']).startswith(".pants.d.gen") or - ScoveragePlatform.global_instance().is_blacklisted(kwargs['address'].spec)): + # Preventing instrumentation of generated targets or targets in Scoverage blacklist option. + if not self.skip_instrumentation(**kwargs): if scalac_plugins: scalac_plugins.append(SCOVERAGE) else: @@ -91,7 +97,6 @@ def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=No compiler_option_sets=compiler_option_sets, **kwargs) - @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): for spec in super().compute_injectable_specs(kwargs, payload): diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index b10e309d7ca..babda40f681 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -77,8 +77,7 @@ def register_junit_options(register, register_jvm_tool): # We need to fingerprint this even though it nominally UI-only affecting option since the # presence of this option alone can implicitly flag on `--coverage`. register('--coverage-open', type=bool, fingerprint=True, - help='Open the generated HTML coverage report in a browser. Implies --coverage ' - 'with cobertura as the engine.') + help='Open the generated HTML coverage report in a browser. Implies --coverage ') register('--coverage-jvm-options', advanced=True, type=list, fingerprint=True, help='JVM flags to be added when running the coverage processor. For example: ' diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index c21dcecdac7..be67bba488d 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -22,7 +22,7 @@ class Factory(Subsystem, JvmToolMixin): # Cannot have the same scope as ScoveragePlatform, i.e they # both cannot share the scope `scoverage`. - options_scope = 'scoverage-runtime' + options_scope = 'scoverage-report' @classmethod def register_options(cls, register): @@ -53,7 +53,7 @@ def scoverage_report_jar(**kwargs): ) register('--target-filters', type=list, default=[], - help='Regex patterns passed to scoverage, specifying which targets should be ' + help='Regex patterns passed to scoverage report generator, specifying which targets should be ' 'included in reports. All targets matching any of the patterns will be ' 'included when generating reports. If no targets are specified, all ' 'targets are included, which would be the same as specifying ".*" as a ' @@ -73,7 +73,7 @@ def create(self, settings, targets, execute_java_for_targets): """ report_path = self.tool_classpath_from_products(settings.context.products, 'scoverage-report', - scope='scoverage-runtime') + scope='scoverage-report') target_filters = Scoverage.Factory.global_instance().get_options().target_filters @@ -105,7 +105,7 @@ def _iter_datafiles(self, output_dir): """ All scoverage instrument files have the name "scoverage.coverage" and all measurement files are called "scoverage.measurements.". - This function is used in [indtrument(output_dir)] function below to clean up + This function is used in [instrument(output_dir)] function below to clean up all pre-existing scoverage files before generating new ones. """ for root, _, files in safe_walk(output_dir): diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 203b98c2870..8e986c79264 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -17,14 +17,7 @@ class ScoveragePlatformTest(TestBase): scoverage_path = '//:scoverage' blacklist_file_path = 'my/file/new_blacklist_scoverage_test' - def setup_scoverage_platform(self, create_file=False): - tmp = self.create_file( - relpath=self.blacklist_file_path, - contents=dedent(""" - a/scala:blacked - """) - ) - + def setup_scoverage_platform(self): options = { ScalaPlatform.options_scope: { 'version': 'custom', @@ -38,18 +31,9 @@ def setup_scoverage_platform(self, create_file=False): } } - options3 = { - ScoveragePlatform.options_scope: { - 'enable_scoverage' : 'False', - 'blacklist_file' : f"{tmp}" - } - } init_subsystem(ScalaPlatform, options) - if not create_file: - init_subsystem(ScoveragePlatform, options2) - else: - init_subsystem(ScoveragePlatform, options3) + init_subsystem(ScoveragePlatform, options2) self.make_target('//:scalastyle', JarLibrary, @@ -152,8 +136,9 @@ def test_blacklist(self): instrumented as long as `scalac_plugins` do not contain `scoverage`. :return: """ - self.setup_scoverage_platform(create_file=True) + self.setup_scoverage_platform() ScoveragePlatform.global_instance().get_options().enable_scoverage = True + ScoveragePlatform.global_instance().get_options().blacklist_targets = ["blacked"] self.create_file( relpath='a/scala/pass.scala', From e13edd8fe5de57da6318a1a6f4997f7de61f1b65 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 26 Jul 2019 09:38:49 -0700 Subject: [PATCH 30/35] removing option; removing target spec as an option --- .../jvm/subsystems/scoverage_platform.py | 8 ++------ .../backend/jvm/tasks/coverage/scoverage.py | 20 +++++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 5b47f3c630a..160d7e8c5af 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -36,11 +36,6 @@ def register_options(cls, register): 'targets matching any of the patterns will not be instrumented. If no targets ' 'are specified, all targets will be instrumented.') - register('--scoverage-target-path', - default='//:scoverage', - type=str, - help='Path to the scoverage dependency.') - def scoverage_jar(self): return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', @@ -69,7 +64,8 @@ def injectables(self, build_graph): @property def injectables_spec_mapping(self): return { - 'scoverage': [f"{self.get_options().scoverage_target_path}"], + # Target spec for scoverage plugin. + 'scoverage': [f"//:scoverage"], } diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index be67bba488d..a81c9bddb84 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -153,16 +153,19 @@ def report(self, output_dir, execution_failed_exception=None): main = 'org.pantsbuild.scoverage.report.ScoverageReport' scoverage_cp = self._report_path + html_report_path = os.path.join(output_dir, 'scoverage', 'reports', 'html') + xml_report_path = os.path.join(output_dir, 'scoverage', 'reports', 'xml') + safe_mkdir(html_report_path, clean=True) + safe_mkdir(xml_report_path, clean=True) final_target_dirs = [] for parent_measurements_dir in self._iter_datadirs(output_dir): final_target_dirs += self.filter_scoverage_targets(parent_measurements_dir) args = ["--measurementsDirPath",f"{output_dir}", - "--htmlDirPath", f"{output_dir}/scoverage/reports/html", - "--xmlDirPath", f"{output_dir}/scoverage/reports/xml", - "--targetFilters", f"{','.join(final_target_dirs)}", - "--cleanOldReports"] + "--htmlDirPath", f"{html_report_path}", + "--xmlDirPath", f"{xml_report_path}", + "--targetFilters", f"{','.join(final_target_dirs)}"] result = self._execute_java(classpath=scoverage_cp, @@ -175,14 +178,15 @@ def report(self, output_dir, execution_failed_exception=None): if result != 0: raise TaskError(f"java {main} ... exited non-zero ({result}) - failed to scoverage-report-generator") - self._settings.log.info(f"Scoverage html reports available at {output_dir}/scoverage/reports/html") - self._settings.log.info(f"Scoverage xml reports available at {output_dir}/scoverage/reports/xml") + self._settings.log.info(f"Scoverage html reports available at {html_report_path}") + self._settings.log.info(f"Scoverage xml reports available at {xml_report_path}") if self._settings.coverage_open: - return os.path.join(output_dir, 'scoverage', 'reports', 'html', 'index.html') + return os.path.join(html_report_path, 'index.html') # Returns the directories under [measurements_dir] which need to - # be passed to the report generator. + # be passed to the report generator. If no filter is specified, + # all the directories are returned. def filter_scoverage_targets(self, measurements_dir): return [d for d in os.listdir(measurements_dir) if self._include_dir(d)] From 4e1b5c8401cd78f69646808d7d9d7293cc9b6689 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 26 Jul 2019 12:39:48 -0700 Subject: [PATCH 31/35] init change in scala_library; linting --- .../jvm/subsystems/scoverage_platform.py | 11 ++-- .../backend/jvm/targets/scala_library.py | 45 ++++++----------- .../backend/jvm/tasks/coverage/manager.py | 50 ++++++++++--------- .../backend/jvm/tasks/coverage/scoverage.py | 48 ++++++++---------- .../jvm/subsystems/test_scoverage_platform.py | 36 +++++-------- 5 files changed, 80 insertions(+), 110 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py index 160d7e8c5af..a252d3b4e3a 100644 --- a/src/python/pants/backend/jvm/subsystems/scoverage_platform.py +++ b/src/python/pants/backend/jvm/subsystems/scoverage_platform.py @@ -3,17 +3,20 @@ import logging import re -from pants.build_graph.injectables_mixin import InjectablesMixin -from pants.subsystem.subsystem import Subsystem -from pants.java.jar.jar_dependency import JarDependency + from pants.backend.jvm.targets.jar_library import JarLibrary from pants.build_graph.address import Address +from pants.build_graph.injectables_mixin import InjectablesMixin +from pants.java.jar.jar_dependency import JarDependency from pants.option.custom_types import target_option +from pants.subsystem.subsystem import Subsystem + logger = logging.getLogger(__name__) SCOVERAGE = "scoverage" + class ScoveragePlatform(InjectablesMixin, Subsystem): """The scoverage platform.""" @@ -36,7 +39,6 @@ def register_options(cls, register): 'targets matching any of the patterns will not be instrumented. If no targets ' 'are specified, all targets will be instrumented.') - def scoverage_jar(self): return [JarDependency(org='com.twitter.scoverage', name='scalac-scoverage-plugin_2.12', rev='1.0.1-twitter'), @@ -68,7 +70,6 @@ def injectables_spec_mapping(self): 'scoverage': [f"//:scoverage"], } - def is_blacklisted(self, target_address_spec) -> bool: """ Checks if the [target] is blacklisted or not. diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index 1c1d057bd7a..da3fba47c3c 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -57,45 +57,28 @@ def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=No 'java_sources': PrimitiveField(self.assert_list(java_sources, key_arg='java_sources')), }) + scalac_plugins = scalac_plugins or [] + scalac_plugin_args = scalac_plugin_args or {} + compiler_option_sets = compiler_option_sets or set() + + # Modify these options in case scoverage is enabled. if ScoveragePlatform.global_instance().get_options().enable_scoverage: - # Settings scalac_plugins - # Preventing instrumentation of generated targets or targets in Scoverage blacklist option. if not self.skip_instrumentation(**kwargs): - if scalac_plugins: - scalac_plugins.append(SCOVERAGE) - else: - scalac_plugins = [SCOVERAGE] + scalac_plugins.append(SCOVERAGE) - # Setting scalac_plugin_args - if scalac_plugin_args: scalac_plugin_args.update( { "scoverage": ["writeToClasspath:true", f"dataDir:{Target.compute_target_id(kwargs['address'])}"] }) - else: - scalac_plugin_args = { - "scoverage": ["writeToClasspath:true", - f"dataDir:{Target.compute_target_id(kwargs['address'])}"] - } - - # Setting compiler_option_sets - if compiler_option_sets: - list(compiler_option_sets).append(SCOVERAGE) - else: - compiler_option_sets = [SCOVERAGE] - - super().__init__(payload=payload, - scalac_plugins=scalac_plugins, - scalac_plugin_args=scalac_plugin_args, - compiler_option_sets=tuple(compiler_option_sets), - **kwargs) - else: - super().__init__(payload=payload, - scalac_plugins=scalac_plugins, - scalac_plugin_args=scalac_plugin_args, - compiler_option_sets=compiler_option_sets, - **kwargs) + + compiler_option_sets.update([SCOVERAGE]) + + super().__init__(payload=payload, + scalac_plugins=scalac_plugins, + scalac_plugin_args=scalac_plugin_args, + compiler_option_sets=compiler_option_sets, + **kwargs) @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index babda40f681..845f28209eb 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -7,21 +7,23 @@ from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.tasks.coverage.cobertura import Cobertura -from pants.backend.jvm.tasks.coverage.scoverage import Scoverage from pants.backend.jvm.tasks.coverage.engine import NoCoverage from pants.backend.jvm.tasks.coverage.jacoco import Jacoco +from pants.backend.jvm.tasks.coverage.scoverage import Scoverage from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import safe_mkdir from pants.util.strutil import safe_shlex_split + logger = logging.getLogger(__name__) + class CodeCoverageSettings: """A class containing settings for code coverage tasks.""" def __init__(self, options, context, workdir, tool_classpath, confs, log, - copy2=shutil.copy2, copytree=shutil.copytree, is_file=os.path.isfile, - safe_md=safe_mkdir): + copy2=shutil.copy2, copytree=shutil.copytree, is_file=os.path.isfile, + safe_md=safe_mkdir): self.options = options self.context = context self.workdir = workdir @@ -47,11 +49,11 @@ def __init__(self, options, context, workdir, tool_classpath, confs, log, @classmethod def from_task(cls, task, workdir=None): return cls(options=task.get_options(), - context=task.context, - workdir=workdir or task.workdir, - tool_classpath=task.tool_classpath, - confs=task.confs, - log=task.context.log) + context=task.context, + workdir=workdir or task.workdir, + tool_classpath=task.tool_classpath, + confs=task.confs, + log=task.context.log) class CodeCoverage(Subsystem): @@ -68,31 +70,33 @@ def subsystem_dependencies(cls): def register_junit_options(register, register_jvm_tool): register('--coverage', type=bool, fingerprint=True, help='Collect code coverage data.') register('--coverage-processor', advanced=True, fingerprint=True, - choices=['cobertura', 'jacoco', 'scoverage'], default=None, - help="Which coverage processor to use if --coverage is enabled. If this option is " - "unset but coverage is enabled implicitly or explicitly, defaults to 'cobertura'. " - "If this option is explicitly set, implies --coverage. If this option is set to " - "scoverage, then first scoverage MUST be enabled by passing option " - "--scoverage-enable-scoverage.") + choices=['cobertura', 'jacoco', 'scoverage'], default=None, + help="Which coverage processor to use if --coverage is enabled. If this option is " + "unset but coverage is enabled implicitly or explicitly, defaults to 'cobertura'. " + "If this option is explicitly set, implies --coverage. If this option is set to " + "scoverage, then first scoverage MUST be enabled by passing option " + "--scoverage-enable-scoverage.") # We need to fingerprint this even though it nominally UI-only affecting option since the # presence of this option alone can implicitly flag on `--coverage`. register('--coverage-open', type=bool, fingerprint=True, - help='Open the generated HTML coverage report in a browser. Implies --coverage ') + help='Open the generated HTML coverage report in a browser. Implies --coverage ') register('--coverage-jvm-options', advanced=True, type=list, fingerprint=True, - help='JVM flags to be added when running the coverage processor. For example: ' - '{flag}=-Xmx4g {flag}=-Xms2g'.format(flag='--coverage-jvm-options')) + help='JVM flags to be added when running the coverage processor. For example: ' + '{flag}=-Xmx4g {flag}=-Xms2g'.format(flag='--coverage-jvm-options')) register('--coverage-force', advanced=True, type=bool, - help='Attempt to run the reporting phase of coverage even if tests failed ' - '(defaults to False, as otherwise the coverage results would be unreliable).') + help='Attempt to run the reporting phase of coverage even if tests failed ' + '(defaults to False, as otherwise the coverage results would be unreliable).') # register options for coverage engines # TODO(jtrobec): get rid of these calls when engines are dependent subsystems Cobertura.register_junit_options(register, register_jvm_tool) + class InvalidCoverageEngine(Exception): """Indicates an invalid coverage engine type was selected.""" + def get_coverage_engine(self, task, output_dir, all_targets, execute_java): options = task.get_options() enable_scoverage = ScoveragePlatform.global_instance().get_options().enable_scoverage @@ -104,9 +108,9 @@ def get_coverage_engine(self, task, output_dir, all_targets, execute_java): if enable_scoverage: if processor not in (None, 'scoverage'): - raise self.InvalidCoverageEngine(f"Scoverage is enabled. " - f"Cannot use {processor} as the engine. Set engine to scoverage " - f"(--test-junit-coverage-processor=scoverage)") + raise self.InvalidCoverageEngine(f"Scoverage is enabled. " + f"Cannot use {processor} as the engine. Set engine to scoverage " + f"(--test-junit-coverage-processor=scoverage)") processor = 'scoverage' if options.coverage or processor or options.is_flagged('coverage_open'): @@ -121,6 +125,6 @@ def get_coverage_engine(self, task, output_dir, all_targets, execute_java): # NB: We should never get here since the `--coverage-processor` is restricted by `choices`, # but for clarity. raise self.InvalidCoverageEngine('Unknown and unexpected coverage processor {!r}!' - .format(options.coverage_processor)) + .format(options.coverage_processor)) else: return NoCoverage() diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index a81c9bddb84..3aa2961cd3c 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -6,14 +6,13 @@ import re from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin -from pants.java.jar.jar_dependency import JarDependency from pants.backend.jvm.tasks.coverage.engine import CoverageEngine from pants.base.exceptions import TaskError +from pants.java.jar.jar_dependency import JarDependency from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import relativize_paths, safe_mkdir, safe_mkdir_for, safe_walk, touch - class Scoverage(CoverageEngine): """Class to run coverage tests with scoverage""" @@ -28,7 +27,6 @@ class Factory(Subsystem, JvmToolMixin): def register_options(cls, register): super(Scoverage.Factory, cls).register_options(register) - def scoverage_jar(name, **kwargs): return JarDependency(org='com.twitter.scoverage', name=name, rev='1.0.1-twitter', **kwargs) @@ -39,27 +37,26 @@ def scoverage_report_jar(**kwargs): return JarDependency(org='org.pantsbuild', name='scoverage-report-generator_2.12', rev='0.0.1', **kwargs) - # We need to inject report generator at runtime. cls.register_jvm_tool(register, 'scoverage-report', classpath=[ - scoverage_report_jar(), - JarDependency(org='commons-io', name='commons-io', rev='2.5'), - JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), - slf4j_jar('slf4j-simple'), slf4j_jar('slf4j-api'), - scoverage_jar('scalac-scoverage-plugin_2.12') + scoverage_report_jar(), + JarDependency(org='commons-io', name='commons-io', rev='2.5'), + JarDependency(org='com.github.scopt', name='scopt_2.12', rev='3.7.0'), + slf4j_jar('slf4j-simple'), slf4j_jar('slf4j-api'), + scoverage_jar('scalac-scoverage-plugin_2.12') ] ) register('--target-filters', type=list, default=[], - help='Regex patterns passed to scoverage report generator, specifying which targets should be ' + help='Regex patterns passed to scoverage report generator, specifying which targets ' + 'should be ' 'included in reports. All targets matching any of the patterns will be ' 'included when generating reports. If no targets are specified, all ' 'targets are included, which would be the same as specifying ".*" as a ' 'filter.') - def create(self, settings, targets, execute_java_for_targets): """ :param settings: Generic code coverage settings. @@ -73,7 +70,7 @@ def create(self, settings, targets, execute_java_for_targets): """ report_path = self.tool_classpath_from_products(settings.context.products, 'scoverage-report', - scope='scoverage-report') + scope='scoverage-report') target_filters = Scoverage.Factory.global_instance().get_options().target_filters @@ -99,7 +96,6 @@ def __init__(self, report_path, target_filters, settings, targets, execute_java_ self._coverage_force = settings.options.coverage_force self._report_path = report_path - # def _iter_datafiles(self, output_dir): """ @@ -127,13 +123,11 @@ def _iter_datadirs(self, output_dir): yield os.path.join(root, d) break - def instrument(self, output_dir): # Since scoverage does compile time instrumentation, we only need to clean-up existing runs. for datafile in self._iter_datafiles(output_dir): os.unlink(datafile) - def run_modifications(self, output_dir): measurement_dir = os.path.join(output_dir, "scoverage", "measurements") safe_mkdir(measurement_dir, clean=True) @@ -141,7 +135,6 @@ def run_modifications(self, output_dir): return self.RunModifications.create(extra_jvm_options=[data_dir_option]) - def report(self, output_dir, execution_failed_exception=None): if execution_failed_exception: self._settings.log.warn('Test failed: {}'.format(execution_failed_exception)) @@ -162,28 +155,27 @@ def report(self, output_dir, execution_failed_exception=None): for parent_measurements_dir in self._iter_datadirs(output_dir): final_target_dirs += self.filter_scoverage_targets(parent_measurements_dir) - args = ["--measurementsDirPath",f"{output_dir}", - "--htmlDirPath", f"{html_report_path}", - "--xmlDirPath", f"{xml_report_path}", - "--targetFilters", f"{','.join(final_target_dirs)}"] - + args = ["--measurementsDirPath", f"{output_dir}", + "--htmlDirPath", f"{html_report_path}", + "--xmlDirPath", f"{xml_report_path}", + "--targetFilters", f"{','.join(final_target_dirs)}"] result = self._execute_java(classpath=scoverage_cp, - main=main, - jvm_options=self._settings.coverage_jvm_options, - args=args, - workunit_factory=self._context.new_workunit, - workunit_name='scoverage-report-generator') + main=main, + jvm_options=self._settings.coverage_jvm_options, + args=args, + workunit_factory=self._context.new_workunit, + workunit_name='scoverage-report-generator') if result != 0: - raise TaskError(f"java {main} ... exited non-zero ({result}) - failed to scoverage-report-generator") + raise TaskError( + f"java {main} ... exited non-zero ({result}) - failed to scoverage-report-generator") self._settings.log.info(f"Scoverage html reports available at {html_report_path}") self._settings.log.info(f"Scoverage xml reports available at {xml_report_path}") if self._settings.coverage_open: return os.path.join(html_report_path, 'index.html') - # Returns the directories under [measurements_dir] which need to # be passed to the report generator. If no filter is specified, # all the directories are returned. diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 8e986c79264..1d0c3b58759 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -1,14 +1,14 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants_test.test_base import TestBase -from pants_test.subsystem.subsystem_util import init_subsystem -from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform + from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform -from pants.backend.jvm.targets.scala_library import ScalaLibrary +from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.backend.jvm.targets.scala_library import ScalaLibrary from pants.java.jar.jar_dependency import JarDependency - +from pants_test.subsystem.subsystem_util import init_subsystem +from pants_test.test_base import TestBase from textwrap import dedent @@ -27,11 +27,10 @@ def setup_scoverage_platform(self): options2 = { ScoveragePlatform.options_scope: { - 'enable_scoverage' : 'False' + 'enable_scoverage': 'False' } } - init_subsystem(ScalaPlatform, options) init_subsystem(ScoveragePlatform, options2) @@ -43,12 +42,12 @@ def setup_scoverage_platform(self): self.make_target('//:scala-repl', JarLibrary, jars=[ - JarDependency(org = 'org.scala-lang', - name = 'jline', - rev = '2.10.5'), - JarDependency(org = 'org.scala-lang', - name = 'scala-compiler', - rev = '2.10.5')]) + JarDependency(org='org.scala-lang', + name='jline', + rev='2.10.5'), + JarDependency(org='org.scala-lang', + name='scala-compiler', + rev='2.10.5')]) self.make_target('//:scalac', JarLibrary, @@ -58,8 +57,6 @@ def setup_scoverage_platform(self): JarLibrary, jars=[JarDependency('org.scala-lang', 'scala-library', '2.10.5')]) - - # ==========> TESTS <============= # ================================ def test_subsystem_defaults(self): @@ -68,7 +65,6 @@ def test_subsystem_defaults(self): subsystem = ScoveragePlatform.global_instance() self.assertEqual(False, subsystem.get_options().enable_scoverage) - self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) def test_subsystem_option_sets(self): init_subsystem(ScoveragePlatform) @@ -77,8 +73,6 @@ def test_subsystem_option_sets(self): subsystem = ScoveragePlatform.global_instance() self.assertEqual(True, subsystem.get_options().enable_scoverage) - self.assertEqual(self.scoverage_path, subsystem.get_options().scoverage_target_path) - def test_library_scoverage_enabled(self): self.setup_scoverage_platform() @@ -102,7 +96,6 @@ def main(args: Array[String]) { self.assertIn('//:scoverage', list(map(lambda t: t.address.spec, scala_target.dependencies))) self.assertIn('scoverage', list(scala_target.compiler_option_sets)) - def test_library_scoverage_disabled(self): self.setup_scoverage_platform() ScoveragePlatform.global_instance().get_options().enable_scoverage = False @@ -123,11 +116,10 @@ def main(args: Array[String]) { self.assertNotIn('scoverage', scala_target.scalac_plugins) if scala_target.scalac_plugin_args: self.assertNotIn('scoverage', scala_target.scalac_plugin_args) - self.assertNotIn('//:scoverage', list(map(lambda t: t.address.spec ,scala_target.dependencies))) + self.assertNotIn('//:scoverage', list(map(lambda t: t.address.spec, scala_target.dependencies))) if scala_target.compiler_option_sets: self.assertNotIn('scoverage', list(scala_target.compiler_option_sets)) - def test_blacklist(self): """ When a target is blacklisted, we do not instrument it. For achieving that, we only @@ -154,5 +146,3 @@ def main(args: Array[String]) { scala_target = self.make_target('a/scala:blacked', ScalaLibrary, sources=['pass.scala']) self.assertNotIn('scoverage', scala_target.scalac_plugins) - - From 02435ccaca98b46b06efc0c32a6b00eb2386088c Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 26 Jul 2019 14:47:03 -0700 Subject: [PATCH 32/35] fixing lint again --- .../backend/jvm/subsystems/test_scoverage_platform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 1d0c3b58759..6ee5f817fe4 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -1,6 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from textwrap import dedent from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform @@ -10,7 +11,7 @@ from pants_test.subsystem.subsystem_util import init_subsystem from pants_test.test_base import TestBase -from textwrap import dedent + class ScoveragePlatformTest(TestBase): From ed7d4b0d2795d543fe51997fd9c7a1d40fb017f3 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 26 Jul 2019 17:15:05 -0700 Subject: [PATCH 33/35] trying to fix lint in test_scoverage_platform --- src/python/pants/backend/jvm/tasks/coverage/manager.py | 1 - src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 3 +-- .../backend/jvm/subsystems/test_scoverage_platform.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/manager.py b/src/python/pants/backend/jvm/tasks/coverage/manager.py index 845f28209eb..4edb6b1c045 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/manager.py +++ b/src/python/pants/backend/jvm/tasks/coverage/manager.py @@ -96,7 +96,6 @@ def register_junit_options(register, register_jvm_tool): class InvalidCoverageEngine(Exception): """Indicates an invalid coverage engine type was selected.""" - def get_coverage_engine(self, task, output_dir, all_targets, execute_java): options = task.get_options() enable_scoverage = ScoveragePlatform.global_instance().get_options().enable_scoverage diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 3aa2961cd3c..0a027bbf11f 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -10,7 +10,7 @@ from pants.base.exceptions import TaskError from pants.java.jar.jar_dependency import JarDependency from pants.subsystem.subsystem import Subsystem -from pants.util.dirutil import relativize_paths, safe_mkdir, safe_mkdir_for, safe_walk, touch +from pants.util.dirutil import safe_mkdir, safe_walk class Scoverage(CoverageEngine): @@ -76,7 +76,6 @@ def create(self, settings, targets, execute_java_for_targets): return Scoverage(report_path, target_filters, settings, targets, execute_java_for_targets) - def __init__(self, report_path, target_filters, settings, targets, execute_java_for_targets): """ :param settings: Generic code coverage settings. diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 6ee5f817fe4..347ddc3f6b7 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -7,13 +7,13 @@ from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.scala_library import ScalaLibrary + from pants.java.jar.jar_dependency import JarDependency + from pants_test.subsystem.subsystem_util import init_subsystem from pants_test.test_base import TestBase - - class ScoveragePlatformTest(TestBase): scoverage_path = '//:scoverage' blacklist_file_path = 'my/file/new_blacklist_scoverage_test' From 7acd82cfda8a40425fe2f7b0dea89bdfc47209cb Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Fri, 26 Jul 2019 20:21:33 -0700 Subject: [PATCH 34/35] changing args indentation in scoverage.py --- src/python/pants/backend/jvm/tasks/coverage/scoverage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py index 0a027bbf11f..6ea52af1137 100644 --- a/src/python/pants/backend/jvm/tasks/coverage/scoverage.py +++ b/src/python/pants/backend/jvm/tasks/coverage/scoverage.py @@ -155,9 +155,9 @@ def report(self, output_dir, execution_failed_exception=None): final_target_dirs += self.filter_scoverage_targets(parent_measurements_dir) args = ["--measurementsDirPath", f"{output_dir}", - "--htmlDirPath", f"{html_report_path}", - "--xmlDirPath", f"{xml_report_path}", - "--targetFilters", f"{','.join(final_target_dirs)}"] + "--htmlDirPath", f"{html_report_path}", + "--xmlDirPath", f"{xml_report_path}", + "--targetFilters", f"{','.join(final_target_dirs)}"] result = self._execute_java(classpath=scoverage_cp, main=main, From d6a2dbc8bbff2918967c1aff6a3d378b23ff9366 Mon Sep 17 00:00:00 2001 From: Sameer Arora Date: Sat, 27 Jul 2019 09:59:18 -0700 Subject: [PATCH 35/35] running git hooks successfully --- .../backend/jvm/subsystems/test_scoverage_platform.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py index 347ddc3f6b7..5197ad9813c 100644 --- a/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py +++ b/tests/python/pants_test/backend/jvm/subsystems/test_scoverage_platform.py @@ -7,9 +7,7 @@ from pants.backend.jvm.subsystems.scoverage_platform import ScoveragePlatform from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.scala_library import ScalaLibrary - from pants.java.jar.jar_dependency import JarDependency - from pants_test.subsystem.subsystem_util import init_subsystem from pants_test.test_base import TestBase