From 5115f1032a1beb0a7ced268d4e7d0f04ea85273f Mon Sep 17 00:00:00 2001 From: Florian Dreier Date: Mon, 4 Nov 2024 11:03:34 +0100 Subject: [PATCH] TS-40875 Fix recursive detection of JAR files in artifactory --- CHANGELOG.md | 4 + agent/README.md | 6 +- .../GitMultiProjectPropertiesLocator.java | 13 ++- .../GitPropertiesLocatorUtils.java | 44 +++++---- .../GitSingleProjectPropertiesLocator.java | 19 ++-- .../jacoco/agent/options/AgentOptions.java | 36 ++++++-- .../agent/options/AgentOptionsParser.java | 86 ++++++++++++------ .../upload/artifactory/ArtifactoryConfig.java | 62 ++----------- .../artifactory/ArtifactoryUploader.java | 1 - .../GitMultiProjectPropertiesLocatorTest.java | 4 +- .../DelayedCommitDescriptorRetrievalTest.java | 4 +- .../src/main/resources/git.properties | 3 - .../main/resources/jar-with-properties.jar | Bin 0 -> 258 bytes 13 files changed, 151 insertions(+), 131 deletions(-) delete mode 100644 system-tests/artifactory-git-properties-detection/src/main/resources/git.properties create mode 100644 system-tests/artifactory-git-properties-detection/src/main/resources/jar-with-properties.jar diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d92dc469..fed1f8f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ We use [semantic versioning](http://semver.org/): - PATCH version when you make backwards compatible bug fixes. # Next version +- [fix] _agent_: `search-git-properties-recursively` was not considered when jar was given via `artifactory-git-properties-jar` +- [deprecation] _agent_: `artifactory-git-properties-jar` is deprecated now. Replace the option with `git-properties-jar`. The old option still works but is an alias now for `git-properties-jar`. +- [feature] _agent_: Added `git-properties-commit-date-format` (replaces `artifactory-git-properties-commit-date-format`), which now also can be used in non-artifactory cases. +- [fix] _agent_: `search-git-properties-recursively` did only consider nested jar files, but no `war`, `ear`, `aar` # 34.1.1 - [fix] _agent_: Loading a profiler configuration from Teamscale was not possible if the potentially necessary proxy settings were not set yet. diff --git a/agent/README.md b/agent/README.md index 22f396681..e02d56739 100644 --- a/agent/README.md +++ b/agent/README.md @@ -176,6 +176,7 @@ patterns with `*`, `**` and `?`. - the properties `git.branch` and `git.commit.time` (in the format `yyyy-MM-dd'T'HH:mm:ssZ` or `yyyy-MM-dd'T'HH:mm:ssXXX`) or - the properties `teamscale.commit.branch` and `teamscale.commit.time` (either as an epoch timestamp or in one of the two formats above) - `search-git-properties-recursively` Specifies whether to search for git.properties files recursively in folders or archive (jar, war, ear, aar) files. Default: true. +- `git-properties-commit-date-format` The Java data pattern `git.commit.time` is encoded with in `git.properties`. Defaults to `yyyy-MM-dd'T'HH:mm:ssZ`. - `teamscale-message` (optional): the commit message shown within Teamscale for the coverage upload (Default is "Agent coverage upload"). - `config-file` (optional): a file which contains one or more of the previously named options as `key=value` entries @@ -232,11 +233,6 @@ patterns with `*`, `**` and `?`. This can be used to encode e.g. a partition name that is parsed later on via Teamscale Artifactory connector options. - `artifactory-path-suffix` (optional): The path within the storage location between the default path and the uploaded artifact. -- `artifactory-git-properties-jar` (optional): Specify a Jar to search a `git.properties` file within. - If not specified, Git commit information is extracted from the first found `git.properties` file. - See `git-properties-jar` for details. -- `artifactory-git-properties-commit-date-format` (optional): - The Java data pattern `git.commit.time` is encoded with in `git.properties`. Defaults to `yyyy-MM-dd'T'HH:mm:ssZ`. ### The new standard upload schema ``` diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocator.java b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocator.java index 6236c916a..68097114e 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocator.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocator.java @@ -4,11 +4,13 @@ import com.teamscale.jacoco.agent.upload.teamscale.DelayedTeamscaleMultiProjectUploader; import com.teamscale.jacoco.agent.util.DaemonThreadFactory; import com.teamscale.jacoco.agent.util.LoggingUtils; +import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.VisibleForTesting; import org.slf4j.Logger; import java.io.File; import java.io.IOException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -27,19 +29,22 @@ public class GitMultiProjectPropertiesLocator implements IGitPropertiesLocator { private final boolean recursiveSearch; - public GitMultiProjectPropertiesLocator(DelayedTeamscaleMultiProjectUploader uploader, boolean recursiveSearch) { + private final @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat; + + public GitMultiProjectPropertiesLocator(DelayedTeamscaleMultiProjectUploader uploader, boolean recursiveSearch, @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) { // using a single threaded executor allows this class to be lock-free this(uploader, Executors .newSingleThreadExecutor( new DaemonThreadFactory(GitMultiProjectPropertiesLocator.class, - "git.properties Jar scanner thread")), recursiveSearch); + "git.properties Jar scanner thread")), recursiveSearch, gitPropertiesCommitTimeFormat); } public GitMultiProjectPropertiesLocator(DelayedTeamscaleMultiProjectUploader uploader, Executor executor, - boolean recursiveSearch) { + boolean recursiveSearch, @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) { this.uploader = uploader; this.executor = executor; this.recursiveSearch = recursiveSearch; + this.gitPropertiesCommitTimeFormat = gitPropertiesCommitTimeFormat; } /** @@ -62,7 +67,7 @@ void searchFile(File file, boolean isJarFile) { List projectAndCommits = GitPropertiesLocatorUtils.getProjectRevisionsFromGitProperties( file, isJarFile, - recursiveSearch); + recursiveSearch, gitPropertiesCommitTimeFormat); if (projectAndCommits.isEmpty()) { logger.debug("No git.properties file found in {}", file); return; diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitPropertiesLocatorUtils.java b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitPropertiesLocatorUtils.java index 8a3f02ac6..b0f0d6819 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitPropertiesLocatorUtils.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitPropertiesLocatorUtils.java @@ -7,8 +7,8 @@ import com.teamscale.report.util.BashFileSkippingInputStream; import org.conqat.lib.commons.collections.Pair; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -85,9 +85,6 @@ public class GitPropertiesLocatorUtils { */ private static final String GIT_PROPERTIES_DEFAULT_GRADLE_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; - /** File ending of Java archive packages */ - public static final String JAR_FILE_ENDING = ".jar"; - /** * Reads the git SHA1 and branch and timestamp from the given jar file's git.properties and builds a commit * descriptor out of it. If no git.properties file can be found, returns null. @@ -95,16 +92,23 @@ public class GitPropertiesLocatorUtils { * @throws IOException If reading the jar file fails. * @throws InvalidGitPropertiesException If a git.properties file is found but it is malformed. */ - public static List getCommitInfoFromGitProperties( - File file, boolean isJarFile, boolean recursiveSearch) throws IOException, InvalidGitPropertiesException { - List> entriesWithProperties = findGitPropertiesInFile(file, isJarFile, - recursiveSearch); + public static List getCommitInfoFromGitProperties(File file, boolean isJarFile, + boolean recursiveSearch, + @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) + throws IOException, InvalidGitPropertiesException { + List> entriesWithProperties = GitPropertiesLocatorUtils.findGitPropertiesInFile(file, + isJarFile, recursiveSearch); List result = new ArrayList<>(); + for (Pair entryWithProperties : entriesWithProperties) { - CommitInfo commitInfo = getCommitInfoFromGitProperties(entryWithProperties.getSecond(), - entryWithProperties.getFirst(), file, null); + String entry = entryWithProperties.getFirst(); + Properties properties = entryWithProperties.getSecond(); + + CommitInfo commitInfo = GitPropertiesLocatorUtils.getCommitInfoFromGitProperties(properties, entry, file, + gitPropertiesCommitTimeFormat); result.add(commitInfo); } + return result; } @@ -122,8 +126,7 @@ public static Pair extractGitPropertiesSearchRoot( switch (protocol) { case "file": File jarOrClassFolderFile = new File(jarOrClassFolderUrl.toURI()); - if (jarOrClassFolderFile.isDirectory() || org.conqat.lib.commons.string.StringUtils.endsWithOneOf( - jarOrClassFolderUrl.getPath().toLowerCase(), ".jar", ".war", ".ear", ".aar")) { + if (jarOrClassFolderFile.isDirectory() || isJarLikeFile(jarOrClassFolderUrl.getPath())) { return Pair.createPair(new File(jarOrClassFolderUrl.toURI()), !jarOrClassFolderFile.isDirectory()); } break; @@ -183,8 +186,7 @@ private static String extractArtefactUrl(URL jarOrClassFolderUrl) { String segment = pathSegments[segmentIdx]; artefactUrlBuilder.append(segment); artefactUrlBuilder.append("/"); - if (org.conqat.lib.commons.string.StringUtils.endsWithOneOf( - segment, ".jar", ".war", ".ear", ".aar")) { + if (isJarLikeFile(segment)) { break; } segmentIdx += 1; @@ -195,6 +197,11 @@ private static String extractArtefactUrl(URL jarOrClassFolderUrl) { return artefactUrlBuilder.toString(); } + private static boolean isJarLikeFile(String segment) { + return org.conqat.lib.commons.string.StringUtils.endsWithOneOf( + segment.toLowerCase(), ".jar", ".war", ".ear", ".aar"); + } + /** * Reads the 'teamscale.project' property values and the git SHA1s or branch + timestamp from all git.properties * files contained in the provided folder or archive file. @@ -203,13 +210,14 @@ private static String extractArtefactUrl(URL jarOrClassFolderUrl) { * @throws InvalidGitPropertiesException If a git.properties file is found but it is malformed. */ public static List getProjectRevisionsFromGitProperties( - File file, boolean isJarFile, boolean recursiveSearch) throws IOException, InvalidGitPropertiesException { + File file, boolean isJarFile, boolean recursiveSearch, + @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) throws IOException, InvalidGitPropertiesException { List> entriesWithProperties = findGitPropertiesInFile(file, isJarFile, recursiveSearch); List result = new ArrayList<>(); for (Pair entryWithProperties : entriesWithProperties) { CommitInfo commitInfo = getCommitInfoFromGitProperties(entryWithProperties.getSecond(), - entryWithProperties.getFirst(), file, null); + entryWithProperties.getFirst(), file, gitPropertiesCommitTimeFormat); String project = entryWithProperties.getSecond().getProperty(GIT_PROPERTIES_TEAMSCALE_PROJECT); if (commitInfo.isEmpty() && StringUtils.isEmpty(project)) { throw new InvalidGitPropertiesException( @@ -273,7 +281,7 @@ private static List> findGitPropertiesInNestedJarFiles( File directoryFile) throws IOException { List> result = new ArrayList<>(); List jarFiles = FileSystemUtils.listFilesRecursively(directoryFile, - file -> file.getName().endsWith(JAR_FILE_ENDING)); + file -> isJarLikeFile(file.getName())); for (File jarFile : jarFiles) { JarInputStream is = new JarInputStream(Files.newInputStream(jarFile.toPath())); String relativeFilePath = directoryFile.getName() + File.separator + directoryFile.toPath() @@ -323,7 +331,7 @@ static List> findGitPropertiesInArchive( Properties gitProperties = new Properties(); gitProperties.load(in); result.add(Pair.createPair(fullEntryName, gitProperties)); - } else if (entry.getName().endsWith(JAR_FILE_ENDING) && recursiveSearch) { + } else if (isJarLikeFile(entry.getName()) && recursiveSearch) { result.addAll(findGitPropertiesInArchive(new JarInputStream(in), fullEntryName, true)); } } diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitSingleProjectPropertiesLocator.java b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitSingleProjectPropertiesLocator.java index a0ae1c43c..096d63aa4 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitSingleProjectPropertiesLocator.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitSingleProjectPropertiesLocator.java @@ -3,10 +3,12 @@ import com.teamscale.jacoco.agent.upload.delay.DelayedUploader; import com.teamscale.jacoco.agent.util.DaemonThreadFactory; import com.teamscale.jacoco.agent.util.LoggingUtils; +import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import java.io.File; import java.io.IOException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -26,15 +28,17 @@ public class GitSingleProjectPropertiesLocator implements IGitPropertiesLocat private final DataExtractor dataExtractor; private final boolean recursiveSearch; + private final @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat; public GitSingleProjectPropertiesLocator(DelayedUploader uploader, DataExtractor dataExtractor, - boolean recursiveSearch) { + boolean recursiveSearch, + @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) { // using a single threaded executor allows this class to be lock-free this(uploader, dataExtractor, Executors .newSingleThreadExecutor( new DaemonThreadFactory(GitSingleProjectPropertiesLocator.class, "git.properties Jar scanner thread")), - recursiveSearch); + recursiveSearch, gitPropertiesCommitTimeFormat); } /** @@ -42,12 +46,14 @@ public GitSingleProjectPropertiesLocator(DelayedUploader uploader, DataExtrac * of this class. */ public GitSingleProjectPropertiesLocator(DelayedUploader uploader, DataExtractor dataExtractor, - Executor executor, - boolean recursiveSearch) { + Executor executor, + boolean recursiveSearch, + @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) { this.uploader = uploader; this.dataExtractor = dataExtractor; this.executor = executor; this.recursiveSearch = recursiveSearch; + this.gitPropertiesCommitTimeFormat = gitPropertiesCommitTimeFormat; } /** @@ -61,7 +67,7 @@ public void searchFileForGitPropertiesAsync(File file, boolean isJarFile) { private void searchFile(File file, boolean isJarFile) { logger.debug("Searching jar file {} for a single git.properties", file); try { - List data = dataExtractor.extractData(file, isJarFile, recursiveSearch); + List data = dataExtractor.extractData(file, isJarFile, recursiveSearch, gitPropertiesCommitTimeFormat); if (data.isEmpty()) { logger.debug("No git.properties files found in {}", file.toString()); return; @@ -103,6 +109,7 @@ private void searchFile(File file, boolean isJarFile) { public interface DataExtractor { /** Extracts data from the JAR. */ List extractData(File file, boolean isJarFile, - boolean recursiveSearch) throws IOException, InvalidGitPropertiesException; + boolean recursiveSearch, + @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) throws IOException, InvalidGitPropertiesException; } } diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptions.java b/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptions.java index a89e21a19..9a5598b63 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptions.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptions.java @@ -79,9 +79,19 @@ public class AgentOptions { /** See {@link AgentOptions#GIT_PROPERTIES_JAR_OPTION} */ /* package */ File gitPropertiesJar; + /** + * Related to {@link AgentOptions#GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION} + */ + public DateTimeFormatter gitPropertiesCommitTimeFormat = null; + /** Option name that allows to specify a jar file that contains the git commit hash in a git.properties file. */ public static final String GIT_PROPERTIES_JAR_OPTION = "git-properties-jar"; + /** + * Specifies the date format in which the commit timestamp in the git.properties file is formatted. + */ + public static final String GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION = "git-properties-commit-date-format"; + /** * The original options passed to the agent. */ @@ -455,7 +465,14 @@ public IUploader createUploader(Instrumentation instrumentation) throws Uploader } @NotNull - private IUploader createArtifactoryUploader(Instrumentation instrumentation) { + private IUploader createArtifactoryUploader(Instrumentation instrumentation) throws UploaderException { + if (gitPropertiesJar != null) { + logger.info("You did not provide a commit to upload to directly, so the Agent will try to" + + "auto-detect it by searching the provided " + GIT_PROPERTIES_JAR_OPTION + " at " + + gitPropertiesJar.getAbsolutePath() + " for a git.properties file."); + artifactoryConfig.commitInfo = ArtifactoryConfig.parseGitProperties(gitPropertiesJar, + this.searchGitPropertiesRecursively, this.gitPropertiesCommitTimeFormat); + } if (!artifactoryConfig.hasCommitInfo()) { logger.info("You did not provide a commit to upload to directly, so the Agent will try and" + " auto-detect it by searching all profiled Jar/War/Ear/... files for a git.properties file."); @@ -518,14 +535,16 @@ private DelayedTeamscaleMultiProjectUploader createTeamscaleMultiProjectUploader private void startGitPropertiesSearchInJarFile(DelayedUploader uploader, File gitPropertiesJar) { GitSingleProjectPropertiesLocator locator = new GitSingleProjectPropertiesLocator<>(uploader, - GitPropertiesLocatorUtils::getProjectRevisionsFromGitProperties, this.searchGitPropertiesRecursively); + GitPropertiesLocatorUtils::getProjectRevisionsFromGitProperties, this.searchGitPropertiesRecursively, + this.gitPropertiesCommitTimeFormat); locator.searchFileForGitPropertiesAsync(gitPropertiesJar, true); } private void registerSingleGitPropertiesLocator(DelayedUploader uploader, Instrumentation instrumentation) { GitSingleProjectPropertiesLocator locator = new GitSingleProjectPropertiesLocator<>(uploader, - GitPropertiesLocatorUtils::getProjectRevisionsFromGitProperties, this.searchGitPropertiesRecursively); + GitPropertiesLocatorUtils::getProjectRevisionsFromGitProperties, this.searchGitPropertiesRecursively, + this.gitPropertiesCommitTimeFormat); instrumentation.addTransformer(new GitPropertiesLocatingTransformer(locator, getLocationIncludeFilter())); } @@ -551,14 +570,14 @@ private DelayedUploader createDelayedSingleProjectTeamscaleUpl private void startMultiGitPropertiesFileSearchInJarFile(DelayedTeamscaleMultiProjectUploader uploader, File gitPropertiesJar) { GitMultiProjectPropertiesLocator locator = new GitMultiProjectPropertiesLocator(uploader, - this.searchGitPropertiesRecursively); + this.searchGitPropertiesRecursively, this.gitPropertiesCommitTimeFormat); locator.searchFileForGitPropertiesAsync(gitPropertiesJar, true); } private void registerMultiGitPropertiesLocator(DelayedTeamscaleMultiProjectUploader uploader, Instrumentation instrumentation) { GitMultiProjectPropertiesLocator locator = new GitMultiProjectPropertiesLocator(uploader, - this.searchGitPropertiesRecursively); + this.searchGitPropertiesRecursively, this.gitPropertiesCommitTimeFormat); instrumentation.addTransformer(new GitPropertiesLocatingTransformer(locator, getLocationIncludeFilter())); } @@ -571,9 +590,8 @@ private IUploader createDelayedArtifactoryUploader(Instrumentation instrumentati }, outputDirectory); GitSingleProjectPropertiesLocator locator = new GitSingleProjectPropertiesLocator<>( uploader, - (file, isJarFile, recursiveSearch) -> ArtifactoryConfig.parseGitProperties( - file, isJarFile, artifactoryConfig.gitPropertiesCommitTimeFormat, recursiveSearch), - this.searchGitPropertiesRecursively); + GitPropertiesLocatorUtils::getCommitInfoFromGitProperties, + this.searchGitPropertiesRecursively, this.gitPropertiesCommitTimeFormat); instrumentation.addTransformer(new GitPropertiesLocatingTransformer(locator, getLocationIncludeFilter())); return uploader; } @@ -731,7 +749,7 @@ public boolean shouldIgnoreUncoveredClasses() { /** @return the {@link TeamscaleProxyOptions} for the given protocol. */ public TeamscaleProxyOptions getTeamscaleProxyOptions(ProxySystemProperties.Protocol protocol) { - if(protocol == ProxySystemProperties.Protocol.HTTP) { + if (protocol == ProxySystemProperties.Protocol.HTTP) { return teamscaleProxyOptionsForHttp; } return teamscaleProxyOptionsForHttps; diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptionsParser.java b/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptionsParser.java index fa0de8279..96acfa971 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptionsParser.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/options/AgentOptionsParser.java @@ -29,9 +29,12 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; +import static com.teamscale.jacoco.agent.upload.artifactory.ArtifactoryConfig.ARTIFACTORY_GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION; +import static com.teamscale.jacoco.agent.upload.artifactory.ArtifactoryConfig.ARTIFACTORY_GIT_PROPERTIES_JAR_OPTION; import static java.util.stream.Collectors.joining; /** @@ -62,15 +65,15 @@ public class AgentOptionsParser { * @param environmentConfigFile The Profiler configuration file given via an environment variable. */ public static AgentOptions parse(String optionsString, String environmentConfigId, String environmentConfigFile, - TeamscaleCredentials credentials, - ILogger logger) throws AgentOptionParseException, AgentOptionReceiveException { + TeamscaleCredentials credentials, + ILogger logger) throws AgentOptionParseException, AgentOptionReceiveException { return new AgentOptionsParser(logger, environmentConfigId, environmentConfigFile, credentials).parse( optionsString); } @VisibleForTesting AgentOptionsParser(ILogger logger, String environmentConfigId, String environmentConfigFile, - TeamscaleCredentials credentials) { + TeamscaleCredentials credentials) { this.logger = logger; this.filePatternResolver = new FilePatternResolver(logger); this.teamscaleConfig = new TeamscaleConfig(logger, filePatternResolver); @@ -119,21 +122,24 @@ public static AgentOptions parse(String optionsString, String environmentConfigI } /** - * Stores the agent options for proxies in the {@link TeamscaleProxySystemProperties} and overwrites - * the password with the password found in the proxy-password-file if necessary. + * Stores the agent options for proxies in the {@link TeamscaleProxySystemProperties} and overwrites the password + * with the password found in the proxy-password-file if necessary. */ @VisibleForTesting public static void putTeamscaleProxyOptionsIntoSystemProperties(AgentOptions options) { - options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP).putTeamscaleProxyOptionsIntoSystemProperties(); - options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS).putTeamscaleProxyOptionsIntoSystemProperties(); + options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP) + .putTeamscaleProxyOptionsIntoSystemProperties(); + options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS) + .putTeamscaleProxyOptionsIntoSystemProperties(); } private void handleConfigId(AgentOptions options) throws AgentOptionReceiveException, AgentOptionParseException { if (environmentConfigId != null) { if (options.teamscaleServer.configId != null) { - logger.warn("You specified an ID for a profiler configuration in Teamscale both in the agent options and using an environment variable." + - " The environment variable will override the ID specified using the agent options." + - " Please use one or the other."); + logger.warn( + "You specified an ID for a profiler configuration in Teamscale both in the agent options and using an environment variable." + + " The environment variable will override the ID specified using the agent options." + + " Please use one or the other."); } handleOptionPart(options, "config-id=" + environmentConfigId); } @@ -156,7 +162,8 @@ private void handleConfigFile(AgentOptions options) throws AgentOptionParseExcep /** * Parses and stores the given option in the format key=value. */ - private void handleOptionPart(AgentOptions options, String optionPart) throws AgentOptionParseException, AgentOptionReceiveException { + private void handleOptionPart(AgentOptions options, + String optionPart) throws AgentOptionParseException, AgentOptionReceiveException { Pair keyAndValue = parseOption(optionPart); handleOption(options, keyAndValue.getFirst(), keyAndValue.getSecond()); } @@ -165,7 +172,7 @@ private void handleOptionPart(AgentOptions options, String optionPart) throws A * Parses and stores the option with the given key and value. */ private void handleOption(AgentOptions options, - String key, String value) throws AgentOptionParseException, AgentOptionReceiveException { + String key, String value) throws AgentOptionParseException, AgentOptionReceiveException { if (key.startsWith("debug")) { handleDebugOption(options, value); return; @@ -179,7 +186,7 @@ private void handleOption(AgentOptions options, return; } if (key.startsWith("artifactory-") && ArtifactoryConfig - .handleArtifactoryOptions(options.artifactoryConfig, filePatternResolver, key, value)) { + .handleArtifactoryOptions(options.artifactoryConfig, key, value)) { return; } if (key.startsWith("azure-") && AzureFileStorageConfig @@ -187,34 +194,40 @@ private void handleOption(AgentOptions options, value)) { return; } - if (key.startsWith("proxy-") && handleProxyOptions(options, StringUtils.stripPrefix(key, "proxy-"), value, filePatternResolver)){ - return; - } + if (key.startsWith("proxy-") && handleProxyOptions(options, StringUtils.stripPrefix(key, "proxy-"), value, + filePatternResolver)) { + return; + } if (handleAgentOptions(options, key, value)) { return; } throw new AgentOptionParseException("Unknown option: " + key); } - private boolean handleProxyOptions(AgentOptions options, String key, String value, FilePatternResolver filePatternResolver) throws AgentOptionParseException { + private boolean handleProxyOptions(AgentOptions options, String key, String value, + FilePatternResolver filePatternResolver) throws AgentOptionParseException { String httpsPrefix = ProxySystemProperties.Protocol.HTTPS + "-"; if (key.startsWith(httpsPrefix) - && options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS).handleTeamscaleProxyOptions(StringUtils.stripPrefix( - key, httpsPrefix), value)) { + && options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS) + .handleTeamscaleProxyOptions(StringUtils.stripPrefix( + key, httpsPrefix), value)) { return true; } String httpPrefix = ProxySystemProperties.Protocol.HTTP + "-"; if (key.startsWith(httpPrefix) - && options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP).handleTeamscaleProxyOptions(StringUtils.stripPrefix( - key, httpPrefix), value)) { + && options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP) + .handleTeamscaleProxyOptions(StringUtils.stripPrefix( + key, httpPrefix), value)) { return true; } - if(key.equals("password-file")) { + if (key.equals("password-file")) { Path proxyPasswordPath = filePatternResolver.parsePath(key, value); - options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS).setProxyPasswordPath(proxyPasswordPath); - options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP).setProxyPasswordPath(proxyPasswordPath); + options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTPS) + .setProxyPasswordPath(proxyPasswordPath); + options.getTeamscaleProxyOptions(ProxySystemProperties.Protocol.HTTP) + .setProxyPasswordPath(proxyPasswordPath); return true; } return false; @@ -296,9 +309,22 @@ private boolean handleAgentOptions(AgentOptions options, String key, String valu case "search-git-properties-recursively": options.searchGitPropertiesRecursively = Boolean.parseBoolean(value); return true; + case ARTIFACTORY_GIT_PROPERTIES_JAR_OPTION: + logger.warn( + "The option " + ARTIFACTORY_GIT_PROPERTIES_JAR_OPTION + " is deprecated. It still has an effect, " + + "but should be replaced with the equivalent option " + AgentOptions.GIT_PROPERTIES_JAR_OPTION + "."); + // intended fallthrough (acts as alias) case AgentOptions.GIT_PROPERTIES_JAR_OPTION: options.gitPropertiesJar = getGitPropertiesJarFile(value); return true; + case ARTIFACTORY_GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION: + logger.warn( + "The option " + ARTIFACTORY_GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION + " is deprecated. It still has an effect, " + + "but should be replaced with the equivalent option " + AgentOptions.GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION + "."); + // intended fallthrough (acts as alias) + case AgentOptions.GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION: + options.gitPropertiesCommitTimeFormat = DateTimeFormatter.ofPattern(value); + return true; case "mode": options.mode = parseEnumValue(key, value, EMode.class); return true; @@ -336,12 +362,14 @@ private void storeConfigId(AgentOptions options, String configId) throws AgentOp options.teamscaleServer.configId = configId; } - private void readConfigFromTeamscale(AgentOptions options) throws AgentOptionParseException, AgentOptionReceiveException { - if(options.teamscaleServer.configId == null) { + private void readConfigFromTeamscale( + AgentOptions options) throws AgentOptionParseException, AgentOptionReceiveException { + if (options.teamscaleServer.configId == null) { return; } - ConfigurationViaTeamscale configuration = ConfigurationViaTeamscale.retrieve(logger, options.teamscaleServer.configId, + ConfigurationViaTeamscale configuration = ConfigurationViaTeamscale.retrieve(logger, + options.teamscaleServer.configId, options.teamscaleServer.url, options.teamscaleServer.userName, options.teamscaleServer.userAccessToken); @@ -386,7 +414,7 @@ public static > T parseEnumValue(String key, String value, Cla * excludes=third.party.* */ private void readConfigFromFile(AgentOptions options, - File configFile) throws AgentOptionParseException, AgentOptionReceiveException { + File configFile) throws AgentOptionParseException, AgentOptionReceiveException { try { String content = FileSystemUtils.readFileUTF8(configFile); readConfigFromString(options, content); @@ -400,7 +428,7 @@ private void readConfigFromFile(AgentOptions options, } private void readConfigFromString(AgentOptions options, - String content) throws AgentOptionParseException, AgentOptionReceiveException { + String content) throws AgentOptionParseException, AgentOptionReceiveException { List configFileKeyValues = org.conqat.lib.commons.string.StringUtils.splitLinesAsList( content); for (String optionKeyValue : configFileKeyValues) { diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryConfig.java b/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryConfig.java index 2ac54d058..fceead42b 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryConfig.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryConfig.java @@ -6,16 +6,14 @@ import com.teamscale.jacoco.agent.commit_resolution.git_properties.InvalidGitPropertiesException; import com.teamscale.jacoco.agent.options.AgentOptionParseException; import com.teamscale.jacoco.agent.options.AgentOptionsParser; -import com.teamscale.jacoco.agent.options.FilePatternResolver; +import com.teamscale.jacoco.agent.upload.UploaderException; import okhttp3.HttpUrl; -import org.conqat.lib.commons.collections.Pair; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.List; -import java.util.Properties; /** Config necessary to upload files to an azure file storage. */ public class ArtifactoryConfig { @@ -98,11 +96,6 @@ public class ArtifactoryConfig { /** The information regarding a commit. */ public CommitInfo commitInfo; - /** - * Related to {@link ArtifactoryConfig#ARTIFACTORY_GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION} - */ - public DateTimeFormatter gitPropertiesCommitTimeFormat = null; - /** Related to {@link ArtifactoryConfig#ARTIFACTORY_API_KEY_OPTION} */ public String apiKey; @@ -114,8 +107,7 @@ public class ArtifactoryConfig { * * @return true if it has successfully processed the given option. */ - public static boolean handleArtifactoryOptions(ArtifactoryConfig options, FilePatternResolver filePatternResolver, - String key, String value) throws AgentOptionParseException { + public static boolean handleArtifactoryOptions(ArtifactoryConfig options, String key, String value) throws AgentOptionParseException { switch (key) { case ARTIFACTORY_URL_OPTION: options.url = AgentOptionsParser.parseUrl(key, value); @@ -135,13 +127,6 @@ public static boolean handleArtifactoryOptions(ArtifactoryConfig options, FilePa case ARTIFACTORY_PATH_SUFFIX: options.pathSuffix = StringUtils.stripSuffix(value, "/"); return true; - case ARTIFACTORY_GIT_PROPERTIES_JAR_OPTION: - options.commitInfo = ArtifactoryConfig.parseGitProperties(filePatternResolver, - options.gitPropertiesCommitTimeFormat, key, value); - return true; - case ARTIFACTORY_GIT_PROPERTIES_COMMIT_DATE_FORMAT_OPTION: - options.gitPropertiesCommitTimeFormat = DateTimeFormatter.ofPattern(value); - return true; case ARTIFACTORY_API_KEY_OPTION: options.apiKey = value; return true; @@ -171,49 +156,22 @@ public boolean hasCommitInfo() { } /** Parses the commit information form a git.properties file. */ - public static CommitInfo parseGitProperties(FilePatternResolver filePatternResolver, - DateTimeFormatter gitPropertiesCommitTimeFormat, String optionName, - String value) - throws AgentOptionParseException { - File jarFile = filePatternResolver.parsePath(optionName, value).toFile(); + public static CommitInfo parseGitProperties( + File jarFile, boolean searchRecursively, @Nullable DateTimeFormatter gitPropertiesCommitTimeFormat) + throws UploaderException { try { - // We can't be sure that the search-git-properties-recursively option is parsed - // already. - // Since we only support one git.properties file for artifactory anyway, - // recursive search is disabled here. - List commitInfo = parseGitProperties(jarFile, true, gitPropertiesCommitTimeFormat, false); + List commitInfo = GitPropertiesLocatorUtils.getCommitInfoFromGitProperties(jarFile, true, searchRecursively, gitPropertiesCommitTimeFormat); if (commitInfo.isEmpty()) { - throw new AgentOptionParseException("Found no git.properties files in " + jarFile); + throw new UploaderException("Found no git.properties files in " + jarFile); } if (commitInfo.size() > 1) { - throw new AgentOptionParseException("Found multiple git.properties files in " + jarFile + throw new UploaderException("Found multiple git.properties files in " + jarFile + ". Uploading to multiple projects is currently not possible with Artifactory. " + "Please contact CQSE if you need this feature."); } return commitInfo.get(0); } catch (IOException | InvalidGitPropertiesException e) { - throw new AgentOptionParseException("Could not locate a valid git.properties file in " + jarFile, e); + throw new UploaderException("Could not locate a valid git.properties file in " + jarFile, e); } } - - /** Parses the commit information from a git.properties file. */ - public static List parseGitProperties(File file, boolean isJarFile, - DateTimeFormatter gitPropertiesCommitTimeFormat, - boolean recursiveSearch) - throws IOException, InvalidGitPropertiesException { - List> entriesWithProperties = GitPropertiesLocatorUtils.findGitPropertiesInFile(file, - isJarFile, recursiveSearch); - List result = new ArrayList<>(); - - for (Pair entryWithProperties : entriesWithProperties) { - String entry = entryWithProperties.getFirst(); - Properties properties = entryWithProperties.getSecond(); - - CommitInfo commitInfo = GitPropertiesLocatorUtils.getCommitInfoFromGitProperties(properties, entry, file, - gitPropertiesCommitTimeFormat); - result.add(commitInfo); - } - - return result; - } } diff --git a/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryUploader.java b/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryUploader.java index f2b672f34..781c4f3c2 100644 --- a/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryUploader.java +++ b/agent/src/main/java/com/teamscale/jacoco/agent/upload/artifactory/ArtifactoryUploader.java @@ -78,7 +78,6 @@ public void reupload(CoverageFile coverageFile, Properties reuploadProperties) { String revision = reuploadProperties.getProperty(REVISION.name()); String commitString = reuploadProperties.getProperty(COMMIT.name()); config.commitInfo = new CommitInfo(revision, CommitDescriptor.parse(commitString)); - config.gitPropertiesCommitTimeFormat = artifactoryConfig.gitPropertiesCommitTimeFormat; config.apiKey = artifactoryConfig.apiKey; config.partition = Strings.emptyToNull(reuploadProperties.getProperty(PARTITION.name())); setUploadPath(coverageFile, config); diff --git a/agent/src/test/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocatorTest.java b/agent/src/test/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocatorTest.java index 271a47948..ec38ee8b6 100644 --- a/agent/src/test/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocatorTest.java +++ b/agent/src/test/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocatorTest.java @@ -23,7 +23,7 @@ void testNoErrorIsThrownWhenGitPropertiesFileDoesNotHaveAProject() { new DelayedTeamscaleMultiProjectUploader((project, revision) -> { projectAndCommits.add(new ProjectAndCommit(project, revision)); return new TeamscaleServer(); - }), true); + }), true, null); File jarFile = new File(getClass().getResource("emptyTeamscaleProjectGitProperties").getFile()); locator.searchFile(jarFile, false); assertThat(projectAndCommits.size()).isEqualTo(1); @@ -41,7 +41,7 @@ void testNoMultipleUploadsToSameProjectAndRevision() { return server; }); GitMultiProjectPropertiesLocator locator = new GitMultiProjectPropertiesLocator( - delayedTeamscaleMultiProjectUploader, true + delayedTeamscaleMultiProjectUploader, true, null ); File jarFile = new File(getClass().getResource("multiple-same-target-git-properties-folder").getFile()); locator.searchFile(jarFile, false); diff --git a/agent/src/test/java/com/teamscale/jacoco/agent/upload/delay/DelayedCommitDescriptorRetrievalTest.java b/agent/src/test/java/com/teamscale/jacoco/agent/upload/delay/DelayedCommitDescriptorRetrievalTest.java index 2320b82cf..9a6fce87b 100644 --- a/agent/src/test/java/com/teamscale/jacoco/agent/upload/delay/DelayedCommitDescriptorRetrievalTest.java +++ b/agent/src/test/java/com/teamscale/jacoco/agent/upload/delay/DelayedCommitDescriptorRetrievalTest.java @@ -33,7 +33,7 @@ public void locatorShouldTriggerUploadOfCachedXmls(@TempDir Path outputPath) thr ExecutorService locatorExecutor = Executors.newSingleThreadExecutor(); GitSingleProjectPropertiesLocator locator = new GitSingleProjectPropertiesLocator<>(store, - GitPropertiesLocatorUtils::getCommitInfoFromGitProperties, locatorExecutor, true); + GitPropertiesLocatorUtils::getCommitInfoFromGitProperties , locatorExecutor, true, null); store.upload(coverageFile); locator.searchFileForGitPropertiesAsync(new File(getClass().getResource("git-properties.jar").toURI()), true); @@ -47,4 +47,4 @@ public void locatorShouldTriggerUploadOfCachedXmls(@TempDir Path outputPath) thr .isFalse(); assertThat(destination.getUploadedFiles().contains(coverageFile)).isTrue(); } -} \ No newline at end of file +} diff --git a/system-tests/artifactory-git-properties-detection/src/main/resources/git.properties b/system-tests/artifactory-git-properties-detection/src/main/resources/git.properties deleted file mode 100644 index f54c8fc31..000000000 --- a/system-tests/artifactory-git-properties-detection/src/main/resources/git.properties +++ /dev/null @@ -1,3 +0,0 @@ -git.commit.id=86f9d655bf8a204d98bc3542e0d15cea38cc7c74 -git.branch=master -git.commit.time=2022-02-24T15:43:23+0100 diff --git a/system-tests/artifactory-git-properties-detection/src/main/resources/jar-with-properties.jar b/system-tests/artifactory-git-properties-detection/src/main/resources/jar-with-properties.jar new file mode 100644 index 0000000000000000000000000000000000000000..643746d93a74d31b9ef11923377eaf5d1482df47 GIT binary patch literal 258 zcmWIWW@Zs#U|`^2SnV1UR(msmHvz~i0%ATOPR}gSD=5k@NG&SKOfB|3W60N_z~gXn z?rI5N&9#d-w|`&?2#~zMva|lm*_ZzR-|plsn)*r2J^grYpK|eyHSP;6ZY#8L3cH-0 zv}dEzWS=GbbVKS-J?z>az3LVF)|g4o4sRH%mxO+HN^`v2$QHmK;LXTn&y348DnR#u xfI`EPMi7naHZBGQkRSsig9JnOVzW6O-ZvS+0!T!FH!B-N4I@JWkkJE{0|3|TN$LOq literal 0 HcmV?d00001