From afba7edc96a0433a9b9ce0cef0396187dca38ea4 Mon Sep 17 00:00:00 2001 From: Geolykt Date: Tue, 29 Mar 2022 21:02:37 +0200 Subject: [PATCH] Compute checksums during publish, deprecate MavenPublishing Once slbrachyura hard-forks those classes will get deleted. Todo: - Snapshot publishing support - FTP repo support (perhaps - we will see later on whether that is viable) --- .../brachyura/fabric/FabricProject.java | 16 ++- .../brachyura/maven/LocalMavenRepository.java | 29 +++-- .../brachyura/maven/MavenPublishing.java | 11 ++ ...AuthentificatedMavenPublishRepository.java | 104 ++++++++++++++++++ .../project/java/SimpleJavaProject.java | 12 +- .../coolcrabs/brachyura/util/PathUtil.java | 2 +- 6 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/publish/AuthentificatedMavenPublishRepository.java diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/fabric/FabricProject.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/fabric/FabricProject.java index e1bb2605..94ae77d7 100644 --- a/brachyura/src/main/java/io/github/coolcrabs/brachyura/fabric/FabricProject.java +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/fabric/FabricProject.java @@ -64,6 +64,7 @@ import io.github.coolcrabs.brachyura.maven.LocalMavenRepository; import io.github.coolcrabs.brachyura.maven.MavenId; import io.github.coolcrabs.brachyura.maven.MavenResolver; +import io.github.coolcrabs.brachyura.maven.publish.AuthentificatedMavenPublishRepository; import io.github.coolcrabs.brachyura.maven.publish.MavenPublisher; import io.github.coolcrabs.brachyura.minecraft.Minecraft; import io.github.coolcrabs.brachyura.minecraft.VersionMeta; @@ -81,7 +82,6 @@ import io.github.coolcrabs.brachyura.processing.sources.ZipProcessingSource; import io.github.coolcrabs.brachyura.project.Task; import io.github.coolcrabs.brachyura.project.java.BaseJavaProject; -import io.github.coolcrabs.brachyura.project.java.SimpleJavaProject; import io.github.coolcrabs.brachyura.util.AtomicDirectory; import io.github.coolcrabs.brachyura.util.AtomicFile; import io.github.coolcrabs.brachyura.util.CloseableArrayList; @@ -226,7 +226,6 @@ public void getTasks(@NotNull Consumer p) { } public void getPublishTasks(Consumer p) { - SimpleJavaProject.createPublishTasks(p, this::build); p.accept(Task.of("publishToMavenLocal", (ThrowingRunnable) () -> { MavenPublisher publisher = new MavenPublisher().addRepository(new LocalMavenRepository(MavenResolver.MAVEN_LOCAL)); List mavendeps = new ArrayList<>(); @@ -240,6 +239,19 @@ public void getPublishTasks(Consumer p) { }); publisher.publishJar(build(), mavendeps); })); + p.accept(Task.of("publish", (ThrowingRunnable) () -> { + MavenPublisher publisher = new MavenPublisher().addRepository(AuthentificatedMavenPublishRepository.fromEnvironmentVariables()); + List mavendeps = new ArrayList<>(); + dependencies.get().forEach(dep -> { + if (dep instanceof MavenDependency) { + mavendeps.add((MavenDependency) dep); + } + }); + modDependencies.get().forEach(modDep -> { + mavendeps.add(modDep.jarDependency); + }); + publisher.publishJar(build(), mavendeps); + })); } @Override diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/LocalMavenRepository.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/LocalMavenRepository.java index 32a9400c..ac801972 100644 --- a/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/LocalMavenRepository.java +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/LocalMavenRepository.java @@ -1,17 +1,23 @@ package io.github.coolcrabs.brachyura.maven; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.DigestInputStream; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.tinylog.Logger; import io.github.coolcrabs.brachyura.maven.publish.PublicationId; import io.github.coolcrabs.brachyura.maven.publish.PublishRepository; +import io.github.coolcrabs.brachyura.util.PathUtil; /** * A simple implementation of the {@link MavenRepository} class that makes use of flatfile storage @@ -39,23 +45,32 @@ public LocalMavenRepository(@NotNull Path root) { @Override public void publish(@NotNull PublicationId id, byte @NotNull [] source) throws IOException { - Path target = root.resolve(id.toPath()); - Logger.info("Publishing " + id.toString() + " to " + target.toAbsolutePath().toString()); - Files.write(target, source); + publish(id, new ByteArrayInputStream(source)); } @Override public void publish(@NotNull PublicationId id, @NotNull InputStream source) throws IOException { Path target = root.resolve(id.toPath()); + Files.createDirectories(target.getParent()); Logger.info("Publishing " + id.toString() + " to " + target.toAbsolutePath().toString()); - Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + try (DigestInputStream digestInSha256 = new DigestInputStream(source, DigestUtils.getSha256Digest())) { + try (DigestInputStream digestInMd5 = new DigestInputStream(digestInSha256, DigestUtils.getMd5Digest())) { + try (DigestInputStream digestInSha1 = new DigestInputStream(digestInMd5, DigestUtils.getSha1Digest())) { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + byte[] checksum = Hex.encodeHexString(digestInMd5.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + Files.write(target.resolveSibling(target.getFileName() + ".sha1"), checksum); + } + byte[] checksum = Hex.encodeHexString(digestInMd5.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + Files.write(target.resolveSibling(target.getFileName() + ".md5"), checksum); + } + byte[] checksum = Hex.encodeHexString(digestInSha256.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + Files.write(target.resolveSibling(target.getFileName() + ".sha256"), checksum); + } } @Override public void publish(@NotNull PublicationId id, @NotNull Path source) throws IOException { - Path target = root.resolve(id.toPath()); - Logger.info("Installing " + id.toString() + " (" + source.toAbsolutePath().toString() + ") to " + target.toAbsolutePath().toString()); - Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + publish(id, PathUtil.inputStream(source)); } @Override diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/MavenPublishing.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/MavenPublishing.java index 2b8c108f..b297cc92 100644 --- a/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/MavenPublishing.java +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/MavenPublishing.java @@ -28,14 +28,17 @@ import org.jetbrains.annotations.Nullable; import org.tinylog.Logger; +@Deprecated public class MavenPublishing { private MavenPublishing() { } + @Deprecated public static class AuthenticatedMaven { final String mavenUrl; final String username; final String password; + @Deprecated public AuthenticatedMaven(String mavenUrl, @Nullable String username, @Nullable String password) { Objects.requireNonNull(mavenUrl, "Unset maven url"); if ((username == null) != (password == null)) throw new UnsupportedOperationException("Username and password should both be set or not set"); @@ -44,10 +47,12 @@ public AuthenticatedMaven(String mavenUrl, @Nullable String username, @Nullable this.password = password; } + @Deprecated public static AuthenticatedMaven ofMavenLocal() { return new AuthenticatedMaven(MavenResolver.MAVEN_LOCAL.toUri().toString(), null, null); } + @Deprecated public static AuthenticatedMaven ofEnv() { return new AuthenticatedMaven( System.getenv("BRACHYURA_PUBLISH_MAVEN"), @@ -62,6 +67,7 @@ public static AuthenticatedMaven ofEnv() { * @param maven * @param dep */ + @Deprecated public static void publish(AuthenticatedMaven maven, JavaJarDependency dep) { publish(maven, dep, stubPom(dep.mavenId)); } @@ -72,6 +78,7 @@ public static void publish(AuthenticatedMaven maven, JavaJarDependency dep) { * @param dep * @param pom */ + @Deprecated public static void publish(AuthenticatedMaven maven, JavaJarDependency dep, Supplier pom) { Objects.requireNonNull(pom); ArrayList a = new ArrayList<>(3); @@ -86,6 +93,7 @@ public static void publish(AuthenticatedMaven maven, JavaJarDependency dep, Supp publish(maven, a); } + @Deprecated public static void publish(AuthenticatedMaven maven, List files) { try { ArrayList a = new ArrayList<>(files.size() * 2); @@ -107,6 +115,7 @@ public static void publish(AuthenticatedMaven maven, List file } } + @Deprecated public static void rawPublish(AuthenticatedMaven maven, List files) { try { String trailSlashRepo; @@ -148,6 +157,7 @@ static String getMavenPath(@NotNull MavenId id, @NotNull String ext) { return id.groupId.replace('.', '/') + "/" + id.artifactId + "/" + id.version + "/" + id.artifactId + "-" + id.version + ext; } + @Deprecated public static class MavenPublishFile { @NotNull final String fileName; @@ -159,6 +169,7 @@ public MavenPublishFile(@NotNull String fileName, Supplier in) { } } + @Deprecated public static Supplier stubPom(MavenId id) { try { ByteArrayOutputStreamEx o = new ByteArrayOutputStreamEx(); diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/publish/AuthentificatedMavenPublishRepository.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/publish/AuthentificatedMavenPublishRepository.java new file mode 100644 index 00000000..788fccd2 --- /dev/null +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/maven/publish/AuthentificatedMavenPublishRepository.java @@ -0,0 +1,104 @@ +package io.github.coolcrabs.brachyura.maven.publish; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.DigestInputStream; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; +import org.tinylog.Logger; + +import io.github.coolcrabs.brachyura.util.NetUtil; +import io.github.coolcrabs.brachyura.util.PathUtil; + +/** + * Basically a backport of the old AuthentificatedMaven feature. + * No guarantees are made on whether this works in the first place, since I do not make + * use of such maven repositories myself. + */ +public class AuthentificatedMavenPublishRepository implements PublishRepository { + + /** + * Obtains an {@link AuthentificatedMavenPublishRepository} based on the environment variables. + * The base URL of the repository is defined by BRACHYURA_PUBLISH_MAVEN, the username is defined by + * BRACHYURA_PUBLISH_USERNAME and the password by BRACHYURA_PUBLISH_PASSWORD. + * + * @return The newly created instance + */ + @NotNull + @Contract(pure = true, value = "-> new") + public static PublishRepository fromEnvironmentVariables() { + return new AuthentificatedMavenPublishRepository( + System.getenv("BRACHYURA_PUBLISH_MAVEN"), + System.getenv("BRACHYURA_PUBLISH_USERNAME"), + System.getenv("BRACHYURA_PUBLISH_PASSWORD") + ); + } + private final String mavenURL; + private final String password; + + private final String username; + + public AuthentificatedMavenPublishRepository(String baseURL, String username, String password) { + this.mavenURL = baseURL; + this.username = username; + this.password = password; + } + + @Override + public void publish(@NotNull PublicationId id, byte @NotNull [] source) throws IOException { + publish(id, new ByteArrayInputStream(source)); + } + + @Override + public void publish(@NotNull PublicationId id, @NotNull InputStream source) throws IOException { + String trailSlashRepo; + String mavenRepoUrl = mavenURL; + if (mavenRepoUrl.codePointBefore(mavenRepoUrl.length()) == '/') { + trailSlashRepo = mavenRepoUrl; + } else { + trailSlashRepo = mavenRepoUrl + '/'; + } + String baseFileName = id.toPath().toString(); + URI basePath = null; + try { + trailSlashRepo = trailSlashRepo.replace(File.separatorChar, '/'); // Ensure consistency with windows + basePath = new URI(trailSlashRepo); + } catch (URISyntaxException ex) { + throw new IOException("Invalid URI: " + trailSlashRepo + ".", ex); + } + + try (DigestInputStream digestInSha256 = new DigestInputStream(source, DigestUtils.getSha256Digest())) { + try (DigestInputStream digestInMd5 = new DigestInputStream(digestInSha256, DigestUtils.getMd5Digest())) { + try (DigestInputStream digestInSha1 = new DigestInputStream(digestInMd5, DigestUtils.getSha1Digest())) { + URL url = basePath.resolve(baseFileName).toURL(); + Logger.info("Publishing " + url.toString()); + NetUtil.put(url, digestInSha1, username, password); + byte[] checksum = Hex.encodeHexString(digestInMd5.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + url = basePath.resolve(baseFileName + ".sha1").toURL(); + NetUtil.put(url, new ByteArrayInputStream(checksum), username, password); + } + byte[] checksum = Hex.encodeHexString(digestInMd5.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + URL checksumURL = basePath.resolve(baseFileName + ".md5").toURL(); + NetUtil.put(checksumURL, new ByteArrayInputStream(checksum), username, password); + } + byte[] checksum = Hex.encodeHexString(digestInSha256.getMessageDigest().digest()).getBytes(StandardCharsets.UTF_8); + URL checksumURL = basePath.resolve(baseFileName + ".sha256").toURL(); + NetUtil.put(checksumURL, new ByteArrayInputStream(checksum), username, password); + } + } + + @Override + public void publish(@NotNull PublicationId id, @NotNull Path source) throws IOException { + publish(id, PathUtil.inputStream(source)); + } +} diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/project/java/SimpleJavaProject.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/project/java/SimpleJavaProject.java index 1d27b0f2..49e94e40 100644 --- a/brachyura/src/main/java/io/github/coolcrabs/brachyura/project/java/SimpleJavaProject.java +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/project/java/SimpleJavaProject.java @@ -17,8 +17,8 @@ import io.github.coolcrabs.brachyura.ide.IdeModule; import io.github.coolcrabs.brachyura.maven.LocalMavenRepository; import io.github.coolcrabs.brachyura.maven.MavenId; -import io.github.coolcrabs.brachyura.maven.MavenPublishing; import io.github.coolcrabs.brachyura.maven.MavenResolver; +import io.github.coolcrabs.brachyura.maven.publish.AuthentificatedMavenPublishRepository; import io.github.coolcrabs.brachyura.maven.publish.MavenPublisher; import io.github.coolcrabs.brachyura.processing.sinks.AtomicZipProcessingSink; import io.github.coolcrabs.brachyura.processing.sources.DirectoryProcessingSource; @@ -50,16 +50,14 @@ public void getTasks(@NotNull Consumer p) { } public void getPublishTasks(Consumer p) { - createPublishTasks(p, this::build); p.accept(Task.of("publishToMavenLocal", (ThrowingRunnable) () -> { MavenPublisher publisher = new MavenPublisher().addRepository(new LocalMavenRepository(MavenResolver.MAVEN_LOCAL)); publisher.publishJar(build(), dependencies.get()); })); - } - - @Deprecated - public static void createPublishTasks(Consumer p, BuildSupplier build) { - p.accept(Task.of("publish", (ThrowingRunnable) () -> MavenPublishing.publish(MavenPublishing.AuthenticatedMaven.ofEnv(), build.get()))); + p.accept(Task.of("publish", (ThrowingRunnable) () -> { + MavenPublisher publisher = new MavenPublisher().addRepository(AuthentificatedMavenPublishRepository.fromEnvironmentVariables()); + publisher.publishJar(build(), dependencies.get()); + })); } @Override diff --git a/brachyura/src/main/java/io/github/coolcrabs/brachyura/util/PathUtil.java b/brachyura/src/main/java/io/github/coolcrabs/brachyura/util/PathUtil.java index 6eec298d..a9717996 100644 --- a/brachyura/src/main/java/io/github/coolcrabs/brachyura/util/PathUtil.java +++ b/brachyura/src/main/java/io/github/coolcrabs/brachyura/util/PathUtil.java @@ -139,10 +139,10 @@ public static Path pathTransform(FileSystem fs, final Path path) { for (Path component : path) { ret = ret.resolve(component.getFileName().toString()); } - assert ret != null; // This is the only way for eclipse to suppress the null warning return ret; } + @NotNull public static InputStream inputStream(@NotNull Path path) { try { return new BufferedInputStream(Files.newInputStream(path));