Skip to content

Commit

Permalink
support for s3 object lambda for downloading artifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
jonkipu committed Mar 17, 2022
1 parent 6ab3b53 commit 18d11fd
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 22 deletions.
17 changes: 10 additions & 7 deletions src/main/java/hudson/plugins/s3/Destination.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ public class Destination implements Serializable {
private static final long serialVersionUID = 1L;
public final String bucketName;
public final String objectName;
public final String s3ObjectLambda;

public Destination(final String userBucketName, final String fileName) {
public Destination(final String userBucketName, final String fileName, String s3ObjectLambda) {

if (userBucketName == null || fileName == null)
throw new IllegalArgumentException("Not defined for null parameters: "+userBucketName+","+fileName);
Expand All @@ -34,6 +35,8 @@ public Destination(final String userBucketName, final String fileName) {
} else {
objectName = s3CompatibleFileName;
}

this.s3ObjectLambda = s3ObjectLambda;
}

private String replaceWindowsBackslashes(String fileName) {
Expand All @@ -46,7 +49,7 @@ public String toString() {
}


public static Destination newFromRun(Run run, String bucketName, String fileName, boolean enableFullpath)
public static Destination newFromRun(Run run, String bucketName, String fileName, boolean enableFullpath, String s3ObjectLambda)
{
final String projectName;

Expand All @@ -58,23 +61,23 @@ public static Destination newFromRun(Run run, String bucketName, String fileName
}

int buildID = run.getNumber();
return new Destination(bucketName, "jobs/" + projectName + "/" + buildID + "/" + fileName);
return new Destination(bucketName, "jobs/" + projectName + "/" + buildID + "/" + fileName, s3ObjectLambda);
}

public static Destination newFromRun(Run run, S3Artifact artifact)
{
return newFromRun(run, artifact.getBucket(), artifact.getName(), artifact.useFullProjectName());
return newFromRun(run, artifact.getBucket(), artifact.getName(), artifact.useFullProjectName(), artifact.getObjectLambda());
}

public static Destination newFromRunDetails(RunDetails runDetails, String bucketName, String fileName, boolean enableFullpath) {
public static Destination newFromRunDetails(RunDetails runDetails, String bucketName, String fileName, boolean enableFullpath, String s3ObjectLambda) {
final String projectName = runDetails.getProjectName(enableFullpath);

int buildID = runDetails.getBuildId();
return new Destination(bucketName, "jobs/" + projectName + "/" + buildID + "/" + fileName);
return new Destination(bucketName, "jobs/" + projectName + "/" + buildID + "/" + fileName, s3ObjectLambda);
}

public static Destination newFromRunDetails(RunDetails runDetails, S3Artifact artifact)
{
return newFromRunDetails(runDetails, artifact.getBucket(), artifact.getName(), artifact.useFullProjectName());
return newFromRunDetails(runDetails, artifact.getBucket(), artifact.getName(), artifact.useFullProjectName(), artifact.getObjectLambda());
}
}
9 changes: 6 additions & 3 deletions src/main/java/hudson/plugins/s3/Entry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public final class Entry implements Describable<Entry> {
* Stores the Region Value
*/
public String selectedRegion;

/**
* Do not publish the artifacts when build fails
*/
Expand All @@ -58,7 +58,7 @@ public final class Entry implements Describable<Entry> {
* Let Jenkins manage the S3 uploaded artifacts
*/
public boolean managedArtifacts;

/**
* Use S3 server side encryption when uploading the artifacts
*/
Expand Down Expand Up @@ -90,11 +90,13 @@ public final class Entry implements Describable<Entry> {
*/
public List<MetadataPair> userMetadata;

public String s3ObjectLambda;

@DataBoundConstructor
public Entry(String bucket, String sourceFile, String excludedFile, String storageClass, String selectedRegion,
boolean noUploadOnFailure, boolean uploadFromSlave, boolean managedArtifacts,
boolean useServerSideEncryption, boolean flatten, boolean gzipFiles, boolean keepForever,
boolean showDirectlyInBrowser, List<MetadataPair> userMetadata) {
boolean showDirectlyInBrowser, List<MetadataPair> userMetadata, String s3ObjectLambda) {
this.bucket = bucket;
this.sourceFile = sourceFile;
this.excludedFile = excludedFile;
Expand All @@ -109,6 +111,7 @@ public Entry(String bucket, String sourceFile, String excludedFile, String stora
this.keepForever = keepForever;
this.userMetadata = userMetadata;
this.showDirectlyInBrowser = showDirectlyInBrowser;
this.s3ObjectLambda = s3ObjectLambda;
}

@Override
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/hudson/plugins/s3/FingerprintRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,17 @@ public class FingerprintRecord implements Serializable {
private boolean showDirectlyInBrowser;
private final S3Profile profile;
private final RunDetails runDetails;
private final String s3ObjectLambda;

public FingerprintRecord(boolean produced, String bucket, String name, String region, String md5sum, RunDetails runDetails, S3Profile profile) {
public FingerprintRecord(boolean produced, String bucket, String name, String region, String md5sum, RunDetails runDetails, S3Profile profile, String s3ObjectLambda) {
this.produced = produced;
this.artifact = new S3Artifact(region, bucket, name);
this.artifact = new S3Artifact(region, bucket, name, s3ObjectLambda);
this.md5sum = md5sum;
this.showDirectlyInBrowser = false;
this.keepForever = false;
this.runDetails = runDetails;
this.profile = profile;
this.s3ObjectLambda = s3ObjectLambda;
}

Fingerprint addRecord(Run<?, ?> run) throws IOException {
Expand Down Expand Up @@ -105,7 +107,7 @@ public S3Artifact getArtifact() {

private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, RunDetails runDetails, FingerprintRecord record) {
final Destination dest = Destination.newFromRunDetails(runDetails, record.getArtifact());
final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName);
final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.s3ObjectLambda, dest.objectName);
request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000));

if (!record.isShowDirectlyInBrowser()) {
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/hudson/plugins/s3/S3Artifact.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ public final class S3Artifact implements Serializable {
private final String name;
private final String region;
private /*almost final*/ Boolean useFullProjectName;
private final String s3ObjectLambda;

public S3Artifact(String region, String bucket, String name) {
public S3Artifact(String region, String bucket, String name, String s3ObjectLambda) {
this.bucket = bucket.intern();
this.name = name.intern();
this.region = region.intern();
this.useFullProjectName = true;
this.s3ObjectLambda = s3ObjectLambda;
}

/* Old version of this plugin used short name,
Expand All @@ -46,10 +48,15 @@ public String getRegion() {
return region;
}

@Exported
public String getObjectLambda() {
return s3ObjectLambda;
}

public Boolean useFullProjectName() {
if (useFullProjectName == null)
return false;

return useFullProjectName;
}
}
}
3 changes: 2 additions & 1 deletion src/main/java/hudson/plugins/s3/S3BucketPublisher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ public void perform(@Nonnull Run<?, ?> run, @Nonnull FilePath ws, @Nonnull Launc

final String bucket = Util.replaceMacro(entry.bucket, envVars);
final String storageClass = Util.replaceMacro(entry.storageClass, envVars);
final String s3ObjectLambda = Util.replaceMacro(entry.s3ObjectLambda, envVars);
final String selRegion = entry.selectedRegion;

final List<FilePath> paths = new ArrayList<>();
Expand Down Expand Up @@ -294,7 +295,7 @@ public void perform(@Nonnull Run<?, ?> run, @Nonnull FilePath ws, @Nonnull Launc
final Map<String, String> escapedMetadata = buildMetadata(envVars, entry);

final List<FingerprintRecord> records = Lists.newArrayList();
final List<FingerprintRecord> fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles);
final List<FingerprintRecord> fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles, s3ObjectLambda);

for (FingerprintRecord fingerprintRecord : fingerprints) {
records.add(fingerprintRecord);
Expand Down
13 changes: 7 additions & 6 deletions src/main/java/hudson/plugins/s3/S3Profile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ public List<FingerprintRecord> upload(Run<?, ?> run,
final boolean uploadFromSlave,
final boolean managedArtifacts,
final boolean useServerSideEncryption,
final boolean gzipFiles) throws IOException, InterruptedException {
final boolean gzipFiles,
final String s3ObjectLambda) throws IOException, InterruptedException {
final List<FingerprintRecord> fingerprints = new ArrayList<>(fileNames.size());

try {
Expand All @@ -143,10 +144,10 @@ public List<FingerprintRecord> upload(Run<?, ?> run,
final Destination dest;
final boolean produced;
if (managedArtifacts) {
dest = Destination.newFromRun(run, bucketName, fileName, true);
dest = Destination.newFromRun(run, bucketName, fileName, true, s3ObjectLambda);
produced = run.getTimeInMillis() <= filePath.lastModified() + 2000;
} else {
dest = new Destination(bucketName, fileName);
dest = new Destination(bucketName, fileName, s3ObjectLambda);
produced = false;
}

Expand All @@ -164,7 +165,7 @@ public List<FingerprintRecord> upload(Run<?, ?> run,
@Override
public FingerprintRecord call() throws IOException, InterruptedException {
final String md5 = invoke(uploadFromSlave, filePath, upload);
return new FingerprintRecord(produced, bucketName, fileName, selregion, md5, new RunDetails(run), profile);
return new FingerprintRecord(produced, bucketName, fileName, selregion, md5, new RunDetails(run), profile, s3ObjectLambda);
}
});

Expand Down Expand Up @@ -209,7 +210,7 @@ public List<String> list(Run build, String bucket) {

final String buildName = build.getDisplayName();
final int buildID = build.getNumber();
final Destination dest = new Destination(bucket, "jobs/" + buildName + '/' + buildID + '/' + name);
final Destination dest = new Destination(bucket, "jobs/" + buildName + '/' + buildID + '/' + name, bucket);

final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(dest.bucketName)
Expand Down Expand Up @@ -252,7 +253,7 @@ public List<FingerprintRecord> downloadAll(Run<?,?> build,
@Override
public FingerprintRecord call() throws IOException, InterruptedException {
final String md5 = target.act(new S3DownloadCallable(accessKey, secretKey, useRole, dest, artifact.getRegion(), getProxy()));
return new FingerprintRecord(true, dest.bucketName, target.getName(), artifact.getRegion(), md5, new RunDetails(build), profile);
return new FingerprintRecord(true, dest.bucketName, target.getName(), artifact.getRegion(), md5, new RunDetails(build), profile, artifact.getObjectLambda());
}
}));
}
Expand Down

0 comments on commit 18d11fd

Please sign in to comment.