diff --git a/SecureService/.gitignore b/SecureService/.gitignore new file mode 100644 index 0000000..7234203 --- /dev/null +++ b/SecureService/.gitignore @@ -0,0 +1,110 @@ + +# Created by https://www.gitignore.io/api/java,linux,intellij + +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Java ### +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + + +# End of https://www.gitignore.io/api/java,linux,intellij + diff --git a/SecureService/Dockerfile.artifact b/SecureService/Dockerfile.artifact new file mode 100644 index 0000000..e6ae4d3 --- /dev/null +++ b/SecureService/Dockerfile.artifact @@ -0,0 +1,29 @@ +FROM maven:latest +#the build +ENV WORKINGDIR=/tmp +ENV MINIO_ACCESS_KEY test +ENV MINIO_SECRET_KEY test123456 + +#download minio +RUN wget -O, --output-document=${WORKINGDIR}/minio https://dl.minio.io/server/minio/release/linux-amd64/minio && \ + chmod +x ${WORKINGDIR}/minio + + +#download opensll for key creation +RUN apt-get -y install openssl + + + +ADD ./src ${WORKINGDIR}/src +ADD ./pom.xml ${WORKINGDIR}/pom.xml +ADD ./start.sh ${WORKINGDIR}/start.sh + + + +WORKDIR ${WORKINGDIR} +RUN mvn install compile -DskipTests +#the runtime +EXPOSE 8080 + +CMD ["sh", "start.sh"] + diff --git a/SecureService/Jenkins/deploy/staging.sh b/SecureService/Jenkins/deploy/staging.sh new file mode 100644 index 0000000..c8bc327 --- /dev/null +++ b/SecureService/Jenkins/deploy/staging.sh @@ -0,0 +1,5 @@ +ssh -i /opt/keypairs/ditas-testbed-keypair.pem cloudsigma@31.171.247.162 << 'ENDSSH' +sudo docker rm -f ditas/SecureService || true +sudo docker pull ditas/SecureService:latest +sudo docker run -p 50008:8080 -d --name SecureService ditas/SecureService:latest +ENDSSH \ No newline at end of file diff --git a/SecureService/README.md b/SecureService/README.md new file mode 100644 index 0000000..40d4c54 --- /dev/null +++ b/SecureService/README.md @@ -0,0 +1,6 @@ +# SecureService + +This service takes a POST request with a MultipartFile in the body and returns an URL with which you can retrieve the file later. +The file is stored encrypted in a Minio DB that only the service has access to. +Whenever the service shuts down the file is lost since the key is only stored in memory. +The file is retrieved by a simple GET request to the service with the URL that was returned earlier. diff --git a/SecureService/SecureService.iml b/SecureService/SecureService.iml new file mode 100644 index 0000000..66fecc1 --- /dev/null +++ b/SecureService/SecureService.iml @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SecureService/jenkinsfile b/SecureService/jenkinsfile new file mode 100644 index 0000000..ee7d087 --- /dev/null +++ b/SecureService/jenkinsfile @@ -0,0 +1,53 @@ +pipeline { + agent none + stages { + stage('Build - test') { + agent { + docker { + image 'maven:latest' + // TODO some cache to avoid npm sintall on every execution? + } + } + steps { + sh 'echo "Building! new"' + //sh 'npm install --prefix src' + sh 'mvn test' + } + } + stage('Image creation') { + agent any + steps { + // The Dockerfile.artifact copies the code into the image and run the jar generation. + echo 'Creating the image...' + + // This will search for a Dockerfile.artifact in the working directory and build the image to the local repository + sh "docker build -t \"ditas/SecureService\" -f Dockerfile.artifact ." + echo "Done" + echo 'Retrieving Docker Hub password from /opt/ditas-docker-hub.passwd...' + + // Get the password from a file. This reads the file from the host, not the container. Slaves already have the password in there. + script { + password = readFile '/opt/ditas-docker-hub.passwd' + } + echo "Done" + echo 'Login to Docker Hub as ditasgeneric...' + sh "docker login -u ditasgeneric -p ${password}" + echo "Done" + echo "Pushing the image ditas/data-utility-resolution-engine:latest..." + sh "docker push ditas/data-utility-resolution-engine:latest" + echo "Done " + } + } + stage('Image deploy') { + agent any + options { + // Don't need to checkout Git again + skipDefaultCheckout true + } + steps { + // Deploy to Staging environment calling the deployment script + sh './jenkins/deploy-staging.sh' + } + } + } +} \ No newline at end of file diff --git a/SecureService/pom.xml b/SecureService/pom.xml new file mode 100644 index 0000000..a3f2288 --- /dev/null +++ b/SecureService/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + ditas + SecureService + 0.0.1-SNAPSHOT + jar + + + org.springframework.boot + spring-boot-starter-parent + 2.0.1.RELEASE + + + + + + io.minio + minio + 3.0.12 + + + com.github.docker-java + docker-java + 3.0.14 + + + com.mashape.unirest + unirest-java + 1.4.9 + + + com.google.code.gson + gson + 2.8.2 + + + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-web + + + + io.springfox + springfox-swagger2 + 2.7.0 + + + io.springfox + springfox-swagger-ui + 2.7.0 + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework + spring-test + RELEASE + compile + + + org.springframework + spring-test + RELEASE + compile + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + \ No newline at end of file diff --git a/SecureService/sec-compose.yaml b/SecureService/sec-compose.yaml new file mode 100644 index 0000000..338f870 --- /dev/null +++ b/SecureService/sec-compose.yaml @@ -0,0 +1,19 @@ +version: '2' + +services: + minio1: + image: minio/minio + volumes: + - minio-data1:/export + ports: + - "9001:9000" + environment: + MINIO_ACCESS_KEY: SNEKIPF05UD33X0CI11X + MINIO_SECRET_KEY: 13u6DAFXUggenXRqS5b0euusw6EwUdlwLUUS1B9d + command: server http://minio1/export + + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + minio-data1: \ No newline at end of file diff --git a/SecureService/src/main/java/de.tub.secureService/Server.java b/SecureService/src/main/java/de.tub.secureService/Server.java new file mode 100644 index 0000000..cec2b3f --- /dev/null +++ b/SecureService/src/main/java/de.tub.secureService/Server.java @@ -0,0 +1,14 @@ +package de.tub.secureService; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Server { + + public static void main(String[] args) { + + SpringApplication.run(Server.class, args); + + } +} diff --git a/SecureService/src/main/java/de.tub.secureService/api/UploadingController.java b/SecureService/src/main/java/de.tub.secureService/api/UploadingController.java new file mode 100644 index 0000000..e4a04f5 --- /dev/null +++ b/SecureService/src/main/java/de.tub.secureService/api/UploadingController.java @@ -0,0 +1,87 @@ +package de.tub.secureService.api; + +import de.tub.secureService.service.MinioUploader; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.http.HttpResponse; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; +import javax.xml.ws.Response; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; + +@RestController +public class UploadingController { + private int salt = 234545; + + /** + * Takes the url of the file that is requested. Takes the fiel out Minio and returns an outputStream + * @param fileName + * @param response + */ + @RequestMapping(value = "/{file_name}", method = RequestMethod.GET) + public void getFile( + @PathVariable("file_name") String fileName, + HttpServletResponse response) { + try { + MinioUploader uploader =new MinioUploader(); + InputStream is = uploader.getObject(fileName); + if(is ==null ){ + response.getOutputStream().print(" "); + response.flushBuffer(); + }else{ + IOUtils.copy(is, response.getOutputStream()); + response.flushBuffer(); + } + } catch (IOException ex) { + throw new RuntimeException("IOError writing file to output stream"); + } + } + + /** + * Takes files that are uploaded to the service and stores them in Minio. Returns the url und which it can later be retrievevd + * @param uploadingFiles + * @return + * @throws IOException + */ + @RequestMapping(method = RequestMethod.POST, value= "/",consumes = "multipart/form-data") + public javax.ws.rs.core.Response uploadingPost(@RequestParam(value = "type") MultipartFile[] uploadingFiles) throws IOException { + //Convert the MultipartFile into a Stream + InputStream completeStream = null; + String contentType = null; + + if(uploadingFiles.length < 1){ + return null; + } + completeStream = uploadingFiles[0].getInputStream(); + long size = 0; + + for(int i=1; i fileMapping = new HashMap<>(); + + public MinioUploader() { + String accessKey = System.getenv("MINIO_ACCESS_KEY"); //"Z0P3J8WHKIMQUCLZWLW9";// + String secretKey = System.getenv("MINIO_SECRET_KEY");//"hs5cYWWMcgsbllIu2pOrSNp2SKofj6O3TGtwnaE3"; + try { + // Create a minioClient with the Minio Server name, Port, Access key and Secret key. + minioClient = new MinioClient(endpoint, accessKey, secretKey); + //Check if the Bucket is already existing + if(!minioClient.bucketExists(bucketName)) { + minioClient.makeBucket(bucketName); + } + + } catch (Exception e) { + System.out.println("Error occurred: " + e); + } + } + + /** + * Wrapper method for the put method of the minioClient. Maps the url to the responding key in a hashMap + * @param objectName + * @param input + * @param size + * @param contentType + */ + public void insertObject (String objectName, InputStream input,long size, String contentType) throws XmlPullParserException, InvalidBucketNameException, NoSuchAlgorithmException, InvalidArgumentException, InsufficientDataException, InvalidKeyException, ErrorResponseException, InvalidAlgorithmParameterException, NoSuchPaddingException, IOException, NoResponseException, IllegalBlockSizeException, InternalException, BadPaddingException { + + //Create a secret key to encrypt the file + SecretKey key = null; + try { + key = KeyGenerator.getInstance("AES").generateKey(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + fileMapping.put(objectName, key); + minioClient.putObject(bucketName,objectName,input, size, contentType, key); + + + } + + public void deleteObject(String bucketName, String objectName) { + try { + minioClient.removeObject(bucketName, objectName); + } catch (Exception e) { + e.printStackTrace(); + } + } + public void changeObject(String bucketName, String objectName, String filename) { + try { + minioClient.removeObject(bucketName, objectName); + minioClient.putObject(bucketName, objectName, filename); + + } catch (Exception e) { + e.printStackTrace(); + } + } + public InputStream getObject(String objectName) { + try { + SecretKey key =fileMapping.get(objectName); + return minioClient.getObject(bucketName, objectName, key); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + +} \ No newline at end of file diff --git a/SecureService/src/main/java/de.tub.secureService/storage/ApplicationStorageService.java b/SecureService/src/main/java/de.tub.secureService/storage/ApplicationStorageService.java new file mode 100644 index 0000000..bfb99c3 --- /dev/null +++ b/SecureService/src/main/java/de.tub.secureService/storage/ApplicationStorageService.java @@ -0,0 +1,39 @@ +package de.tub.secureService.storage; + +import org.springframework.core.io.Resource; +import org.springframework.web.multipart.MultipartFile; + +import java.nio.file.Path; +import java.util.stream.Stream; + +public class ApplicationStorageService implements StorageService { + @Override + public void init() { + + } + + @Override + public void store(MultipartFile file) { + + } + + @Override + public Stream loadAll() { + return null; + } + + @Override + public Path load(String filename) { + return null; + } + + @Override + public Resource loadAsResource(String filename) { + return null; + } + + @Override + public void deleteAll() { + + } +} diff --git a/SecureService/src/main/java/de.tub.secureService/storage/StorageService.java b/SecureService/src/main/java/de.tub.secureService/storage/StorageService.java new file mode 100644 index 0000000..12740be --- /dev/null +++ b/SecureService/src/main/java/de.tub.secureService/storage/StorageService.java @@ -0,0 +1,24 @@ +package de.tub.secureService.storage; + +import org.springframework.core.io.Resource; +import org.springframework.web.multipart.MultipartFile; + +import java.nio.file.Path; +import java.util.stream.Stream; + + + +public interface StorageService { + void init(); + + void store(MultipartFile file); + + Stream loadAll(); + + Path load(String filename); + + Resource loadAsResource(String filename); + + void deleteAll(); + +} diff --git a/SecureService/src/main/ressources/log4j.properties b/SecureService/src/main/ressources/log4j.properties new file mode 100644 index 0000000..393e087 --- /dev/null +++ b/SecureService/src/main/ressources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file diff --git a/SecureService/start.sh b/SecureService/start.sh new file mode 100644 index 0000000..fa2e6d9 --- /dev/null +++ b/SecureService/start.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +MINIO_ACCESS_KEY="$(openssl rand -base64 21)" +MINIO_ACCESS_KEY="$MINIO_ACCESS_KEY" | tr = a +export MINIO_ACCESS_KEY +MINIO_SECRET_KEY="$(openssl rand -base64 33)" +MINIO_SECRET_KEY="$MINIO_SECRET_KEY" | tr = a +export MINIO_SECRET_KEY + + + +./minio server /mnt/data & +echo $MINIO_ACCESS_KEY & +exec mvn spring-boot:run diff --git a/SecureTester/pom.xml b/SecureTester/pom.xml new file mode 100644 index 0000000..d6dbad6 --- /dev/null +++ b/SecureTester/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + ditas + SecureTester + 1.0-SNAPSHOT + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 10 + 10 + + + + + + + + + + com.mashape.unirest + unirest-java + 1.4.9 + + + + org.apache.httpcomponents + httpclient + 4.3.6 + + + org.apache.httpcomponents + httpasyncclient + 4.0.2 + + + org.apache.httpcomponents + httpmime + 4.3.6 + + + org.json + json + 20140107 + + + junit + junit + 4.11 + test + + + commons-io + commons-io + 2.4 + test + + + + log4j + log4j + 1.2.17 + + + + + + \ No newline at end of file diff --git a/SecureTester/src/main/java/RequestSender.java b/SecureTester/src/main/java/RequestSender.java new file mode 100644 index 0000000..6d9e5f9 --- /dev/null +++ b/SecureTester/src/main/java/RequestSender.java @@ -0,0 +1,178 @@ +import com.mashape.unirest.http.HttpResponse; +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.http.exceptions.UnirestException; +import org.apache.log4j.Logger; +import java.io.*; + +public class RequestSender { + static Logger log = Logger.getLogger(RequestSender.class.getName()); + + public static void main(String[] args){ + test("172.17.0.2",8080,"./../SecureService" ); + } + + public static void test(String ip, int port, String directory){ + String address = "http://"+ip+":"+ port+"/"; + String dockerName = buildDocker(directory); + Process pro = runDocker(dockerName, directory); + String container = getIDfromContainer(pro); + if(waitForDocker(address,240000)==false){ + log.info("Timeout in block 1"); + } + int rand = (int) (Math.random() *100000); + String content = Integer.toString(rand); + String multiRes = sendRandomMultipartFile(address , content); + String getRes = getFile(multiRes,address); + log.info("File content same Docker instance: \""+getRes+"\""); + if(waitForDocker(address,240000)==false){ + log.info("Timeout in block 2"); + } + stopContainer(container); + pro.destroy(); + pro = runDocker(dockerName,directory); + container = getIDfromContainer(pro); + if(waitForDocker(address,240000)==false){ + log.info("Timeout in block 3"); + } + getRes = getFile(multiRes,address); + log.info("File content different Docker instance: \""+ getRes+"\""+ " Expected: \" \""); + stopContainer(container); + pro.destroy(); + } + + public static String sendMultipartFile(String address) { + File multiFile = new File("/home/paavo/Work/TestFiles/MinioMulti"); + HttpResponse jsonResponse = null; + HttpResponse response = null; + try { + //jsonResponse = + response = Unirest.post(address) + .field("type", multiFile) + .asString(); + //.asJson(); + } catch (UnirestException e) { + e.printStackTrace(); + } + return response.getBody();//jsonResponse.getBody().toString(); + } + + public static String sendRandomMultipartFile(String address, String content) { + PrintWriter writer = null; + try { + writer = new PrintWriter("testFile.txt", "UTF-8"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + writer.print(content); + log.info("Content of the random file: \""+content+"\""); + writer.close(); + File multiFile = new File("testFile.txt"); + HttpResponse response = null; + try { + response = Unirest.post(address) + .field("type", multiFile) + .asString(); + } catch (UnirestException e) { + e.printStackTrace(); + } + return response.getBody(); + } + + public static String buildDocker(String directory){ + ProcessBuilder pb = new ProcessBuilder("docker", "build" ,"-f","Dockerfile.artifact", "."); + pb.directory(new File(directory)); + try { + Process buildIt = pb.start(); + buildIt.waitFor(); + InputStream in = buildIt.getInputStream(); + String res = inputStreamToString(in); + String [] ress = res.split( " "); + String name = ress[ress.length-1]; + buildIt.destroy(); + return name.trim(); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static Process runDocker(String name, String directory){ + ProcessBuilder pbr = new ProcessBuilder("docker", "run" , "--detach", name); + pbr.directory(new File(directory)); + try{ + Process runIt = pbr.start(); + return runIt; + }catch (Exception e){ + e.printStackTrace(); + } + return null; + } + + public static void stopContainer(String name){ + ProcessBuilder pbr = new ProcessBuilder("docker", "stop" , name); + try{ + Process stopIt = pbr.start(); + stopIt.waitFor(); + stopIt.destroy(); + }catch (Exception e){ + e.printStackTrace(); + } + } + + public static boolean waitForDocker(String address, int timeout){ + boolean temp = false; + while(!temp) { + temp = true; + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + try { + Unirest.get(address + "/d").asString(); + } catch (UnirestException e) { + temp = false; + } + } + return temp; + } + + public static String getFile(String url, String address) { + HttpResponse response = null; + try { + response = Unirest.get(address+url).asString(); + } catch (UnirestException e) { + e.printStackTrace(); + } + response.getStatus(); + return response.getBody(); + } + + public static String inputStreamToString(InputStream inputStream) throws IOException { + try(ByteArrayOutputStream result = new ByteArrayOutputStream()) { + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + return result.toString("UTF8"); + //return result.toString(UTF_8); + } + } + + public static String getIDfromContainer(Process pro){ + String res= null; + try { + InputStream in = pro.getInputStream(); + res = inputStreamToString(in); + + } catch (IOException e) { + e.printStackTrace(); + } + return res.trim(); + } + +} \ No newline at end of file diff --git a/SecureTester/src/main/resources/log4j.properties b/SecureTester/src/main/resources/log4j.properties new file mode 100644 index 0000000..393e087 --- /dev/null +++ b/SecureTester/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file