Add tar zip

.github/workflows/gradle.yml

@@ -10,12 +10,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@v3
       with:
-        java-version: 1.8
-
+        java-version: 8
+        distribution: temurin
+
     - name: build and test cadc-app-kit
       run: cd cadc-app-kit && ../gradlew --info clean build javadoc checkstyleMain install
 

cadc-download-manager-server/build.gradle

@@ -10,7 +10,7 @@ repositories {
     mavenLocal()
 }
 
-version = '1.5.0'
+version = '1.6.0'
 group = 'org.opencadc'
 sourceCompatibility = 1.8
 
@@ -29,11 +29,15 @@ dependencies {
     implementation 'org.opencadc:cadc-registry:[1.0,)'
     implementation 'org.opencadc:cadc-rest:[1.3.3,)'
     implementation 'org.opencadc:cadc-util:[1.6,)'
+
+    // Locking down cadc-vos to a version that is JDK 1.8 compatible.
+    // Use version >= 2.x when JDK 11 is supported.
+    implementation 'org.opencadc:cadc-vos:1.2.3'
+
     implementation 'org.opencadc:cadc-web-util:[1.2.10,)'
     implementation 'commons-io:commons-io:[2.7,)'
 
     testImplementation 'junit:junit:4.13'
-    testImplementation 'org.easymock:easymock:3.6'
 }
 
 apply from: '../opencadc.gradle'

cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/DispatcherServlet.java

@@ -114,6 +114,8 @@ public class DispatcherServlet extends HttpServlet {
     public static String HTMLLIST = "HTML List";
     public static String WEBSTART = "Java Webstart";
     public static String SHELL_SCRIPT = "Shell Script";
+    public static String TAR_PACKAGE = "TAR";
+    public static String ZIP_PACKAGE = "ZIP";
 
 
     /// Used during JSP compilation
@@ -139,6 +141,10 @@ public static String getDownloadMethod(HttpServletRequest request) {
                 target = "/wget.jsp";
             } else if (SHELL_SCRIPT.equals(method)) {
                 target = ShellScriptServlet.SCRIPT_TARGET;
+            } else if (DispatcherServlet.TAR_PACKAGE.equals(method)) {
+                target = TARPackageServlet.TAR_PACKAGE_TARGET;
+            } else if (DispatcherServlet.ZIP_PACKAGE.equals(method)) {
+                target = ZIPPackageServlet.ZIP_PACKAGE_TARGET;
             } else {
                 return null;
             }
@@ -194,8 +200,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
             throw ex;
         } catch (Exception e) {
             if (e instanceof RuntimeException) {
-                RuntimeException rex = (RuntimeException) e;
-                throw rex;
+                throw (RuntimeException) e;
             }
         } finally {
             Long dt = System.currentTimeMillis() - start;
@@ -205,7 +210,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
     }
 
 
-    private class DownloadAction implements PrivilegedExceptionAction<Object> {
+    private static class DownloadAction implements PrivilegedExceptionAction<Object> {
         HttpServletRequest request;
         HttpServletResponse response;
 
@@ -218,14 +223,14 @@ public Object run() throws Exception {
             // forward
             DownloadRequest downloadReq = (DownloadRequest) request.getAttribute(INTERNAL_FORWARD_PARAMETER);
 
-            // Set up input handling
-            DLMInputHandler inputHandler = new DLMInputHandler(request);
-
             if (downloadReq == null) {
+                // Set up input handling
+                DLMInputHandler inputHandler = new DLMInputHandler(request);
+
                 // external post
                 inputHandler.parseInput();
 
-                downloadReq = inputHandler.getDownloadRequest();
+                downloadReq = DLMInputHandler.getDownloadRequest();
                 Set<DownloadTuple> tupleList = downloadReq.getTuples();
                 List<Exception> validationErrList = downloadReq.getValidationErrors();
 

cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/PackageServlet.java

@@ -0,0 +1,130 @@
+package ca.nrc.cadc.dlm.server;
+
+import ca.nrc.cadc.auth.AuthMethod;
+import ca.nrc.cadc.auth.AuthenticationUtil;
+import ca.nrc.cadc.config.ApplicationConfiguration;
+import ca.nrc.cadc.dlm.DownloadRequest;
+import ca.nrc.cadc.net.HttpPost;
+import ca.nrc.cadc.reg.Standards;
+import ca.nrc.cadc.reg.client.RegistryClient;
+import ca.nrc.cadc.util.StringUtil;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.security.auth.Subject;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+
+public abstract class PackageServlet extends HttpServlet {
+    private static final String PACKAGE_SERVICE_RESOURCE_ID_KEY = "org.opencadc.dlm.package-download.service.id";
+    private static final String RESPONSE_FORMAT_PAYLOAD_KEY = "RESPONSEFORMAT";
+    private static final String ID_PAYLOAD_KEY = "ID";
+    private static final String RUN_ID_PAYLOAD_KEY = "runid";
+
+    final ApplicationConfiguration configuration = new ApplicationConfiguration(DispatcherServlet.DEFAULT_CONFIG_FILE_PATH);
+
+
+    /**
+     * Handle POSTed download request from an external page.
+     *
+     * @param request  The HTTP Request
+     * @param response The HTTP Response
+     */
+    @Override
+    protected void doPost(HttpServletRequest request, HttpServletResponse response) {
+        final URI packageServiceURI = this.configuration.lookupServiceURI(PackageServlet.PACKAGE_SERVICE_RESOURCE_ID_KEY, null);
+        if (packageServiceURI == null) {
+            throw new IllegalStateException("No service configured to perform Package Downloads");
+        } else {
+            doRedirect(request, response, packageServiceURI);
+        }
+    }
+
+    /**
+     * Handle creating a payload and obtaining a redirect URL from the main service.
+     * @param request               The HTTP Request.
+     * @param response              The HTTP Response.
+     * @param packageServiceURI     The URI of the configured
+     */
+    void doRedirect(final HttpServletRequest request, final HttpServletResponse response, final URI packageServiceURI) {
+        final Subject currentSubject = AuthenticationUtil.getCurrentSubject();
+        final URL packageServiceURL = lookupServiceURL(packageServiceURI, currentSubject);
+
+        Subject.doAs(currentSubject, (PrivilegedAction<Void>) () -> {
+            final HttpPost httpPost = new HttpPost(packageServiceURL, getPayload(request), false);
+
+            try {
+                httpPost.prepare();
+            } catch (Exception exception) {
+                throw new RuntimeException(exception.getMessage(), exception);
+            }
+
+            final URL redirect = httpPost.getRedirectURL();
+            if (redirect == null) {
+                throw new IllegalArgumentException("Unable to complete download (no redirect).");
+            } else {
+                try {
+                    response.sendRedirect(redirect.toExternalForm());
+                } catch (IOException ioException) {
+                    throw new IllegalStateException("Redirect is invalid: " + redirect);
+                }
+            }
+
+            return null;
+        });
+    }
+
+    private Map<String, Object> getPayload(final HttpServletRequest request) {
+        final DownloadRequest downloadRequest = getDownloadRequest(request);
+
+        final Map<String, Object> payload = new HashMap<>();
+
+        final List<String> publisherIDs =
+            downloadRequest.getTuples().stream().map(downloadTuple -> downloadTuple.getID().toString()).collect(Collectors.toList());
+
+        if (publisherIDs.isEmpty()) {
+            throw new IllegalArgumentException("Nothing specified to download.  Use tuple=<URI>.");
+        }
+
+        payload.put(PackageServlet.ID_PAYLOAD_KEY, publisherIDs);
+
+        final String contentType = getContentType();
+        if (!StringUtil.hasText(contentType)) {
+            throw new IllegalStateException("Poorly configured package type (no content type specified)");
+        }
+
+        payload.put(PackageServlet.RESPONSE_FORMAT_PAYLOAD_KEY, contentType);
+
+        if (StringUtil.hasText(downloadRequest.runID)) {
+            payload.put(PackageServlet.RUN_ID_PAYLOAD_KEY, downloadRequest.runID);
+        }
+
+        return payload;
+    }
+
+    private URL lookupServiceURL(URI packageServiceURI, Subject currentSubject) {
+        final RegistryClient registryClient = new RegistryClient();
+        final AuthMethod authMethod = currentSubject == null ? AuthMethod.ANON : AuthenticationUtil.getAuthMethod(currentSubject);
+        return registryClient.getServiceURL(packageServiceURI, Standards.PKG_10, authMethod);
+    }
+
+    private DownloadRequest getDownloadRequest(final HttpServletRequest request) {
+        final DownloadRequest downloadReq = (DownloadRequest) request.getAttribute("downloadRequest");
+        downloadReq.runID = (String) request.getAttribute("runid");
+
+        return downloadReq;
+    }
+
+    /**
+     * Pull the specific content type to set in the payload.
+     * @return      String content-type, never null.
+     */
+    abstract String getContentType();
+}

cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/TARPackageServlet.java

@@ -0,0 +1,80 @@
+/*
+ ************************************************************************
+ *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
+ **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
+ *
+ *  (c) 2024.                            (c) 2024.
+ *  Government of Canada                 Gouvernement du Canada
+ *  National Research Council            Conseil national de recherches
+ *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
+ *  All rights reserved                  Tous droits réservés
+ *
+ *  NRC disclaims any warranties,        Le CNRC dénie toute garantie
+ *  expressed, implied, or               énoncée, implicite ou légale,
+ *  statutory, of any kind with          de quelque nature que ce
+ *  respect to the software,             soit, concernant le logiciel,
+ *  including without limitation         y compris sans restriction
+ *  any warranty of merchantability      toute garantie de valeur
+ *  or fitness for a particular          marchande ou de pertinence
+ *  purpose. NRC shall not be            pour un usage particulier.
+ *  liable in any event for any          Le CNRC ne pourra en aucun cas
+ *  damages, whether direct or           être tenu responsable de tout
+ *  indirect, special or general,        dommage, direct ou indirect,
+ *  consequential or incidental,         particulier ou général,
+ *  arising from the use of the          accessoire ou fortuit, résultant
+ *  software.  Neither the name          de l'utilisation du logiciel. Ni
+ *  of the National Research             le nom du Conseil National de
+ *  Council of Canada nor the            Recherches du Canada ni les noms
+ *  names of its contributors may        de ses  participants ne peuvent
+ *  be used to endorse or promote        être utilisés pour approuver ou
+ *  products derived from this           promouvoir les produits dérivés
+ *  software without specific prior      de ce logiciel sans autorisation
+ *  written permission.                  préalable et particulière
+ *                                       par écrit.
+ *
+ *  This file is part of the             Ce fichier fait partie du projet
+ *  OpenCADC project.                    OpenCADC.
+ *
+ *  OpenCADC is free software:           OpenCADC est un logiciel libre ;
+ *  you can redistribute it and/or       vous pouvez le redistribuer ou le
+ *  modify it under the terms of         modifier suivant les termes de
+ *  the GNU Affero General Public        la “GNU Affero General Public
+ *  License as published by the          License” telle que publiée
+ *  Free Software Foundation,            par la Free Software Foundation
+ *  either version 3 of the              : soit la version 3 de cette
+ *  License, or (at your option)         licence, soit (à votre gré)
+ *  any later version.                   toute version ultérieure.
+ *
+ *  OpenCADC is distributed in the       OpenCADC est distribué
+ *  hope that it will be useful,         dans l’espoir qu’il vous
+ *  but WITHOUT ANY WARRANTY;            sera utile, mais SANS AUCUNE
+ *  without even the implied             GARANTIE : sans même la garantie
+ *  warranty of MERCHANTABILITY          implicite de COMMERCIALISABILITÉ
+ *  or FITNESS FOR A PARTICULAR          ni d’ADÉQUATION À UN OBJECTIF
+ *  PURPOSE.  See the GNU Affero         PARTICULIER. Consultez la Licence
+ *  General Public License for           Générale Publique GNU Affero
+ *  more details.                        pour plus de détails.
+ *
+ *  You should have received             Vous devriez avoir reçu une
+ *  a copy of the GNU Affero             copie de la Licence Générale
+ *  General Public License along         Publique GNU Affero avec
+ *  with OpenCADC.  If not, see          OpenCADC ; si ce n’est
+ *  <http://www.gnu.org/licenses/>.      pas le cas, consultez :
+ *                                       <http://www.gnu.org/licenses/>.
+ *
+ *
+ ************************************************************************
+ */
+
+package ca.nrc.cadc.dlm.server;
+
+public class TARPackageServlet extends PackageServlet {
+    public static final String TAR_PACKAGE_TARGET = "/tar-package";
+    private static final String TAR_CONTENT_TYPE = "application/x-tar";
+
+
+    @Override
+    String getContentType() {
+        return TARPackageServlet.TAR_CONTENT_TYPE;
+    }
+}

cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/ZIPPackageServlet.java

@@ -0,0 +1,80 @@
+/*
+ ************************************************************************
+ *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
+ **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
+ *
+ *  (c) 2024.                            (c) 2024.
+ *  Government of Canada                 Gouvernement du Canada
+ *  National Research Council            Conseil national de recherches
+ *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
+ *  All rights reserved                  Tous droits réservés
+ *
+ *  NRC disclaims any warranties,        Le CNRC dénie toute garantie
+ *  expressed, implied, or               énoncée, implicite ou légale,
+ *  statutory, of any kind with          de quelque nature que ce
+ *  respect to the software,             soit, concernant le logiciel,
+ *  including without limitation         y compris sans restriction
+ *  any warranty of merchantability      toute garantie de valeur
+ *  or fitness for a particular          marchande ou de pertinence
+ *  purpose. NRC shall not be            pour un usage particulier.
+ *  liable in any event for any          Le CNRC ne pourra en aucun cas
+ *  damages, whether direct or           être tenu responsable de tout
+ *  indirect, special or general,        dommage, direct ou indirect,
+ *  consequential or incidental,         particulier ou général,
+ *  arising from the use of the          accessoire ou fortuit, résultant
+ *  software.  Neither the name          de l'utilisation du logiciel. Ni
+ *  of the National Research             le nom du Conseil National de
+ *  Council of Canada nor the            Recherches du Canada ni les noms
+ *  names of its contributors may        de ses  participants ne peuvent
+ *  be used to endorse or promote        être utilisés pour approuver ou
+ *  products derived from this           promouvoir les produits dérivés
+ *  software without specific prior      de ce logiciel sans autorisation
+ *  written permission.                  préalable et particulière
+ *                                       par écrit.
+ *
+ *  This file is part of the             Ce fichier fait partie du projet
+ *  OpenCADC project.                    OpenCADC.
+ *
+ *  OpenCADC is free software:           OpenCADC est un logiciel libre ;
+ *  you can redistribute it and/or       vous pouvez le redistribuer ou le
+ *  modify it under the terms of         modifier suivant les termes de
+ *  the GNU Affero General Public        la “GNU Affero General Public
+ *  License as published by the          License” telle que publiée
+ *  Free Software Foundation,            par la Free Software Foundation
+ *  either version 3 of the              : soit la version 3 de cette
+ *  License, or (at your option)         licence, soit (à votre gré)
+ *  any later version.                   toute version ultérieure.
+ *
+ *  OpenCADC is distributed in the       OpenCADC est distribué
+ *  hope that it will be useful,         dans l’espoir qu’il vous
+ *  but WITHOUT ANY WARRANTY;            sera utile, mais SANS AUCUNE
+ *  without even the implied             GARANTIE : sans même la garantie
+ *  warranty of MERCHANTABILITY          implicite de COMMERCIALISABILITÉ
+ *  or FITNESS FOR A PARTICULAR          ni d’ADÉQUATION À UN OBJECTIF
+ *  PURPOSE.  See the GNU Affero         PARTICULIER. Consultez la Licence
+ *  General Public License for           Générale Publique GNU Affero
+ *  more details.                        pour plus de détails.
+ *
+ *  You should have received             Vous devriez avoir reçu une
+ *  a copy of the GNU Affero             copie de la Licence Générale
+ *  General Public License along         Publique GNU Affero avec
+ *  with OpenCADC.  If not, see          OpenCADC ; si ce n’est
+ *  <http://www.gnu.org/licenses/>.      pas le cas, consultez :
+ *                                       <http://www.gnu.org/licenses/>.
+ *
+ *
+ ************************************************************************
+ */
+
+package ca.nrc.cadc.dlm.server;
+
+public class ZIPPackageServlet extends PackageServlet {
+    public static final String ZIP_PACKAGE_TARGET = "/zip-package";
+    private static final String ZIP_CONTENT_TYPE = "application/zip";
+
+
+    @Override
+    String getContentType() {
+        return ZIPPackageServlet.ZIP_CONTENT_TYPE;
+    }
+}