From d544578a92f951062dfca496b9dd539b4c6c9838 Mon Sep 17 00:00:00 2001 From: Vasili Gulevich Date: Sat, 21 Oct 2023 16:43:26 +0400 Subject: [PATCH] Follow P2 contract of cached file's extension (#2938) This is a backport from 5.0.0 P2 relies on correct file extensions to parse cached files. This change prevents cached file name from changing over HTTP redirect. See https://github.com/eclipse-equinox/p2/issues/355 Fix HttpServer concurrency bug. URLs returned from HttpServer.getAccessedUrls() are now stripped of context prefix. No callers have used these values until now. Fix concurrency bug in test utility class HttpServer request logging. Add verification of request logs to ensure that metadata repository does indeed unjars fresh, non-cached artifact. Ensure that cached XML file does not produce false negative in tests. Refactor tests. --- .../transport/SharedHttpCacheStorage.java | 10 +- .../projects/target.content_jar/category.xml | 4 + tycho-its/projects/target.content_jar/pom.xml | 49 +++++++ .../target.content_jar/targetplatform.target | 11 ++ .../repositories/content_jar/artifacts.jar | Bin 0 -> 580 bytes .../repositories/content_jar/content.jar | Bin 0 -> 988 bytes ...sue_2938_reproducer_1.0.0.202310211419.jar | Bin 0 -> 356 bytes .../tycho/test/target/OfflineModeTest.java | 2 +- .../tycho/test/tycho2938/ContentJarTest.java | 131 ++++++++++++++++++ .../eclipse/tycho/test/util/HttpServer.java | 115 ++++++++++----- 10 files changed, 287 insertions(+), 35 deletions(-) create mode 100644 tycho-its/projects/target.content_jar/category.xml create mode 100644 tycho-its/projects/target.content_jar/pom.xml create mode 100644 tycho-its/projects/target.content_jar/targetplatform.target create mode 100644 tycho-its/repositories/content_jar/artifacts.jar create mode 100644 tycho-its/repositories/content_jar/content.jar create mode 100644 tycho-its/repositories/content_jar/features/issue_2938_reproducer_1.0.0.202310211419.jar create mode 100644 tycho-its/src/test/java/org/eclipse/tycho/test/tycho2938/ContentJarTest.java diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java index ca3e8cc54d..32237fc0e8 100644 --- a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java @@ -262,8 +262,16 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor } updateHeader(response, code); if (isRedirected(code)) { - return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) + File cachedFile = SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) .getCacheFile(transportFactory); + // https://github.com/eclipse-tycho/tycho/issues/2938 + // Redirect may change extension. P2's SimpleMetadataRepositoryFactory relies on + // accurate file extension to be cached. + // Copying file to accommodate original request and its file extension. + // Once https://github.com/eclipse-equinox/p2/issues/355 is fixed, cachedFile + // may be returned directly without copying. + FileUtils.copyFile(cachedFile, file); + return file; } if (exits) { FileUtils.forceDelete(file); diff --git a/tycho-its/projects/target.content_jar/category.xml b/tycho-its/projects/target.content_jar/category.xml new file mode 100644 index 0000000000..7eaddf9158 --- /dev/null +++ b/tycho-its/projects/target.content_jar/category.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tycho-its/projects/target.content_jar/pom.xml b/tycho-its/projects/target.content_jar/pom.xml new file mode 100644 index 0000000000..a90b318d2e --- /dev/null +++ b/tycho-its/projects/target.content_jar/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + tycho-its-project.p2Repository.slicerDependencies + aggregator + 1.0.0 + eclipse-repository + + + UTF-8 + + + + + + org.eclipse.tycho + tycho-maven-plugin + ${tycho-version} + true + + + org.eclipse.tycho + target-platform-configuration + ${tycho-version} + + p2 + JavaSE-11 + + targetplatform.target + + + + win32 + win32 + x86_64 + + + linux + gtk + x86_64 + + + + + + + + diff --git a/tycho-its/projects/target.content_jar/targetplatform.target b/tycho-its/projects/target.content_jar/targetplatform.target new file mode 100644 index 0000000000..1f4ffb4d15 --- /dev/null +++ b/tycho-its/projects/target.content_jar/targetplatform.target @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/tycho-its/repositories/content_jar/artifacts.jar b/tycho-its/repositories/content_jar/artifacts.jar new file mode 100644 index 0000000000000000000000000000000000000000..5ea52128514c053d9db74b60e76d861d64e46476 GIT binary patch literal 580 zcmWIWW@Zs#;Nak3NGS>pXFvkH3@i+ZMJ1VOiOD6!dKI}j|4uP5^ah_Tx@91+@3V-N zL1@v6p4>?XuCxXJW|;frjK;M?IquU!{%#RyD__rZLM?Zp?EEdO-@DH*U;T93)FoHN7(e9Z7>zm(`3utB_S9jxtiBS#p2Ni z-A&@he_ilwThw%1Y|itwohC7D5ohF|`)`i?X)xD+{xXx@T{$J$Iu`7U+N-1=w5-^4 z@W{tIMafA$Cj)27+*D{>6IivubnkD0-WsLy-Y4fCN+tX5=G}CCK}*Z;YOj(HYCmT6 zeUDYr50f)eKFl`t^s#D@_Nx{3evc3AE4+RC|MWB3T6yK+B7SG>S^mxb^XWO8@(%WG zZ10-)ep^!f{o3qRI;vGROC3FbKGVE^MM1poMXB?Y1uHbI#1lf#oW7}5wJEBFO{aFd zALM_2=Hd-kYa8w*ucoZAi=@_E8r~o7S1H4(;KuQ>a&>l#?2O7)3007~|@^SzG literal 0 HcmV?d00001 diff --git a/tycho-its/repositories/content_jar/content.jar b/tycho-its/repositories/content_jar/content.jar new file mode 100644 index 0000000000000000000000000000000000000000..a69c2002cdb62c7d499b3fda2537cd2de4d62347 GIT binary patch literal 988 zcmWIWW@Zs#;Nak3NGS>pXFvkn3@i-E`FSO&c_n%kxjFw%F)+N1xSc0uC{TMo{70_z zN}caaqAIIqxvZMCFXU{ni0id?40&g7eO=m8Q+s@a_+}jzk5wUAN4YEXo}alIcl~Oc60T?B<}*>2hY$NsS#p%R*nXKTlUb{8>^$D%g2{lgKQ|g<^Ff+>htX z)>^|k_n^j(nh8#l9iD>X0-Gmf2xSR59hrGW;E?}mp^pNajx=~_@TY2@5jd=rS)r?R zYJW}ur;Xmho1BWyD_dAnB{CM>R9>`o+Ur(T6lEPn` z^eAypWBn5m8cJtGuFT0)xYn+cyx>@pd;fv5cC9lGTHBImuG*p+`72)MT=fl|uNQ7U zQr$Amea7z>P9}%KJ_&0>Xx9}iCbn|6MGlkT=rj1-K+D4+k!(|9zS&9 z>}Fn&VdQxK_LRcKSM$uG&ZaH5tk}J?`r=QOAIPaKsZgx?n z=4?+pZ*&*zJTAAFA$q!xg+#hs@VbQ(vf6=;3H3A9=zHH1U2EZL;GlO#XM@>#eQD!c zmCt(WRxSGNw7IJK<(KI(YqQkFTe%Nbwtje5X|XWW=J+fr<@?uPfBtFH!Y&fe=jZp@ zZmqJbg05W~vw)l z^Hh%$tx1zt-t;v4e#>`zfi}nT68mQs(VYC?_Zau zembeUUcP<4?#|sx$&-?%edKHMNY(M*y7<`Rqx;u1uybcdy0nX!SNbRiudJ9k(Pd87 z*<(Cc-Yweh_vxPM^O|j(Kc?l1Rvp^*e*W|)O!2a-vr2P=&5D`>u6CF5e`5aN_{_rXL&wc*C^^4T?ORuCimH7+btmjEr-_ID}&CcP{oN&O0 snSnt^kbxn{x6`*Cz0B=?{kOD>^v<1?gKtmZA0G|xBNB{r; literal 0 HcmV?d00001 diff --git a/tycho-its/repositories/content_jar/features/issue_2938_reproducer_1.0.0.202310211419.jar b/tycho-its/repositories/content_jar/features/issue_2938_reproducer_1.0.0.202310211419.jar new file mode 100644 index 0000000000000000000000000000000000000000..9c1b4111bae105dbc4f5bf84a5d3dfbb984437d4 GIT binary patch literal 356 zcmWIWW@Zs#;Nak3NGS>pXFvknKz3ScVo7OHs$NBIPVK~lU55-rT)+SAn(%jv@7z|e zXz{+L2SsVBN~X299A>plFIiP@FUh-K1t z-=FvD&b(aJ^NBlTopZ;D#~r&C9J2bzPhLlpa$P0HxjQf7gUV5`< zsbuo}9LLiy)#ra6cC&QYRBJb< zrgxc?Lq~uAp$Oi$ZtzqIL@bEVL} z4L$!D0=(HdnC{Ca-2r;}3nK%N&&VXgfCv_3IZ&{m0;E6+@MdKLNiqVVC6E>c8pgl? E0OQYrumAu6 literal 0 HcmV?d00001 diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/target/OfflineModeTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/target/OfflineModeTest.java index cade4b8e20..7bdf1ea364 100644 --- a/tycho-its/src/test/java/org/eclipse/tycho/test/target/OfflineModeTest.java +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/target/OfflineModeTest.java @@ -92,7 +92,7 @@ private void runAndVerifyOnlineBuild(Verifier verifier) throws VerificationExcep verifier.executeGoal("integration-test"); verifier.verifyErrorFreeLog(); assertFalse(server.getAccessedUrls("test").isEmpty()); - server.getAccessedUrls("test").clear(); + server.clear(); } } diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/tycho2938/ContentJarTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/tycho2938/ContentJarTest.java new file mode 100644 index 0000000000..467924882f --- /dev/null +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/tycho2938/ContentJarTest.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * Copyright (c) 2023, 2023 Sonatype Inc. and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Vasili Gulevich - initial implementation + *******************************************************************************/ +package org.eclipse.tycho.test.tycho2938; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; + +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.io.FileUtils; +import org.apache.maven.it.VerificationException; +import org.apache.maven.it.Verifier; +import org.eclipse.tycho.test.AbstractTychoIntegrationTest; +import org.eclipse.tycho.test.util.HttpServer; +import org.eclipse.tycho.test.util.ResourceUtil; +import org.eclipse.tycho.test.util.TargetDefinitionUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +public class ContentJarTest extends AbstractTychoIntegrationTest { + private HttpServer server; + private static final String TARGET_FEATURE_PATH = "/features/issue_2938_reproducer_1.0.0.202310211419.jar"; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private Verifier verifier; + private String mainRepoUrl; + + @Before + public void startServer() throws Exception { + server = HttpServer.startServer(); + File repositoryRoot = temporaryFolder.getRoot(); + this.mainRepoUrl = server.addServer("repoA", repositoryRoot); + File originalResource = ResourceUtil.resolveTestResource("repositories/content_jar"); + FileUtils.copyDirectory(originalResource, repositoryRoot); + verifier = getVerifier("target.content_jar", false); + verifier.deleteArtifacts("p2.org.eclipse.update.feature", "issue_2938_reproducer", "1.0.0.202310211419"); + } + + @After + public void stopServer() throws Exception { + if (server != null) { + server.stop(); + } + } + + @Test + public void noRedirect() throws Exception { + configureRepositoryInTargetDefinition(mainRepoUrl); + verifier.executeGoal("package"); + verifier.verifyErrorFreeLog(); + assertVisited(TARGET_FEATURE_PATH); + } + + @Test + public void redirectKeepFilename() throws Exception { + String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath); + configureRepositoryInTargetDefinition(redirectedUrl); + verifier.executeGoal("package"); + verifier.verifyErrorFreeLog(); + assertVisited(TARGET_FEATURE_PATH); + } + + @Test + public void redirectToBadLocation() throws Exception { + String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath + "_invalid"); + configureRepositoryInTargetDefinition(redirectedUrl); + Assert.assertThrows(VerificationException.class, () -> verifier.executeGoal("package")); + verifier.verifyTextInLog("No repository found at " + redirectedUrl); + assertVisited("/content.jar_invalid"); + } + + @Test + public void redirectToMangledLocations() throws Exception { + File repositoryRoot = temporaryFolder.getRoot(); + mangleFileNames(repositoryRoot.toPath()); + + // https://github.com/eclipse-tycho/tycho/issues/2938 + // Redirect may change extension. + String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath + "_invalid"); + + configureRepositoryInTargetDefinition(redirectedUrl); + verifier.executeGoal("package"); + verifier.verifyErrorFreeLog(); + assertVisited("/content.jar_invalid"); + assertVisited(TARGET_FEATURE_PATH + "_invalid"); + } + + private void assertVisited(String path) { + List accessedUrls = server.getAccessedUrls("repoA"); + Assert.assertTrue(String.format("Path %s should be visited, %s were visited instead", path, accessedUrls), + accessedUrls.contains(path)); + } + + private void mangleFileNames(Path repositoryRoot) throws IOException { + Files.walkFileTree(repositoryRoot, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.move(file, file.getParent().resolve(file.getFileName() + "_invalid")); + return super.visitFile(file, attrs); + } + }); + } + + private void configureRepositoryInTargetDefinition(String url) + throws IOException, ParserConfigurationException, SAXException { + File platformFile = new File(verifier.getBasedir(), "targetplatform.target"); + TargetDefinitionUtil.setRepositoryURLs(platformFile, "repoA", url); + } + +} diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/util/HttpServer.java b/tycho-its/src/test/java/org/eclipse/tycho/test/util/HttpServer.java index 682e6fb942..d22a54bb3f 100644 --- a/tycho-its/src/test/java/org/eclipse/tycho/test/util/HttpServer.java +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/util/HttpServer.java @@ -14,13 +14,20 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; +import java.util.EnumSet; import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import javax.servlet.DispatcherType; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -35,43 +42,53 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.MultiMap; import org.eclipse.jetty.util.URIUtil; import org.eclipse.jetty.util.security.Constraint; import org.eclipse.jetty.util.security.Password; public class HttpServer { - private static class MonitoringServlet extends DefaultServlet { - private List accessedURIs = new ArrayList<>(); + private class Monitoring implements Filter { @Override - public String getInitParameter(String name) { - // no directory listing allowed - if ("dirAllowed".equals(name)) { - return "false"; - } else { - return super.getInitParameter(name); - } + public void init(FilterConfig filterConfig) throws ServletException { } - public List getAccessedURIs() { - return accessedURIs; + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + Request httprequest = Request.getBaseRequest(request); + String context = httprequest.getContextPath(); + assert context.startsWith("/"); + context = context.substring(1); + synchronized (contextName2accessedUrls) { + contextName2accessedUrls.add(context, httprequest.getServletPath()); + } + chain.doFilter(request, response); } @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - accessedURIs.add(((Request) request).getHttpURI().toString()); - super.doGet(request, response); + public void destroy() { + + } + } + + private static class RedirectServlet extends HttpServlet { + private final Function relativeUrlToNewUrl; + + public RedirectServlet(Function relativeUrlToNewUrl2) { + super(); + this.relativeUrlToNewUrl = relativeUrlToNewUrl2; } @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - accessedURIs.add(((Request) request).getHttpURI().toString()); - super.doPost(request, response); + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + resp.sendRedirect(relativeUrlToNewUrl.apply(req.getServletPath())); } + } private static final int BIND_ATTEMPTS = 20; @@ -80,7 +97,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) private final int port; - private final Map contextName2servletsMap = new HashMap<>(); + private final MultiMap contextName2accessedUrls = new MultiMap<>(); private ContextHandlerCollection contexts; @@ -153,18 +170,28 @@ public void stop() throws Exception { public String addServer(String contextName, final File content) { ServletContextHandler context = new ServletContextHandler(contexts, URIUtil.SLASH + contextName); + context.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false"); context.setResourceBase(content.getAbsolutePath()); + context.addServlet(DefaultServlet.class, URIUtil.SLASH); + registerContext(context); + return getUrl(contextName); + } - MonitoringServlet monitoringServlet = new MonitoringServlet(); - contextName2servletsMap.put(contextName, monitoringServlet); - context.addServlet(new ServletHolder(monitoringServlet), URIUtil.SLASH); - contexts.addHandler(context); - try { - context.start(); - } catch (Exception e) { - throw new RuntimeException(e); - } + /** + * + * @param contextName - a path prefix to handle + * @param relativeUrlToNewUrl - redirecting function. Takes a path within + * context starting with slash and return an new + * absolute URL to redirect to (Location header + * value). + * @return an URL prefix to redirect from + */ + public String addRedirect(String contextName, Function relativeUrlToNewUrl) { + ServletContextHandler context = new ServletContextHandler(contexts, URIUtil.SLASH + contextName); + context.addServlet(new ServletHolder(new RedirectServlet(relativeUrlToNewUrl)), URIUtil.SLASH); + registerContext(context); return getUrl(contextName); + } public String getUrl(String contextName) { @@ -172,7 +199,29 @@ public String getUrl(String contextName) { } public List getAccessedUrls(String contextName) { - return contextName2servletsMap.get(contextName).getAccessedURIs(); + synchronized (contextName2accessedUrls) { + return List.copyOf(contextName2accessedUrls.getOrDefault(contextName, Collections.emptyList())); + } + } + + /** + * Reset state. + * + * Clear access logs. Does not affect configuration. + **/ + public void clear() { + synchronized (contextName2accessedUrls) { + contextName2accessedUrls.clear(); + } } + private void registerContext(ServletContextHandler context) { + context.addFilter(new FilterHolder(new Monitoring()), "*", EnumSet.of(DispatcherType.REQUEST)); + contexts.addHandler(context); + try { + context.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } }