From 1ea70b718716a1714e30c8d045f7c0f87beb66be Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 18 Oct 2024 17:24:02 +0900 Subject: [PATCH] Replace littleproxy with jetty-proxy (to remove dependency on log4j) littleproxy has not been updated since 2017 and has various out of date dependencies. Since we're using jetty for HTTP server testing anyway, we may as well use it as the proxy too. --- engine/pom.xml | 6 -- modules/pom.xml | 18 ++-- .../modules/fetcher/FetchHTTPTest.java | 102 +++++++++--------- 3 files changed, 61 insertions(+), 65 deletions(-) diff --git a/engine/pom.xml b/engine/pom.xml index fb345c715..dd1606151 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -82,12 +82,6 @@ 2.3.1 runtime - - org.littleshoot - littleproxy - 0.4 - test - org.slf4j slf4j-api diff --git a/modules/pom.xml b/modules/pom.xml index ed7ab631b..6ebbbc6ed 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -50,16 +50,16 @@ ${jetty.version} - org.littleshoot - littleproxy - 0.4 + org.eclipse.jetty + jetty-servlet + ${jetty.version} + test + + + org.eclipse.jetty + jetty-proxy + ${jetty.version} test - - - org.slf4j - slf4j-log4j12 - - org.slf4j diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index ee02c3a6e..98ad98b0c 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -32,27 +32,18 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.io.UnsupportedEncodingException; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; +import java.net.*; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.*; import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ServerSocketFactory; import javax.net.ssl.SSLException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; @@ -72,15 +63,16 @@ import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; import org.bbottema.javasocksproxyserver.SocksServer; -import org.jboss.netty.handler.codec.http.HttpRequest; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.proxy.ProxyServlet; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.littleshoot.proxy.DefaultHttpProxyServer; -import org.littleshoot.proxy.HttpFilter; -import org.littleshoot.proxy.HttpRequestFilter; -import org.littleshoot.proxy.ProxyAuthorizationHandler; public class FetchHTTPTest { @@ -440,27 +432,51 @@ public void tryHttpBindAddress(String addr) throws Exception { runDefaultChecks(curi, "httpBindAddress"); } - protected static class ProxiedRequestRememberer implements HttpRequestFilter { - protected HttpRequest lastProxiedRequest = null; - public HttpRequest getLastProxiedRequest() { - return lastProxiedRequest; + public static class TestProxyServlet extends ProxyServlet { + @Override + protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) { + super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse); + proxyResponse.addHeader("Via", "test-proxy-servlet"); } @Override - public void filter(HttpRequest httpRequest) { - lastProxiedRequest = httpRequest; + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String expectedUser = (String) getServletContext().getAttribute("proxy-user"); + if (expectedUser != null) { + String expectedPassword = (String) getServletContext().getAttribute("proxy-password"); + String expectedHeader = "Basic " + Base64.getEncoder().encodeToString((expectedUser + ":" + expectedPassword).getBytes()); + String authHeader = request.getHeader("Proxy-Authorization"); + if (!expectedHeader.equals(authHeader)) { + response.setHeader("Proxy-Authenticate", "Basic realm=\"test\""); + response.sendError(407); + return; + } + } + super.service(request, response); } + } - public void clear() { - lastProxiedRequest = null; - } + private static Server newHttpProxy() { + return newHttpProxy(null, null); + } + + private static Server newHttpProxy(String user, String password) { + Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877)); + ConnectHandler connectHandler = new ConnectHandler(); + httpProxyServer.setHandler(connectHandler); + ServletContextHandler context = new ServletContextHandler(connectHandler, "/", + ServletContextHandler.SESSIONS); + ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class); + context.addServlet(proxyServlet, "/*"); + context.setAttribute("proxy-user", user); + context.setAttribute("proxy-password", password); + return httpProxyServer; } @Test public void testHttpProxy() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(null, null); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -471,13 +487,10 @@ public void testHttpProxy() throws Exception { String requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); - assertNotNull(curi.getHttpResponseHeader("Via")); + assertEquals("test-proxy-servlet", curi.getHttpResponseHeader("Via")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - // check that our little proxy server really handled a request - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); - runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -486,16 +499,8 @@ public void testHttpProxy() throws Exception { @Test public void testHttpProxyAuth() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() { - @Override - public boolean authenticate(String userName, String password) { - // logger.info("username=" + userName + " password=" + password); - return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password); - } - }); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy("http-proxy-user", "http-proxy-password"); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -511,12 +516,10 @@ public boolean authenticate(String userName, String password) { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate")); assertEquals(407, curi.getFetchStatus()); // fetch original again now that credentials should be populated - proxiedRequestRememberer.clear(); curi = makeCrawlURI("http://localhost:7777/"); fetcher().process(curi); @@ -524,7 +527,6 @@ public boolean authenticate(String userName, String password) { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); assertNotNull(curi.getHttpResponseHeader("Via")); - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -533,8 +535,8 @@ public boolean authenticate(String userName, String password) { @Test public void testHttpsProxy() throws Exception { - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost");