diff --git a/.gitignore b/.gitignore
index 3b625ee7b..5c3213123 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,18 @@ target
/.classpath
/.settings
/.project
+*/.classpath
+*/.settings
+*/.project
/.idea
atlassian-ide-plugin.xml
-.DS_Store
\ No newline at end of file
+.DS_Store
+dependency-reduced-pom.xml
+littleproxy_cert
+littleproxy_keystore.jks
+
+# mockserver-related files
+ClientCertificate.pem
+ClientPrivateKey.pem
+ClientPublicKey.pem
+mockserver_keystore.jks
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000..016ceb70b
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,12 @@
+sudo: false
+
+language: java
+jdk:
+ # Not running tests against openjdk7, since the SunEC is not included in travis-ci's version of openjdk7.
+ # Not running tests against oraclejdk7, since travis-ci no longer provides it.
+ # - openjdk7
+ - oraclejdk8
+
+cache:
+ directories:
+ - $HOME/.m2
diff --git a/README.md b/README.md
index 5502aacbe..e932b282c 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,96 @@
-BrowserMob Proxy
-================
+# BrowserMob Proxy
-BrowserMob Proxy is a simple utility that makes it easy to capture performance data from browsers, typically written using automation toolkits such as Selenium and Watir.
+BrowserMob Proxy allows you to manipulate HTTP requests and responses, capture HTTP content, and export performance data as a [HAR file](http://www.softwareishard.com/blog/har-12-spec/).
+BMP works well as a standalone proxy server, but it is especially useful when embedded in Selenium tests.
-Features
---------
+The latest version of BrowserMob Proxy is 2.1.5, powered by [LittleProxy](https://github.com/adamfisk/LittleProxy).
-The proxy is programmatically controlled via a REST interface or by being embedded directly inside Java-based programs and unit tests. It captures performance data the [HAR format](http://groups.google.com/group/http-archive-specification). It addition it also can actually control HTTP traffic, such as:
+If you're running BrowserMob Proxy within a Java application or Selenium test, get started with [Embedded Mode](#getting-started-embedded-mode). If you want to run BMP from the
+command line as a standalone proxy, start with [Standalone](#getting-started-standalone).
+
+### Getting started: Embedded Mode
+To use BrowserMob Proxy in your tests or application, add the `browsermob-core` dependency to your pom:
+```xml
+
+ net.lightbody.bmp
+ browsermob-core
+ 2.1.5
+ test
+
+```
+
+Start the proxy:
+```java
+ BrowserMobProxy proxy = new BrowserMobProxyServer();
+ proxy.start(0);
+ int port = proxy.getPort(); // get the JVM-assigned port
+ // Selenium or HTTP client configuration goes here
+```
+
+Then configure your HTTP client to use a proxy running at the specified port.
+
+**Using with Selenium?** See the [Using with Selenium](#using-with-selenium) section.
+
+### Getting started: Standalone
+To run in standalone mode from the command line, first download the latest release from the [releases page](https://github.com/lightbody/browsermob-proxy/releases), or [build the latest from source](#building-the-latest-from-source).
+
+Start the REST API:
+```sh
+ ./browsermob-proxy -port 8080
+```
+
+Then create a proxy server instance:
+```sh
+ curl -X POST http://localhost:8080/proxy
+ {"port":8081}
+```
+
+The "port" is the port of the newly-created proxy instance, so configure your HTTP client or web browser to use a proxy on the returned port.
+For more information on the features available in the REST API, see [the REST API documentation](#rest-api).
+
+## Changes since 2.0.0
+
+The new [BrowserMobProxyServer class](browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java) has replaced the legacy ProxyServer implementation. The legacy implementation is no longer actively supported; all new code should use `BrowserMobProxyServer`. We highly recommend that existing code migrate to the new implementation.
+
+The most important changes from 2.0 are:
+
+- [Separate REST API and Embedded Mode modules](#embedded-mode). Include only the functionality you need.
+- [New BrowserMobProxy interface](browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java). The new interface will completely replace the legacy 2.0 ProxyServer contract in version 3.0 and higher.
+- [LittleProxy support](#littleproxy-support). More stable and more powerful than the legacy Jetty back-end.
+
+### New BrowserMobProxy API
+
+BrowserMob Proxy 2.1 includes a [new BrowserMobProxy interface](browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java) to interact with BrowserMob Proxy programmatically. The new interface defines the functionality that BrowserMob Proxy will support in future releases (including 3.0+). To ease migration, both the legacy (Jetty-based) ProxyServer class and the new, LittleProxy-powered BrowserMobProxy class support the new BrowserMobProxy interface.
+
+We _highly_ recommend migrating existing code to the BrowserMobProxy interface using the `BrowserMobProxyServer` class.
+
+### Using the LittleProxy implementation with 2.0.0 code
+
+The legacy interface, implicitly defined by the ProxyServer class, has been extracted into `net.lightbody.bmp.proxy.LegacyProxyServer` and is now officially deprecated. The new LittleProxy-based implementation will implement LegacyProxyServer for all 2.1.x releases. This means you can switch to the LittleProxy-powered implementation with minimal change to existing code ([with the exception of interceptors](#http-request-manipulation)):
+
+```java
+ // With the Jetty-based 2.0.0 release, BMP was created like this:
+ ProxyServer proxyServer = new ProxyServer();
+ proxyServer.start();
+ // [...]
+
+ // To use the LittleProxy-powered 2.1.5 release, simply change to
+ // the LegacyProxyServer interface and the adapter for the new
+ // LittleProxy-based implementation:
+ LegacyProxyServer proxyServer = new BrowserMobProxyServerLegacyAdapter();
+ proxyServer.start();
+ // Almost all deprecated 2.0.0 methods are supported by the
+ // new BrowserMobProxyServerLegacyAdapter implementation, so in most cases,
+ // no further code changes are necessary
+```
+
+LegacyProxyServer will not be supported after 3.0 is released, so we recommend migrating to the `BrowserMobProxy` interface as soon as possible. The new interface provides additional functionality and is compatible with both the legacy Jetty-based ProxyServer implementation [(with some exceptions)](new-interface-compatibility.md) and the new LittleProxy implementation.
+
+If you must continue using the legacy Jetty-based implementation, include the `browsermob-core-legacy` artifact instead of `browsermob-core`.
+
+## Features and Usage
+
+The proxy is programmatically controlled via a REST interface or by being embedded directly inside Java-based programs and unit tests. It captures performance data in the [HAR format](http://groups.google.com/group/http-archive-specification). In addition it can actually control HTTP traffic, such as:
- blacklisting and whitelisting certain URL patterns
- simulating various bandwidth and latency
@@ -15,183 +99,303 @@ The proxy is programmatically controlled via a REST interface or by being embedd
- controlling DNS and request timeouts
- automatic BASIC authorization
-REST API
---------
+### REST API
+
+**New in 2.1:** LittleProxy is the default implementation of the REST API. You may specify `--use-littleproxy false` to disable LittleProxy in favor of the legacy Jetty 5-based implementation.
To get started, first start the proxy by running `browsermob-proxy` or `browsermob-proxy.bat` in the bin directory:
- $ sh browsermob-proxy -port 9090
+ $ sh browsermob-proxy -port 8080
INFO 05/31 03:12:48 o.b.p.Main - Starting up...
2011-05-30 20:12:49.517:INFO::jetty-7.3.0.v20110203
2011-05-30 20:12:49.689:INFO::started o.e.j.s.ServletContextHandler{/,null}
- 2011-05-30 20:12:49.820:INFO::Started SelectChannelConnector@0.0.0.0:9090
+ 2011-05-30 20:12:49.820:INFO::Started SelectChannelConnector@0.0.0.0:8080
Once started, there won't be an actual proxy running until you create a new proxy. You can do this by POSTing to /proxy:
- [~]$ curl -X POST http://localhost:9090/proxy
- {"port":9091}
+ [~]$ curl -X POST http://localhost:8080/proxy
+ {"port":8081}
or optionally specify your own port:
- [~]$ curl -X POST -d 'port=9099' http://localhost:9090/proxy
- {"port":9099}
-
-Once that is done, a new proxy will be available on the port returned. All you have to do is point a browser to that proxy on that port and you should be able to browser the internet. The following additional APIs will then be available:
-
- - PUT /proxy/[port]/har - creates a new HAR attached to the proxy and returns the HAR content if there was a previous HAR. Supports the following parameters:
- - initialPageRef - the string name of the first page ref that should be used in the HAR. Defaults to "Page 1".
- - captureHeaders - Boolean, capture headers
- - captureContent - Boolean, capture content bodies
- - captureBinaryContent - Boolean, capture binary content
- - PUT /proxy/[port]/har/pageRef - starts a new page on the existing HAR. Supports the following parameters:
- - pageRef - the string name of the first page ref that should be used in the HAR. Defaults to "Page N" where N is the next page number.
- - PUT /proxy/[port]/har/pageRef - creates a new HAR attached to the proxy and returns the HAR content if there was a previous HAR
- - DELETE /proxy/[port] - shuts down the proxy and closes the port
- - GET /proxy/[port]/har - returns the JSON/HAR content representing all the HTTP traffic passed through the proxy
- - PUT /proxy/[port]/whitelist - Sets a list of URL patterns to whitelist. Takes the following parameters:
- - regex - a comma separated list of regular expressions
- - status - the HTTP status code to return for URLs that do not match the whitelist
- - PUT /proxy/[port]/blacklist - Set a URL to blacklist. Takes the following parameters:
- - regex - the blacklist regular expression
- - status - the HTTP status code to return for URLs that are blacklisted
- - PUT /proxy/[port]/limit - Limit the bandwidth through the proxy. Takes the following parameters:
- - downstreamKbps - Sets the downstream kbps
- - upstreamKbps - Sets the upstream kbps
- - latency - Add the given latency to each HTTP request
- - enable - (true/false) a boolean that enable bandwidth limiter. By default the limit is disabled, although setting any of the properties above will implicitly enable throttling
- - payloadPercentage - a number ]0, 100] specifying what percentage of data sent is payload. e.g. use this to take into account overhead due to tcp/ip.
- - maxBitsPerSecond - The max bits per seconds you want this instance of StreamManager to respect.
- - POST /proxy/[port]/headers - Set and override HTTP Request headers. For example setting a custom User-Agent.
- - Payload data should be json encoded set of headers (not url-encoded)
- - POST /proxy/[port]/hosts - Overrides normal DNS lookups and remaps the given hosts with the associated IP address
- - Payload data should be json encoded set of name/value pairs (ex: {"example.com": "1.2.3.4"})
- - POST /proxy/[port]/auth/basic/[domain] - Sets automatic basic authentication for the specified domain
- - Payload data should be json encoded username and password name/value pairs (ex: {"username": "myUsername", "password": "myPassword"}
- - PUT /proxy/[port]/wait - wait till all request are being made
- - quietPeriodInMs - Sets quiet period in milliseconds
- - timeoutInMs - Sets timeout in milliseconds
- - PUT /proxy/[port]/timeout - Handles different proxy timeouts. Takes the following parameters:
- - requestTimeout - request timeout in milliseconds
- - readTimeout - read timeout in milliseconds. Which is the timeout for waiting for data or, put differently, a maximum period inactivity between two consecutive data packets). A timeout value of zero is interpreted as an infinite timeout.
- - connectionTimeout - Determines the timeout in milliseconds until a connection is established. A timeout value of zero is interpreted as an infinite timeout.
- - dnsCacheTimeout - Sets the maximum length of time that records will be stored in this Cache. A negative value disables this feature (that is, sets no limit).
- - PUT /proxy/[port]/rewrite - Redirecting URL's
- - matchRegex - a matching URL regular expression
- - replace - replacement URL
- - PUT /proxy/[port]/retry - Setting the retry count
- - retrycount - the number of times a method will be retried
- - DELETE /proxy/[port]/dns/cache - Empties the Cache.
+ [~]$ curl -X POST -d 'port=8089' http://localhost:8080/proxy
+ {"port":8089}
+
+or if running BrowserMob Proxy in a multi-homed environment, specify a desired bind address (default is `0.0.0.0`):
+
+ [~]$ curl -X POST -d 'bindAddress=192.168.1.222' http://localhost:8080/proxy
+ {"port":8086}
+
+Once that is done, a new proxy will be available on the port returned. All you have to do is point a browser to that proxy on that port and you should be able to browse the internet. The following additional APIs will then be available:
+
+Description | HTTP method | Request path | Request parameters
+--- | :---: | :---: | ---
+Get a list of ports attached to `ProxyServer` instances managed by `ProxyManager` | GET | */proxy* ||
+Creates a new proxy to run requests off of | POST | */proxy* |
*port* - Integer, The specific port to start the proxy service on. Optional, default is generated and returned in response.
*proxyUsername* - String, The username to use to authenticate with the chained proxy. Optional, default to null.
*proxyPassword* - String, The password to use to authenticate with the chained proxy. Optional, default to null.
*bindAddress* - String, If running BrowserMob Proxy in a multi-homed environment, specify a desired bind address. Optional, default to "0.0.0.0".
*serverBindAddress* - String, If running BrowserMob Proxy in a multi-homed environment, specify a desired server bind address. Optional, default to "0.0.0.0".
*useEcc* - Boolean. True, Uses Elliptic Curve Cryptography for certificate impersonation. Optional, default to "false".
*trustAllServers* - Boolean. True, Disables verification of all upstream servers' SSL certificates. All upstream servers will be trusted, even if they do not present valid certificates signed by certification authorities in the JDK's trust store. Optional, default to "false".
|
+Creates a new HAR attached to the proxy and returns the HAR content if there was a previous HAR. *[port]* in request path it is port where your proxy was started | PUT |*/proxy/[port]/har* |
*captureHeaders* - Boolean, capture headers or not. Optional, default to "false".
*captureCookies* - Boolean, capture cookies or not. Optional, default to "false".
*captureContent* - Boolean, capture content bodies or not. Optional, default to "false".
*captureBinaryContent* - Boolean, capture binary content or not. Optional, default to "false".
*initialPageRef* - The string name of The first page ref that should be used in the HAR. Optional, default to "Page 1".
*initialPageTitle* - The title of first HAR page. Optional, default to *initialPageRef*.
+Starts a new page on the existing HAR. *[port]* in request path it is port where your proxy was started | PUT | */proxy/[port]/har/pageRef* |
*pageRef* - The string name of the first page ref that should be used in the HAR. Optional, default to "Page N" where N is the next page number.
*pageTitle* - The title of new HAR page. Optional, default to `pageRef`.
+Shuts down the proxy and closes the port. *[port]* in request path it is port where your proxy was started | DELETE | */proxy/[port]* ||
+Returns the JSON/HAR content representing all the HTTP traffic passed through the proxy (provided you have already created the HAR with [this method](#harcreate)) | GET | */proxy/[port]/har* ||
+Displays whitelisted items | GET | */proxy/[port]/whitelist* ||
+Sets a list of URL patterns to whitelist | PUT | */proxy/[port]/whitelist* |
*regex* - A comma separated list of regular expressions.
*status* - The HTTP status code to return for URLs that do not match the whitelist.
|
+Clears all URL patterns from the whitelist | DELETE | */proxy/[port]/whitelist* ||
+Displays blacklisted items | GET | */proxy/[port]/blacklist* ||
+Set a URL to blacklist | PUT | */proxy/[port]/blacklist* |
*regex* - The blacklist regular expression.
*status* - The HTTP status code to return for URLs that are blacklisted.
*method* - The regular expression for matching HTTP method (GET, POST, PUT, etc). Optional, by default processing all HTTP method.
|
+Clears all URL patterns from the blacklist | DELETE | */proxy/[port]/blacklist* ||
+Limit the bandwidth through the proxy on the *[port]* | PUT | */proxy/[port]/limit* |
*downstreamKbps* - Sets the downstream bandwidth limit in kbps. Optional.
*upstreamKbps* - Sets the upstream bandwidth limit kbps. Optional, by default unlimited.
*downstreamMaxKB* - Specifies how many kilobytes in total the client is allowed to download through the proxy. Optional, by default unlimited.
*upstreamMaxKB* - Specifies how many kilobytes in total the client is allowed to upload through the proxy. Optional, by default unlimited.
*latency* - Add the given latency to each HTTP request. Optional, by default all requests are invoked without latency.
*enable* - A boolean that enable bandwidth limiter. Optional, by default to "false", but setting any of the properties above will implicitly enable throttling
*payloadPercentage* - Specifying what percentage of data sent is payload, e.g. use this to take into account overhead due to tcp/ip. Optional.
*maxBitsPerSecond* - The max bits per seconds you want this instance of StreamManager to respect. Optional.
+Displays the amount of data remaining to be uploaded/downloaded until the limit is reached | GET | */proxy/[port]/limit* ||
+Set and override HTTP Request headers | POST | */proxy/[port]/headers* | Payload data should be **JSON** encoded set of headers. Where key is a header name (such as "User-Agent") and value is a value of HTTP header to setup (such as "BrowserMob-Agent"). Example: `{"User-Agent": "BrowserMob-Agent"}`|
+Overrides normal DNS lookups and remaps the given hosts with the associated IP address | POST | */proxy/[port]/hosts* | Payload data should be **JSON** encoded set of hosts. Where key is a host name (such as "example.com") and value is a IP address which associatied with host hame (such as "1.2.3.4"'). Example: `{"example.com": "1.2.3.4"}`|
+Sets automatic basic authentication for the specified domain | POST | */proxy/[port]/auth/basic/[domain]* | Payload data should be **JSON** encoded username and password name/value pairs. Example: `{"username": "myUsername", "password": "myPassword"}`|
+Wait till all request are being made | PUT | */proxy/[port]/wait* |
*quietPeriodInMs* - Wait till all request are being made. Optional.
*timeoutInMs* - Sets quiet period in milliseconds. Optional.
|
+Handles different proxy timeouts | PUT | *proxy/[port]/timeout* |
Payload data should be **JSON** encoded set of parameters. Where key is a parameters name (such as "connectionTimeout") and value is a value of parameter to setup (such as "500")
*requestTimeout* - Request timeout in milliseconds. A timeout value of -1 is interpreted as infinite timeout. Optional, default to "-1".
*readTimeout* - Read timeout in milliseconds. Which is the timeout for waiting for data or, put differently, a maximum period inactivity between two consecutive data packets). A timeout value of zero is interpreted as an infinite timeout. Optional, default to "60000".
*connectionTimeout* - Determines the timeout in milliseconds until a connection is established. A timeout value of zero is interpreted as an infinite timeout. Optional, default to "60000".
*dnsCacheTimeout* - Sets the maximum length of time that records will be stored in this Cache. A nonpositive value disables this feature (that is, sets no limit). Optional, default to "0".
|
+Removes all URL redirection rules currently in effect | DELETE | */proxy/[port]/rewrite* ||
+Setting the retry count | PUT | */proxy/[port]/retry* |
*retrycount* - The number of times a method will be retried.
|
+Empties the DNS cache | DELETE | */proxy/[port]/dns/cache* ||
+| [REST API interceptors with LittleProxy](#interceptorsRESTapiLP) |||
+|Describe your own request interception | POST | */proxy/[port]/filter/request* | A string which determinates interceptor rules. See more [here](#interceptorsRESTapiLPRequestFilter) |
+|Describe your own response interception | POST | */proxy/[port]/filter/response* | A string which determinates interceptor rules. See more [here](#interceptorsRESTapiLPResponseFilter) |
+| [REST API with Legacy interceptors](#interceptorsRESTapiLegacy) ||||
+|Describe your own request interception | POST | */proxy/[port]/interceptor/request* | A string which determinates interceptor rules. See more [here](#interceptorsRESTapiLegacy) |
+|Describe your own response interception | POST | */proxy/[port]/interceptor/response* | A string which determinates interceptor rules. See more [here](#interceptorsRESTapiLegacy) |
For example, once you've started the proxy you can create a new HAR to start recording data like so:
- [~]$ curl -X PUT -d 'initialPageRef=Foo' http://localhost:8080/proxy/9091/har
+ [~]$ curl -X PUT -d 'initialPageRef=Foo' http://localhost:8080/proxy/8081/har
Now when traffic goes through port 9091 it will be attached to a page reference named "Foo". Consult the HAR specification for more info on what a "pageRef" is. You can also start a new pageRef like so:
- [~]$ curl -X PUT -d 'pageRef=Bar' http://localhost:8080/proxy/9091/har/pageRef
+ [~]$ curl -X PUT -d 'pageRef=Bar' http://localhost:8080/proxy/8081/har/pageRef
-That will ensure no more HTTP requests get attached to the old pageRef (Foo) and start getting attached to the new pageRef (Bar). You can also get the HAR content at any time like so:
+That will ensure no more HTTP requests get attached to the old pageRef (Foo) and start getting attached to the new pageRef (Bar). After creating the HAR, you can get its content at any time like so:
- [~]$ curl http://localhost:8080/proxy/9091/har
+ [~]$ curl http://localhost:8080/proxy/8081/har
Sometimes you will want to route requests through an upstream proxy server. In this case specify your proxy server by adding the httpProxy parameter to your create proxy request:
- [~]$ curl -X POST http://localhost:9090/proxy?httpProxy=yourproxyserver.com:8080
- {"port":9091}
+ [~]$ curl -X POST http://localhost:8080/proxy?httpProxy=yourproxyserver.com:8080
+ {"port":8081}
Alternatively, you can specify the upstream proxy config for all proxies created using the standard JVM [system properties for HTTP proxies](http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html).
Note that you can still override the default upstream proxy via the POST payload, but if you omit the payload the JVM
system properties will be used to specify the upstream proxy.
-*TODO*: Other REST APIs supporting all the BrowserMob Proxy features will be added soon.
-
-Embedded Mode
--------------
+### Command-line Arguments
-If you're using Java and Selenium, the easiest way to get started is to embed the project directly in your test. First, you'll need to make sure that all the dependencies are imported in to the project. You can find them in the *lib* directory. Or, if you're using Maven, you can add this to your pom:
-
-
- biz.neustar
- browsermob-proxy
- LATEST_VERSION (ex: 2.0-beta-7)
- test
-
+ - -port \
+ - Port on which the API listens. Default value is 8080.
+ - -address
+ - Address to which the API is bound. Default value is 0.0.0.0.
+ - -proxyPortRange \-\
+ - Range of ports reserved for proxies. Only applies if *port* parameter is not supplied in the POST request. Default values are \+1 to \+500+1.
+ - -ttl \
+ - Proxy will be automatically deleted after a specified time period. Off by default.
-Once done, you can start a proxy using `org.browsermob.proxy.ProxyServer`:
+### Embedded Mode
- ProxyServer server = new ProxyServer(9090);
- server.start();
+**New in 2.1:** New Embedded Mode module
-This class supports every feature that the proxy supports. In fact, the REST API is a subset of the methods exposed here, so new features will show up here before they show up in the REST API. Consult the Javadocs for the full API.
+**New in 2.1:** New [BrowserMobProxy interface](#new-browsermobproxy-api) for Embedded Mode
-If your project already defines a Selenium dependency then you may want to exclude the version that browsermob-proxy pulls in, like so:
+BrowserMob Proxy 2.1 separates the Embedded Mode and REST API into two modules. If you only need Embedded Mode functionality, add the `browsermob-core` artifact as a dependency. The REST API artifact is `browsermob-rest`.
+If you're using Java and Selenium, the easiest way to get started is to embed the project directly in your test. First, you'll need to make sure that all the dependencies are imported in to the project. You can find them in the *lib* directory. Or, if you're using Maven, you can add this to your pom:
+```xml
- biz.neustar
- browsermob-proxy
- LATEST_VERSION (ex: 2.0-beta-7)
+ net.lightbody.bmp
+ browsermob-core
+ 2.1.5test
-
-
- org.seleniumhq.selenium
- selenium-api
-
-
+```
+
+Once done, you can start a proxy using `net.lightbody.bmp.BrowserMobProxy`:
+```java
+ BrowserMobProxy proxy = new BrowserMobProxyServer();
+ proxy.start(0);
+ // get the JVM-assigned port and get to work!
+ int port = proxy.getPort();
+ //...
+```
+Consult the Javadocs on the `net.lightbody.bmp.BrowserMobProxy` class for the full API.
-Using With Selenium
--------------------
+### Using With Selenium
-You can use the REST API with Selenium however you want. But if you're writing your tests in Java and using Selenium 2, this is the easiest way to use it:
+**Selenium 3 users**: Due to a [geckodriver issue](https://github.com/mozilla/geckodriver/issues/97), Firefox 51 and lower do not properly support proxies with WebDriver's DesiredCapabilities. See [this answer](http://stackoverflow.com/a/41373808/4256475) for a suitable work-around.
+BrowserMob Proxy makes it easy to use a proxy in Selenium tests:
+```java
// start the proxy
- ProxyServer server = new ProxyServer(4444);
- server.start();
+ BrowserMobProxy proxy = new BrowserMobProxyServer();
+ proxy.start(0);
// get the Selenium proxy object
- Proxy proxy = server.seleniumProxy();
+ Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
// configure it as a desired capability
DesiredCapabilities capabilities = new DesiredCapabilities();
- capabilities.setCapability(CapabilityType.PROXY, proxy);
+ capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
// start the browser up
WebDriver driver = new FirefoxDriver(capabilities);
+ // enable more detailed HAR capture, if desired (see CaptureType for the complete list)
+ proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
+
// create a new HAR with the label "yahoo.com"
- server.newHar("yahoo.com");
+ proxy.newHar("yahoo.com");
// open yahoo.com
driver.get("http://yahoo.com");
// get the HAR data
- Har har = server.getHar();
+ Har har = proxy.getHar();
+```
+
+**Note**: If you're running running tests on a Selenium grid, you will need to customize the Selenium Proxy object
+created by `createSeleniumProxy()` to point to the hostname of the machine that your test is running on. You can also run a standalone
+BrowserMob Proxy instance on a separate machine and configure the Selenium Proxy object to use that proxy.
+### HTTP Request Manipulation
-HTTP Request Manipulation
--------------------
+**HTTP request manipulation has changed in 2.1.0+ with LittleProxy.** The LittleProxy-based interceptors are easier to use and more reliable. The legacy ProxyServer implementation **will not** support the new interceptor methods.
-While not yet available via the REST interface, you can manipulate the requests like so:
+#### 2.1.0+ (LittleProxy) interceptors
- server.addRequestInterceptor(new RequestInterceptor() {
+There are four new methods to support request and response interception in LittleProxy:
+
+ - `addRequestFilter`
+ - `addResponseFilter`
+ - `addFirstHttpFilterFactory`
+ - `addLastHttpFilterFactory`
+
+For most use cases, including inspecting and modifying requests/responses, `addRequestFilter` and `addResponseFilter` will be sufficient. The request and response filters are easy to use:
+```java
+ proxy.addRequestFilter(new RequestFilter() {
+ @Override
+ public HttpResponse filterRequest(HttpRequest request, HttpMessageContents contents, HttpMessageInfo messageInfo) {
+ if (messageInfo.getOriginalUri().endsWith("/some-endpoint-to-intercept")) {
+ // retrieve the existing message contents as a String or, for binary contents, as a byte[]
+ String messageContents = contents.getTextContents();
+
+ // do some manipulation of the contents
+ String newContents = messageContents.replaceAll("original-string", "my-modified-string");
+ //[...]
+
+ // replace the existing content by calling setTextContents() or setBinaryContents()
+ contents.setTextContents(newContents);
+ }
+
+ // in the request filter, you can return an HttpResponse object to "short-circuit" the request
+ return null;
+ }
+ });
+
+ // responses are equally as simple:
+ proxy.addResponseFilter(new ResponseFilter() {
+ @Override
+ public void filterResponse(HttpResponse response, HttpMessageContents contents, HttpMessageInfo messageInfo) {
+ if (/*...some filtering criteria...*/) {
+ contents.setTextContents("This message body will appear in all responses!");
+ }
+ }
+ });
+```
+
+With Java 8, the syntax is even more concise:
+```java
+ proxy.addResponseFilter((response, contents, messageInfo) -> {
+ if (/*...some filtering criteria...*/) {
+ contents.setTextContents("This message body will appear in all responses!");
+ }
+ });
+```
+
+See the javadoc for the `RequestFilter` and `ResponseFilter` classes for more information.
+
+For fine-grained control over the request and response lifecycle, you can add "filter factories" directly using `addFirstHttpFilterFactory` and `addLastHttpFilterFactory` (see the examples in the InterceptorTest unit tests).
+
+#### REST API interceptors with LittleProxy
+
+When running the REST API with LittleProxy enabled, you cannot use the legacy `/:port/interceptor/` endpoints. Instead, POST the javascript payload to the new `/:port/filter/request` and `/:port/filter/response` endpoints.
+
+##### Request filters
+
+Javascript request filters have access to the variables `request` (type `io.netty.handler.codec.http.HttpRequest`), `contents` (type `net.lightbody.bmp.util.HttpMessageContents`), and `messageInfo` (type `net.lightbody.bmp.util.HttpMessageInfo`). `messageInfo` contains additional information about the message, including whether the message is sent over HTTP or HTTPS, as well as the original request received from the client before any changes made by previous filters. If the javascript returns an object of type `io.netty.handler.codec.http.HttpResponse`, the HTTP request will "short-circuit" and return the response immediately.
+
+**Example: Modify User-Agent header**
+
+```sh
+curl -i -X POST -H 'Content-Type: text/plain' -d "request.headers().remove('User-Agent'); request.headers().add('User-Agent', 'My-Custom-User-Agent-String 1.0');" http://localhost:8080/proxy/8081/filter/request
+```
+
+##### Response filters
+
+Javascript response filters have access to the variables `response` (type `io.netty.handler.codec.http.HttpResponse`), `contents` (type `net.lightbody.bmp.util.HttpMessageContents`), and `messageInfo` (type `net.lightbody.bmp.util.HttpMessageInfo`). As in the request filter, `messageInfo` contains additional information about the message.
+
+**Example: Modify response body**
+
+```sh
+curl -i -X POST -H 'Content-Type: text/plain' -d "contents.setTextContents('Response successfully intercepted');" http://localhost:8080/proxy/8081/filter/response
+```
+
+#### Legacy interceptors
+
+If you are using the legacy ProxyServer implementation, you can manipulate the requests like so:
+```java
+ BrowserMobProxy server = new ProxyServer();
+ ((LegacyProxyServer)server).addRequestInterceptor(new RequestInterceptor() {
@Override
- public void process(BrowserMobHttpRequest request) {
+ public void process(BrowserMobHttpRequest request, Har har) {
request.getMethod().removeHeaders("User-Agent");
request.getMethod().addHeader("User-Agent", "Bananabot/1.0");
}
});
+```
+You can also POST a JavaScript payload to `/:port/interceptor/request` and `/:port/interceptor/response` using the REST interface. The functions will have a `request`/`response` variable, respectively, and a `har` variable (which may be null if a HAR isn't set up yet). The JavaScript code will be run by [Rhino](https://github.com/mozilla/rhino) and have access to the same Java API in the example above:
+
+ [~]$ curl -X POST -H 'Content-Type: text/plain' -d 'request.getMethod().removeHeaders("User-Agent");' http://localhost:8080/proxy/8081/interceptor/request
+
+Consult the Java API docs for more info.
+
+### SSL Support
+
+**BrowserMob Proxy 2.1.0+ now supports full MITM:** For most users, MITM will work out-of-the-box with default settings. Install the [ca-certificate-rsa.cer](/browsermob-core/src/main/resources/sslSupport/ca-certificate-rsa.cer) file in your browser or HTTP client to avoid untrusted certificate warnings. Generally, it is safer to generate your own private key, rather than using the .cer files distributed with BrowserMob Proxy. See the [README file in the `mitm` module](/mitm/README.md) for instructions on generating or using your own root certificate and private key with MITM.
+
+**Legacy Jetty-based ProxyServer support for MITM:** The legacy `ProxyServer` implementation uses the same `ca-certificate-rsa.cer` root certificate as the default BrowserMobProxyServer implementation. The previous cybervillainsCA.cer certificate has been removed.
+
+**Note: DO NOT** permanently install the .cer files distributed with BrowserMob Proxy in users' browsers. They should be used for testing only and must not be used with general web browsing.
-We will soon be adding support for this advanced capability in the REST interface as well, using JavaScript snippets that can be posted as the interceptor code.
+If you're doing testing with Selenium, you'll want to make sure that the browser profile that gets set up by Selenium not only has the proxy configured, but also has the CA installed. Unfortunately, there is no API for doing this in Selenium; it must be done manually for each browser and environment.
-SSL Support
------------
+### NodeJS Support
-While the proxy supports SSL, it requires that a Certificate Authority be installed in to the browser. This allows the browser to trust all the SSL traffic coming from the proxy, which will be proxied using a classic man-in-the-middle technique. IT IS CRITICAL THAT YOU NOT INSTALL THIS CERTIFICATE AUTHORITY ON A BROWSER THAT IS USED FOR ANYTHING OTHER THAN TESTING.
+NodeJS bindings for browswermob-proxy are available [here](https://github.com/zzo/browsermob-node). Built-in support for [Selenium](http://seleniumhq.org) or use [CapserJS-on-PhantomJS](http://casperjs.org) or anything else to drive traffic for HAR generation.
-If you're doing testing with Selenium, you'll want to make sure that the browser profile that gets set up by Selenium not only has the proxy configured, but also has the CA installed. Unfortuantely, there is no API for doing this in Selenium, so you'll have to solve it uniquely for each browser type. We hope to make this easier in upcoming releases.
+### Logging
-NodeJS Support
---------------
+When running in stand-alone mode, the proxy loads the default logging configuration from the conf/bmp-logging.yaml file. To increase/decrease the logging level, change the logging entry for net.lightbody.bmp.
-NodeJS bindings for browswermob-proxy are available [here](https://github.com/zzo/browsermob-node). Built-in support for [Selenium][http://seleniumhq.com] or use [CapserJS-on-PhantomJS](http://casperjs.org) or anything else to drive traffic for HAR generation.
+### DNS Resolution
+
+The BrowserMobProxyServer implementation uses native DNS resolution by default, but supports custom DNS resolution and advanced DNS manipulation. See the [ClientUtil](browsermob-proxy/browsermob-core/src/main/java/net/lightbody/bmp/client/ClientUtil.java) class for information on DNS manipulation using the dnsjava resolver.
+
+## Building the latest from source
+
+You'll need maven (`brew install maven` if you're on OS X):
+
+ [~]$ mvn -DskipTests
+
+You'll find the standalone BrowserMob Proxy distributable zip at `browsermob-dist/target/browsermob-proxy-2.1.5-SNAPSHOT-bin.zip`. Unzip the contents and run the `browsermob-proxy` or `browsermob-proxy.bat` files in the `bin` directory.
+
+When you build the latest code from source, you'll have access to the latest snapshot release. To use the SNAPSHOT version in your code, modify the version in your pom:
+```xml
+
+ net.lightbody.bmp
+ browsermob-core
+ 2.1.6-SNAPSHOT
+ test
+
+```
diff --git a/browsermob-core/pom.xml b/browsermob-core/pom.xml
new file mode 100644
index 000000000..2071434ee
--- /dev/null
+++ b/browsermob-core/pom.xml
@@ -0,0 +1,249 @@
+
+
+ jar
+
+
+ browsermob-proxy
+ net.lightbody.bmp
+ 2.1.6-SNAPSHOT
+
+ 4.0.0
+
+ browsermob-core
+ BrowserMob Proxy Core (LittleProxy) Module
+
+
+ 7.6.16.v20140903
+
+
+
+
+
+ src/main/resources
+ true
+
+ net/lightbody/bmp/version
+
+
+
+ src/main/resources
+ false
+
+ **/**
+
+
+ net/lightbody/bmp/version
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+ -Xmx1g -XX:MaxPermSize=256m
+
+
+
+
+
+
+
+ net.lightbody.bmp
+ littleproxy
+
+
+ com.barchart.udt
+ barchart-udt-bundle
+
+
+ commons-cli
+ commons-cli
+
+
+
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
+
+ com.google.guava
+ guava
+
+
+
+ dnsjava
+ dnsjava
+ 2.1.8
+
+
+
+ org.seleniumhq.selenium
+ selenium-api
+ true
+
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+ org.slf4j
+ jcl-over-slf4j
+
+
+
+
+ com.jcraft
+ jzlib
+
+
+ io.netty
+ netty
+
+
+
+
+
+
+ io.netty
+ netty-all
+
+
+
+ org.bouncycastle
+ bcprov-jdk15on
+
+
+
+ org.bouncycastle
+ bcpkix-jdk15on
+
+
+
+ net.lightbody.bmp
+ mitm
+ ${project.version}
+
+
+
+
+ org.javassist
+ javassist
+ true
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ test
+
+
+ org.apache.logging.log4j
+ log4j-core
+ test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ test
+
+
+ org.seleniumhq.selenium
+ selenium-firefox-driver
+ test
+
+
+ junit
+ junit
+ test
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
+
+ org.mock-server
+ mockserver-netty
+ test
+
+
+ ch.qos.logback
+ logback-classic
+
+
+
+ io.netty
+ netty-codec-socks
+
+
+ io.netty
+ netty-buffer
+
+
+ io.netty
+ netty-codec
+
+
+ io.netty
+ netty-codec-http
+
+
+ io.netty
+ netty-common
+
+
+ io.netty
+ netty-handler
+
+
+ io.netty
+ netty-transport
+
+
+
+
+
+ org.eclipse.jetty
+ jetty-server
+ ${unit-test-jetty.version}
+ test
+
+
+
+ org.eclipse.jetty
+ jetty-servlet
+ ${unit-test-jetty.version}
+ test
+
+
+
+ org.eclipse.jetty
+ jetty-servlets
+ ${unit-test-jetty.version}
+ test
+
+
+
+ org.hamcrest
+ hamcrest-library
+ test
+
+
+
\ No newline at end of file
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java
new file mode 100644
index 000000000..a04ffe7af
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java
@@ -0,0 +1,643 @@
+package net.lightbody.bmp;
+
+import net.lightbody.bmp.core.har.Har;
+import net.lightbody.bmp.filters.RequestFilter;
+import net.lightbody.bmp.filters.ResponseFilter;
+import net.lightbody.bmp.mitm.TrustSource;
+import net.lightbody.bmp.proxy.BlacklistEntry;
+import net.lightbody.bmp.proxy.CaptureType;
+import net.lightbody.bmp.proxy.auth.AuthType;
+import net.lightbody.bmp.proxy.dns.AdvancedHostResolver;
+import org.littleshoot.proxy.HttpFiltersSource;
+import org.littleshoot.proxy.MitmManager;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public interface BrowserMobProxy {
+ /**
+ * Starts the proxy on port 0 (a JVM-selected open port). The proxy will bind the listener to the wildcard address (0:0:0:0 - all network interfaces).
+ *
+ * @throws java.lang.IllegalStateException if the proxy has already been started
+ */
+ void start();
+
+ /**
+ * Starts the proxy on the specified port. The proxy will bind the listener to the wildcard address (0:0:0:0 - all network interfaces).
+ *
+ * @param port port to listen on
+ * @throws java.lang.IllegalStateException if the proxy has already been started
+ */
+ void start(int port);
+
+ /**
+ * Starts the proxy on the specified port. The proxy will listen for connections on the network interface specified by the bindAddress, and will
+ * also initiate connections to upstream servers on the same network interface.
+ *
+ * @param port port to listen on
+ * @param bindAddress address of the network interface on which the proxy will listen for connections and also attempt to connect to upstream servers.
+ * @throws java.lang.IllegalStateException if the proxy has already been started
+ */
+ void start(int port, InetAddress bindAddress);
+
+ /**
+ * Starts the proxy on the specified port. The proxy will listen for connections on the network interface specified by the clientBindAddress, and will
+ * initiate connections to upstream servers from the network interface specified by the serverBindAddress.
+ *
+ * @param port port to listen on
+ * @param clientBindAddress address of the network interface on which the proxy will listen for connections
+ * @param serverBindAddress address of the network interface on which the proxy will connect to upstream servers
+ * @throws java.lang.IllegalStateException if the proxy has already been started
+ */
+ void start(int port, InetAddress clientBindAddress, InetAddress serverBindAddress);
+
+ /**
+ * Returns true if the proxy is started and listening for connections, otherwise false.
+ */
+ boolean isStarted();
+
+ /**
+ * Stops accepting new client connections and initiates a graceful shutdown of the proxy server, waiting up to 5 seconds for network
+ * traffic to stop. If the proxy was previously stopped or aborted, this method has no effect.
+ *
+ * @throws java.lang.IllegalStateException if the proxy has not been started.
+ */
+ void stop();
+
+ /**
+ * Like {@link #stop()}, shuts down the proxy server and no longer accepts incoming connections, but does not wait for any existing
+ * network traffic to cease. Any existing connections to clients or to servers may be force-killed immediately.
+ * If the proxy was previously stopped or aborted, this method has no effect.
+ *
+ * @throws java.lang.IllegalStateException if the proxy has not been started
+ */
+ void abort();
+
+ /**
+ * Returns the address of the network interface on which the proxy is listening for client connections.
+ *
+ * @return the client bind address, or null if the proxy has not been started
+ */
+ InetAddress getClientBindAddress();
+
+ /**
+ * Returns the actual port on which the proxy is listening for client connections.
+ *
+ * @throws java.lang.IllegalStateException if the proxy has not been started
+ */
+ int getPort();
+
+ /**
+ * Returns the address address of the network interface the proxy will use to initiate upstream connections. If no server bind address
+ * has been set, this method returns null, even if the proxy has been started.
+ *
+ * @return server bind address if one has been set, otherwise null
+ */
+ InetAddress getServerBindAddress();
+
+ /**
+ * Retrieves the current HAR.
+ *
+ * @return current HAR, or null if HAR capture is not enabled
+ */
+ Har getHar();
+
+ /**
+ * Starts a new HAR file with the default page name (see {@link #newPage()}. Enables HAR capture if it was not previously enabled.
+ *
+ * @return existing HAR file, or null if none exists or HAR capture was disabled
+ */
+ Har newHar();
+
+ /**
+ * Starts a new HAR file with the specified initialPageRef as the page name and page title. Enables HAR capture if it was not previously enabled.
+ *
+ * @param initialPageRef initial page name of the new HAR file
+ * @return existing HAR file, or null if none exists or HAR capture was disabled
+ */
+ Har newHar(String initialPageRef);
+
+ /**
+ * Starts a new HAR file with the specified page name and page title. Enables HAR capture if it was not previously enabled.
+ *
+ * @param initialPageRef initial page name of the new HAR file
+ * @param initialPageTitle initial page title of the new HAR file
+ * @return existing HAR file, or null if none exists or HAR capture was disabled
+ */
+ Har newHar(String initialPageRef, String initialPageTitle);
+
+ /**
+ * Sets the data types that will be captured in the HAR file for future requests. Replaces any existing capture types with the specified
+ * capture types. A null or empty set will not disable HAR capture, but will disable collection of
+ * additional {@link net.lightbody.bmp.proxy.CaptureType} data types. {@link net.lightbody.bmp.proxy.CaptureType} provides several
+ * convenience methods to retrieve commonly-used capture settings.
+ *
+ * Note: HAR capture must still be explicitly enabled via {@link #newHar()} or {@link #newHar(String)} to begin capturing
+ * any request and response contents.
+ *
+ * @param captureTypes HAR data types to capture
+ */
+ void setHarCaptureTypes(Set captureTypes);
+
+ /**
+ * Sets the data types that will be captured in the HAR file for future requests. Replaces any existing capture types with the specified
+ * capture types. A null or empty set will not disable HAR capture, but will disable collection of
+ * additional {@link net.lightbody.bmp.proxy.CaptureType} data types. {@link net.lightbody.bmp.proxy.CaptureType} provides several
+ * convenience methods to retrieve commonly-used capture settings.
+ *
+ * Note: HAR capture must still be explicitly enabled via {@link #newHar()} or {@link #newHar(String)} to begin capturing
+ * any request and response contents.
+ *
+ * @param captureTypes HAR data types to capture
+ */
+ void setHarCaptureTypes(CaptureType... captureTypes);
+
+ /**
+ * @return A copy of HAR capture types currently in effect. The EnumSet cannot be used to modify the HAR capture types currently in effect.
+ */
+ EnumSet getHarCaptureTypes();
+
+ /**
+ * Enables the specified HAR capture types. Does not replace or disable any other capture types that may already be enabled.
+ *
+ * @param captureTypes capture types to enable
+ */
+ void enableHarCaptureTypes(Set captureTypes);
+
+ /**
+ * Enables the specified HAR capture types. Does not replace or disable any other capture types that may already be enabled.
+ *
+ * @param captureTypes capture types to enable
+ */
+ void enableHarCaptureTypes(CaptureType... captureTypes);
+
+ /**
+ * Disables the specified HAR capture types. Does not replace or disable any other capture types that may already be enabled.
+ *
+ * @param captureTypes capture types to disable
+ */
+ void disableHarCaptureTypes(Set captureTypes);
+
+ /**
+ * Disables the specified HAR capture types. Does not replace or disable any other capture types that may already be enabled.
+ *
+ * @param captureTypes capture types to disable
+ */
+ void disableHarCaptureTypes(CaptureType... captureTypes);
+
+ /**
+ * Starts a new HAR page using the default page naming convention. The default page naming convention is "Page #", where "#" resets to 1
+ * every time {@link #newHar()} or {@link #newHar(String)} is called, and increments on every subsequent call to {@link #newPage()} or
+ * {@link #newHar(String)}. Populates the {@link net.lightbody.bmp.core.har.HarPageTimings#onLoad} value based on the amount of time
+ * the current page has been captured.
+ *
+ * @return the HAR as it existed immediately after ending the current page
+ * @throws java.lang.IllegalStateException if HAR capture has not been enabled via {@link #newHar()} or {@link #newHar(String)}
+ */
+ Har newPage();
+
+ /**
+ * Starts a new HAR page using the specified pageRef as the page name and the page title. Populates the
+ * {@link net.lightbody.bmp.core.har.HarPageTimings#onLoad} value based on the amount of time the current page has been captured.
+ *
+ * @param pageRef name of the new page
+ * @return the HAR as it existed immediately after ending the current page
+ * @throws java.lang.IllegalStateException if HAR capture has not been enabled via {@link #newHar()} or {@link #newHar(String)}
+ */
+ Har newPage(String pageRef);
+
+ /**
+ * Starts a new HAR page using the specified pageRef as the page name and the pageTitle as the page title. Populates the
+ * {@link net.lightbody.bmp.core.har.HarPageTimings#onLoad} value based on the amount of time the current page has been captured.
+ *
+ * @param pageRef name of the new page
+ * @param pageTitle title of the new page
+ * @return the HAR as it existed immediately after ending the current page
+ * @throws java.lang.IllegalStateException if HAR capture has not been enabled via {@link #newHar()} or {@link #newHar(String)}
+ */
+ Har newPage(String pageRef, String pageTitle);
+
+ /**
+ * Stops capturing traffic in the HAR. Populates the {@link net.lightbody.bmp.core.har.HarPageTimings#onLoad} value for the current page
+ * based on the amount of time it has been captured.
+ *
+ * @return the existing HAR
+ */
+ Har endHar();
+
+ /**
+ * Sets the maximum bandwidth to consume when reading server responses.
+ *
+ * @param bytesPerSecond maximum bandwidth, in bytes per second
+ */
+ void setReadBandwidthLimit(long bytesPerSecond);
+
+ /**
+ * Returns the current bandwidth limit for reading, in bytes per second.
+ */
+ long getReadBandwidthLimit();
+
+ /**
+ * Sets the maximum bandwidth to consume when sending requests to servers.
+ *
+ * @param bytesPerSecond maximum bandwidth, in bytes per second
+ */
+ void setWriteBandwidthLimit(long bytesPerSecond);
+
+ /**
+ * Returns the current bandwidth limit for writing, in bytes per second.
+ */
+ long getWriteBandwidthLimit();
+
+ /**
+ * The minimum amount of time that will elapse between the time the proxy begins receiving a response from the server and the time the
+ * proxy begins sending the response to the client.
+ *
+ * @param latency minimum latency, or 0 for no minimum
+ * @param timeUnit TimeUnit for the latency
+ */
+ void setLatency(long latency, TimeUnit timeUnit);
+
+ /**
+ * Maximum amount of time to wait to establish a connection to a remote server. If the connection has not been established within the
+ * specified time, the proxy will respond with an HTTP 502 Bad Gateway. The default value is 60 seconds.
+ *
+ * @param connectionTimeout maximum time to wait to establish a connection to a server, or 0 to wait indefinitely
+ * @param timeUnit TimeUnit for the connectionTimeout
+ */
+ void setConnectTimeout(int connectionTimeout, TimeUnit timeUnit);
+
+ /**
+ * Maximum amount of time to allow a connection to remain idle. A connection becomes idle when it has not received data from a server
+ * within the the specified timeout. If the proxy has not yet begun to forward the response to the client, the proxy
+ * will respond with an HTTP 504 Gateway Timeout. If the proxy has already started forwarding the response to the client, the
+ * connection to the client may be closed abruptly. The default value is 60 seconds.
+ *
+ * @param idleConnectionTimeout maximum time to allow a connection to remain idle, or 0 to wait indefinitely.
+ * @param timeUnit TimeUnit for the idleConnectionTimeout
+ */
+ void setIdleConnectionTimeout(int idleConnectionTimeout, TimeUnit timeUnit);
+
+ /**
+ * Maximum amount of time to wait for an HTTP response from the remote server after the request has been sent in its entirety. The HTTP
+ * request must complete within the specified time. If the proxy has not yet begun to forward the response to the client, the proxy
+ * will respond with an HTTP 504 Gateway Timeout. If the proxy has already started forwarding the response to the client, the
+ * connection to the client may be closed abruptly. The default value is 0 (wait indefinitely).
+ *
+ * @param requestTimeout maximum time to wait for an HTTP response, or 0 to wait indefinitely
+ * @param timeUnit TimeUnit for the requestTimeout
+ */
+ void setRequestTimeout(int requestTimeout, TimeUnit timeUnit);
+
+ /**
+ * Enables automatic authorization for the specified domain and auth type. Every request sent to the specified domain will contain the
+ * specified authorization information.
+ *
+ * @param domain domain automatically send authorization information to
+ * @param username authorization username
+ * @param password authorization password
+ * @param authType authorization type
+ */
+ void autoAuthorization(String domain, String username, String password, AuthType authType);
+
+ /**
+ * Stops automatic authorization for the specified domain.
+ *
+ * @param domain domain to stop automatically sending authorization information to
+ */
+ void stopAutoAuthorization(String domain);
+
+ /**
+ * Enables chained proxy authorization using the Proxy-Authorization header described in RFC 7235, section 4.4 (https://tools.ietf.org/html/rfc7235#section-4.4).
+ * Currently, only {@link AuthType#BASIC} authentication is supported.
+ *
+ * @param username the username to use to authenticate with the chained proxy
+ * @param password the password to use to authenticate with the chained proxy
+ * @param authType the auth type to use (currently, must be BASIC)
+ */
+ void chainedProxyAuthorization(String username, String password, AuthType authType);
+
+ /**
+ * Adds a rewrite rule for the specified URL-matching regular expression. If there are any existing rewrite rules, the new rewrite
+ * rule will be applied last, after all other rewrite rules are applied. The specified urlPattern will be replaced with the specified
+ * replacement expression. The urlPattern is treated as a Java regular expression and must be properly escaped (see {@link java.util.regex.Pattern}).
+ * The replacementExpression may consist of capture groups specified in the urlPattern, denoted
+ * by a $ (see {@link java.util.regex.Matcher#appendReplacement(StringBuffer, String)}.
+ *
+ * For HTTP requests (not HTTPS), if the hostname and/or port is changed as a result of a rewrite rule, the Host header of the request will be modified
+ * to reflect the updated hostname/port. For HTTPS requests, the host and port cannot be changed by rewrite rules
+ * (use {@link #getHostNameResolver()} and {@link AdvancedHostResolver#remapHost(String, String)} to direct HTTPS requests
+ * to a different host).
+ *
+ * Note: The rewriting applies to the entire URL, including scheme (http:// or https://), hostname/address, port, and query string. Note that this means
+ * a urlPattern of {@code "http://www\.website\.com/page"} will NOT match {@code http://www.website.com:80/page}.
+ *
+ * For example, the following rewrite rule:
+ *
+ *
+ *
+ * will match an HTTP request (but not HTTPS!) to www.yahoo.com or www.bing.com with exactly 1 query parameter,
+ * and replace it with a call to www.google.com with an 'originalDomain' query parameter, as well as the original query parameter.
+ *
+ * When applied to the URL:
+ *
+ *
+ * @param urlPattern URL-matching regular expression
+ * @param replacementExpression an expression, which may optionally contain capture groups, which will replace any URL which matches urlPattern
+ */
+ void rewriteUrl(String urlPattern, String replacementExpression);
+
+ /**
+ * Replaces existing rewrite rules with the specified patterns and replacement expressions. The rules will be applied in the order
+ * specified by the Map's iterator.
+ *
+ * See {@link #rewriteUrl(String, String)} for details on the format of the rewrite rules.
+ *
+ * @param rewriteRules {@code Map}
+ */
+ void rewriteUrls(Map rewriteRules);
+
+ /**
+ * Returns all rewrite rules currently in effect. Iterating over the returned Map is guaranteed to return rewrite rules
+ * in the order in which the rules are actually applied.
+ *
+ * @return {@code Map}
+ */
+ Map getRewriteRules();
+
+ /**
+ * Removes an existing rewrite rule whose urlPattern matches the specified pattern.
+ *
+ * @param urlPattern rewrite rule pattern to remove
+ */
+ void removeRewriteRule(String urlPattern);
+
+ /**
+ * Clears all existing rewrite rules.
+ */
+ void clearRewriteRules();
+
+ /**
+ * Adds a URL-matching regular expression to the blacklist. Requests that match a blacklisted URL will return the specified HTTP
+ * statusCode for all HTTP methods. If there are existing patterns on the blacklist, the urlPattern will be evaluated last,
+ * after the URL is checked against all other blacklist entries.
+ *
+ * The urlPattern matches the full URL of the request, including scheme, host, and port, path, and query parameters
+ * for both HTTP and HTTPS requests. For example, to blacklist both HTTP and HTTPS requests to www.google.com,
+ * use a urlPattern of "https?://www\\.google\\.com/.*".
+ *
+ * @param urlPattern URL-matching regular expression to blacklist
+ * @param statusCode HTTP status code to return
+ */
+ void blacklistRequests(String urlPattern, int statusCode);
+
+ /**
+ * Adds a URL-matching regular expression to the blacklist. Requests that match a blacklisted URL will return the specified HTTP
+ * statusCode only when the request's HTTP method (GET, POST, PUT, etc.) matches the specified httpMethodPattern regular expression.
+ * If there are existing patterns on the blacklist, the urlPattern will be evaluated last, after the URL is checked against all
+ * other blacklist entries.
+ *
+ * See {@link #blacklistRequests(String, int)} for details on the URL the urlPattern will match.
+ *
+ * @param urlPattern URL-matching regular expression to blacklist
+ * @param statusCode HTTP status code to return
+ * @param httpMethodPattern regular expression matching a request's HTTP method
+ */
+ void blacklistRequests(String urlPattern, int statusCode, String httpMethodPattern);
+
+ /**
+ * Replaces any existing blacklist with the specified blacklist. URLs will be evaluated against the blacklist in the order
+ * specified by the Collection's iterator.
+ *
+ * @param blacklist new blacklist entries
+ */
+ void setBlacklist(Collection blacklist);
+
+ /**
+ * Returns all blacklist entries currently in effect. Iterating over the returned Collection is guaranteed to return
+ * blacklist entries in the order in which URLs are actually evaluated against the blacklist.
+ *
+ * @return blacklist entries, or an empty collection if none exist
+ */
+ Collection getBlacklist();
+
+ /**
+ * Clears any existing blacklist.
+ */
+ void clearBlacklist();
+
+ /**
+ * Whitelists URLs matching the specified regular expression patterns. Replaces any existing whitelist.
+ * The urlPattern matches the full URL of the request, including scheme, host, and port, path, and query parameters
+ * for both HTTP and HTTPS requests. For example, to whitelist both HTTP and HTTPS requests to www.google.com, use a urlPattern
+ * of "https?://www\\.google\\.com/.*".
+ *
+ * Note: All HTTP CONNECT requests are automatically whitelisted and cannot be short-circuited using the
+ * whitelist response code.
+ *
+ * @param urlPatterns URL-matching regular expressions to whitelist; null or an empty collection will enable an empty whitelist
+ * @param statusCode HTTP status code to return to clients when a URL matches a pattern
+ */
+ void whitelistRequests(Collection urlPatterns, int statusCode);
+
+ /**
+ * Adds a URL-matching regular expression to an existing whitelist.
+ *
+ * @param urlPattern URL-matching regular expressions to whitelist
+ * @throws java.lang.IllegalStateException if the whitelist is not enabled
+ */
+ void addWhitelistPattern(String urlPattern);
+
+ /**
+ * Enables the whitelist, but with no matching URLs. All requests will generated the specified HTTP statusCode.
+ *
+ * @param statusCode HTTP status code to return to clients on all requests
+ */
+ void enableEmptyWhitelist(int statusCode);
+
+ /**
+ * Clears any existing whitelist and disables whitelisting.
+ */
+ void disableWhitelist();
+
+ /**
+ * Returns the URL-matching regular expressions currently in effect. If the whitelist is disabled, this method always returns an empty collection.
+ * If the whitelist is enabled but empty, this method return an empty collection.
+ *
+ * @return whitelist currently in effect, or an empty collection if the whitelist is disabled or empty
+ */
+ Collection getWhitelistUrls();
+
+ /**
+ * Returns the status code returned for all URLs that do not match the whitelist. If the whitelist is not currently enabled, returns -1.
+ *
+ * @return HTTP status code returned for non-whitelisted URLs, or -1 if the whitelist is disabled.
+ */
+ int getWhitelistStatusCode();
+
+ /**
+ * Returns true if the whitelist is enabled, otherwise false.
+ */
+ boolean isWhitelistEnabled();
+
+ /**
+ * Adds the specified HTTP headers to every request. Replaces any existing additional headers with the specified headers.
+ *
+ * @param headers {@code Map} to append to every request.
+ */
+ void addHeaders(Map headers);
+
+ /**
+ * Adds a new HTTP header to every request. If the header already exists on the request, it will be replaced with the specified header.
+ *
+ * @param name name of the header to add
+ * @param value new header's value
+ */
+ void addHeader(String name, String value);
+
+ /**
+ * Removes a header previously added with {@link #addHeader(String name, String value)}.
+ *
+ * @param name previously-added header's name
+ */
+ void removeHeader(String name);
+
+ /**
+ * Removes all headers previously added with {@link #addHeader(String name, String value)}.
+ */
+ void removeAllHeaders();
+
+ /**
+ * Returns all headers previously added with {@link #addHeader(String name, String value)}.
+ *
+ * @return {@code Map}
+ */
+ Map getAllHeaders();
+
+ /**
+ * Sets the resolver that will be used to look up host names. To chain multiple resolvers, wrap a list
+ * of resolvers in a {@link net.lightbody.bmp.proxy.dns.ChainedHostResolver}.
+ *
+ * @param resolver host name resolver
+ */
+ void setHostNameResolver(AdvancedHostResolver resolver);
+
+ /**
+ * Returns the current host name resolver.
+ *
+ * @return current host name resolver
+ */
+ AdvancedHostResolver getHostNameResolver();
+
+ /**
+ * Waits for existing network traffic to stop, and for the specified quietPeriod to elapse. Returns true if there is no network traffic
+ * for the quiet period within the specified timeout, otherwise returns false.
+ *
+ * @param quietPeriod amount of time after which network traffic will be considered "stopped"
+ * @param timeout maximum amount of time to wait for network traffic to stop
+ * @param timeUnit TimeUnit for the quietPeriod and timeout
+ * @return true if network traffic is stopped, otherwise false
+ */
+ boolean waitForQuiescence(long quietPeriod, long timeout, TimeUnit timeUnit);
+
+ /**
+ * Instructs this proxy to route traffic through an upstream proxy.
+ *
+ * Note: A chained proxy must be set before the proxy is started, though it can be changed after the proxy is started.
+ *
+ * @param chainedProxyAddress address of the upstream proxy
+ */
+ void setChainedProxy(InetSocketAddress chainedProxyAddress);
+
+ /**
+ * Returns the address and port of the upstream proxy.
+ *
+ * @return address and port of the upstream proxy, or null of there is none.
+ */
+ InetSocketAddress getChainedProxy();
+
+ /**
+ * Adds a new filter factory (request/response interceptor) to the beginning of the HttpFilters chain.
+ *
+ * Usage note: The actual filter (interceptor) instance is created on every request by implementing the
+ * {@link HttpFiltersSource#filterRequest(io.netty.handler.codec.http.HttpRequest, io.netty.channel.ChannelHandlerContext)} method and returning an
+ * {@link org.littleshoot.proxy.HttpFilters} instance (typically, a subclass of {@link org.littleshoot.proxy.HttpFiltersAdapter}).
+ * To disable or bypass a filter on a per-request basis, the filterRequest() method may return null.
+ *
+ * @param filterFactory factory to generate HttpFilters
+ */
+ void addFirstHttpFilterFactory(HttpFiltersSource filterFactory);
+
+ /**
+ * Adds a new filter factory (request/response interceptor) to the end of the HttpFilters chain.
+ *
+ * Usage note: The actual filter (interceptor) instance is created on every request by implementing the
+ * {@link HttpFiltersSource#filterRequest(io.netty.handler.codec.http.HttpRequest, io.netty.channel.ChannelHandlerContext)} method and returning an
+ * {@link org.littleshoot.proxy.HttpFilters} instance (typically, a subclass of {@link org.littleshoot.proxy.HttpFiltersAdapter}).
+ * To disable or bypass a filter on a per-request basis, the filterRequest() method may return null.
+ *
+ * @param filterFactory factory to generate HttpFilters
+ */
+ void addLastHttpFilterFactory(HttpFiltersSource filterFactory);
+
+ /**
+ * Adds a new ResponseFilter that can be used to examine and manipulate the response before sending it to the client.
+ *
+ * @param filter filter instance
+ */
+ void addResponseFilter(ResponseFilter filter);
+
+ /**
+ * Adds a new RequestFilter that can be used to examine and manipulate the request before sending it to the server.
+ *
+ * @param filter filter instance
+ */
+ void addRequestFilter(RequestFilter filter);
+
+ /**
+ * Completely disables MITM for this proxy server. The proxy will no longer intercept HTTPS requests, but they will
+ * still be pass-through proxied. This option must be set before the proxy is started; otherwise an IllegalStateException will be thrown.
+ *
+ * @param mitmDisabled when true, MITM capture will be disabled
+ * @throws java.lang.IllegalStateException if the proxy is already started
+ */
+ void setMitmDisabled(boolean mitmDisabled);
+
+ /**
+ * Sets the MITM manager, which is responsible for generating forged SSL certificates to present to clients. By default,
+ * BrowserMob Proxy uses the ca-certificate-rsa.cer root certificate for impersonation. See the documentation at
+ * {@link net.lightbody.bmp.mitm.manager.ImpersonatingMitmManager} and {@link net.lightbody.bmp.mitm.manager.ImpersonatingMitmManager.Builder}
+ * for details on customizing the root and server certificate generation.
+ *
+ * @param mitmManager MITM manager to use
+ */
+ void setMitmManager(MitmManager mitmManager);
+
+ /**
+ * Disables verification of all upstream servers' SSL certificates. All upstream servers will be trusted, even if they
+ * do not present valid certificates signed by certification authorities in the JDK's trust store. This option
+ * exposes the proxy to MITM attacks and should only be used when testing in trusted environments.
+ *
+ * @param trustAllServers when true, disables upstream server certificate verification
+ */
+ void setTrustAllServers(boolean trustAllServers);
+
+ /**
+ * Sets the {@link TrustSource} that contains trusted root certificate authorities that will be used to validate
+ * upstream servers' certificates. When null, disables certificate validation (see warning at {@link #setTrustAllServers(boolean)}).
+ *
+ * @param trustSource TrustSource containing root CAs, or null to disable upstream server validation
+ */
+ void setTrustSource(TrustSource trustSource);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java
new file mode 100644
index 000000000..d4ddc5543
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java
@@ -0,0 +1,1175 @@
+package net.lightbody.bmp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.MapMaker;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import net.lightbody.bmp.client.ClientUtil;
+import net.lightbody.bmp.core.har.Har;
+import net.lightbody.bmp.core.har.HarLog;
+import net.lightbody.bmp.core.har.HarNameVersion;
+import net.lightbody.bmp.core.har.HarPage;
+import net.lightbody.bmp.filters.AddHeadersFilter;
+import net.lightbody.bmp.filters.AutoBasicAuthFilter;
+import net.lightbody.bmp.filters.BlacklistFilter;
+import net.lightbody.bmp.filters.BrowserMobHttpFilterChain;
+import net.lightbody.bmp.filters.HarCaptureFilter;
+import net.lightbody.bmp.filters.HttpConnectHarCaptureFilter;
+import net.lightbody.bmp.filters.HttpsHostCaptureFilter;
+import net.lightbody.bmp.filters.HttpsOriginalHostCaptureFilter;
+import net.lightbody.bmp.filters.LatencyFilter;
+import net.lightbody.bmp.filters.RegisterRequestFilter;
+import net.lightbody.bmp.filters.RequestFilter;
+import net.lightbody.bmp.filters.RequestFilterAdapter;
+import net.lightbody.bmp.filters.ResolvedHostnameCacheFilter;
+import net.lightbody.bmp.filters.ResponseFilter;
+import net.lightbody.bmp.filters.ResponseFilterAdapter;
+import net.lightbody.bmp.filters.RewriteUrlFilter;
+import net.lightbody.bmp.filters.UnregisterRequestFilter;
+import net.lightbody.bmp.filters.WhitelistFilter;
+import net.lightbody.bmp.mitm.KeyStoreFileCertificateSource;
+import net.lightbody.bmp.mitm.TrustSource;
+import net.lightbody.bmp.mitm.keys.ECKeyGenerator;
+import net.lightbody.bmp.mitm.keys.RSAKeyGenerator;
+import net.lightbody.bmp.mitm.manager.ImpersonatingMitmManager;
+import net.lightbody.bmp.proxy.ActivityMonitor;
+import net.lightbody.bmp.proxy.BlacklistEntry;
+import net.lightbody.bmp.proxy.CaptureType;
+import net.lightbody.bmp.proxy.RewriteRule;
+import net.lightbody.bmp.proxy.Whitelist;
+import net.lightbody.bmp.proxy.auth.AuthType;
+import net.lightbody.bmp.proxy.dns.AdvancedHostResolver;
+import net.lightbody.bmp.proxy.dns.DelegatingHostResolver;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import net.lightbody.bmp.util.BrowserMobProxyUtil;
+import org.littleshoot.proxy.ChainedProxy;
+import org.littleshoot.proxy.ChainedProxyAdapter;
+import org.littleshoot.proxy.ChainedProxyManager;
+import org.littleshoot.proxy.HttpFilters;
+import org.littleshoot.proxy.HttpFiltersSource;
+import org.littleshoot.proxy.HttpFiltersSourceAdapter;
+import org.littleshoot.proxy.HttpProxyServer;
+import org.littleshoot.proxy.HttpProxyServerBootstrap;
+import org.littleshoot.proxy.MitmManager;
+import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
+import org.littleshoot.proxy.impl.ProxyUtils;
+import org.littleshoot.proxy.impl.ThreadPoolConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
+/**
+ * A LittleProxy-based implementation of {@link net.lightbody.bmp.BrowserMobProxy}.
+ */
+public class BrowserMobProxyServer implements BrowserMobProxy {
+ private static final Logger log = LoggerFactory.getLogger(BrowserMobProxyServer.class);
+
+ private static final HarNameVersion HAR_CREATOR_VERSION = new HarNameVersion("BrowserMob Proxy", BrowserMobProxyUtil.getVersionString());
+
+ /* Default MITM resources */
+ private static final String RSA_KEYSTORE_RESOURCE = "/sslSupport/ca-keystore-rsa.p12";
+ private static final String EC_KEYSTORE_RESOURCE = "/sslSupport/ca-keystore-ec.p12";
+ private static final String KEYSTORE_TYPE = "PKCS12";
+ private static final String KEYSTORE_PRIVATE_KEY_ALIAS = "key";
+ private static final String KEYSTORE_PASSWORD = "password";
+
+ /**
+ * The default pseudonym to use when adding the Via header to proxied requests.
+ */
+ public static final String VIA_HEADER_ALIAS = "browsermobproxy";
+
+ /**
+ * True only after the proxy has been successfully started.
+ */
+ private final AtomicBoolean started = new AtomicBoolean(false);
+
+ /**
+ * True only after the proxy has been successfully started, then successfully stopped or aborted.
+ */
+ private final AtomicBoolean stopped = new AtomicBoolean(false);
+
+ /**
+ * Tracks the current page count, for use when auto-generating HAR page names.
+ */
+ private final AtomicInteger harPageCount = new AtomicInteger(0);
+
+ /**
+ * When true, MITM will be disabled. The proxy will no longer intercept HTTPS requests, but they will still be proxied.
+ */
+ private volatile boolean mitmDisabled = false;
+
+ /**
+ * The MITM manager that will be used for HTTPS requests.
+ */
+ private volatile MitmManager mitmManager;
+
+ /**
+ * The list of filterFactories that will generate the filters that implement browsermob-proxy behavior.
+ */
+ private final List filterFactories = new CopyOnWriteArrayList<>();
+
+ /**
+ * List of rejected URL patterns
+ */
+ private volatile Collection blacklistEntries = new CopyOnWriteArrayList<>();
+
+ /**
+ * List of URLs to rewrite
+ */
+ private volatile CopyOnWriteArrayList rewriteRules = new CopyOnWriteArrayList<>();
+
+ /**
+ * The LittleProxy instance that performs all proxy operations.
+ */
+ private volatile HttpProxyServer proxyServer;
+
+ /**
+ * No capture types are enabled by default.
+ */
+ private volatile EnumSet harCaptureTypes = EnumSet.noneOf(CaptureType.class);
+
+ /**
+ * The current HAR being captured.
+ */
+ private volatile Har har;
+ /**
+ * The current HarPage to which new requests will be associated.
+ */
+ private volatile HarPage currentHarPage;
+ /**
+ * Maximum bandwidth to consume when reading responses from servers.
+ */
+ private volatile long readBandwidthLimitBps;
+ /**
+ * Maximum bandwidth to consume when writing requests to servers.
+ */
+ private volatile long writeBandwidthLimitBps;
+ /**
+ * List of accepted URL patterns. Unlisted URL patterns will be rejected with the response code contained in the Whitelist.
+ */
+ private final AtomicReference whitelist = new AtomicReference<>(Whitelist.WHITELIST_DISABLED);
+
+ /**
+ * Additional headers that will be sent with every request. The map is declared as a ConcurrentMap to indicate that writes may be performed
+ * by other threads concurrently (e.g. due to an incoming REST call), but the concurrencyLevel is set to 1 because modifications to the
+ * additionalHeaders are rare, and in most cases happen only once, at start-up.
+ */
+ private volatile ConcurrentMap additionalHeaders = new MapMaker().concurrencyLevel(1).makeMap();
+
+ /**
+ * The amount of time to wait while connecting to a server.
+ */
+ private volatile int connectTimeoutMs;
+
+ /**
+ * The amount of time a connection to a server can remain idle while receiving data from the server.
+ */
+ private volatile int idleConnectionTimeoutSec;
+
+ /**
+ * The amount of time to wait before forwarding the response to the client.
+ */
+ private volatile int latencyMs;
+
+ /**
+ * Set to true once the HAR capture filter has been added to the filter chain.
+ */
+ private final AtomicBoolean harCaptureFilterEnabled = new AtomicBoolean(false);
+
+ /**
+ * Set to true when LittleProxy has been bootstrapped with the default chained proxy. This allows modifying the chained proxy
+ * after the proxy has been started.
+ */
+ private final AtomicBoolean bootstrappedWithDefaultChainedProxy = new AtomicBoolean(false);
+
+ /**
+ * The address of an upstream chained proxy to route traffic through.
+ */
+ private volatile InetSocketAddress upstreamProxyAddress;
+
+ /**
+ * The chained proxy manager that manages upstream proxies.
+ */
+ private volatile ChainedProxyManager chainedProxyManager;
+
+ /**
+ * The address of the network interface from which the proxy will initiate connections.
+ */
+ private volatile InetAddress serverBindAddress;
+
+ /**
+ * The TrustSource that will be used to validate servers' certificates. If null, will not validate server certificates.
+ */
+ private volatile TrustSource trustSource = TrustSource.defaultTrustSource();
+
+ /**
+ * When true, use Elliptic Curve keys and certificates when impersonating upstream servers.
+ */
+ private volatile boolean useEcc = false;
+
+ /**
+ * Resolver to use when resolving hostnames to IP addresses. This is a bridge between {@link org.littleshoot.proxy.HostResolver} and
+ * {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver}. It allows the resolvers to be changed on-the-fly without re-bootstrapping the
+ * littleproxy server. The default resolver (native JDK resolver) can be changed using {@link #setHostNameResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)} and
+ * supplying one of the pre-defined resolvers in {@link ClientUtil}, such as {@link ClientUtil#createDnsJavaWithNativeFallbackResolver()}
+ * or {@link ClientUtil#createDnsJavaResolver()}. You can also build your own resolver, or use {@link net.lightbody.bmp.proxy.dns.ChainedHostResolver}
+ * to chain together multiple DNS resolvers.
+ */
+ private final DelegatingHostResolver delegatingResolver = new DelegatingHostResolver(ClientUtil.createNativeCacheManipulatingResolver());
+
+ private final ActivityMonitor activityMonitor = new ActivityMonitor();
+
+ /**
+ * The acceptor and worker thread configuration for the Netty thread pools.
+ */
+ private volatile ThreadPoolConfiguration threadPoolConfiguration;
+
+ /**
+ * A mapping of hostnames to base64-encoded Basic auth credentials that will be added to the Authorization header for
+ * matching requests.
+ */
+ private final ConcurrentMap basicAuthCredentials = new MapMaker()
+ .concurrencyLevel(1)
+ .makeMap();
+
+ /**
+ * Base64-encoded credentials to use to authenticate with the upstream proxy.
+ */
+ private volatile String chainedProxyCredentials;
+
+ public BrowserMobProxyServer() {
+ }
+
+ @Override
+ public void start(int port, InetAddress clientBindAddress, InetAddress serverBindAddress) {
+ boolean notStarted = started.compareAndSet(false, true);
+ if (!notStarted) {
+ throw new IllegalStateException("Proxy server is already started. Not restarting.");
+ }
+
+ InetSocketAddress clientBindSocket;
+ if (clientBindAddress == null) {
+ // if no client bind address was specified, bind to the wildcard address
+ clientBindSocket = new InetSocketAddress(port);
+ } else {
+ clientBindSocket = new InetSocketAddress(clientBindAddress, port);
+ }
+
+ this.serverBindAddress = serverBindAddress;
+
+ // initialize all the default BrowserMob filter factories that provide core BMP functionality
+ addBrowserMobFilters();
+
+ HttpProxyServerBootstrap bootstrap = DefaultHttpProxyServer.bootstrap()
+ .withFiltersSource(new HttpFiltersSource() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext channelHandlerContext) {
+ return new BrowserMobHttpFilterChain(BrowserMobProxyServer.this, originalRequest, channelHandlerContext);
+ }
+
+ @Override
+ public int getMaximumRequestBufferSizeInBytes() {
+ return getMaximumRequestBufferSize();
+ }
+
+ @Override
+ public int getMaximumResponseBufferSizeInBytes() {
+ return getMaximumResponseBufferSize();
+ }
+ })
+ .withServerResolver(delegatingResolver)
+ .withAddress(clientBindSocket)
+ .withConnectTimeout(connectTimeoutMs)
+ .withIdleConnectionTimeout(idleConnectionTimeoutSec)
+ .withProxyAlias(VIA_HEADER_ALIAS);
+
+ if (serverBindAddress != null) {
+ bootstrap.withNetworkInterface(new InetSocketAddress(serverBindAddress, 0));
+ }
+
+
+ if (!mitmDisabled) {
+ if (mitmManager == null) {
+ mitmManager = ImpersonatingMitmManager.builder()
+ .rootCertificateSource(new KeyStoreFileCertificateSource(
+ KEYSTORE_TYPE,
+ useEcc ? EC_KEYSTORE_RESOURCE : RSA_KEYSTORE_RESOURCE,
+ KEYSTORE_PRIVATE_KEY_ALIAS,
+ KEYSTORE_PASSWORD))
+ .serverKeyGenerator(useEcc ? new ECKeyGenerator() : new RSAKeyGenerator())
+ .trustSource(trustSource)
+ .build();
+ }
+
+ bootstrap.withManInTheMiddle(mitmManager);
+ }
+
+ if (readBandwidthLimitBps > 0 || writeBandwidthLimitBps > 0) {
+ bootstrap.withThrottling(readBandwidthLimitBps, writeBandwidthLimitBps);
+ }
+
+ if (chainedProxyManager != null) {
+ bootstrap.withChainProxyManager(chainedProxyManager);
+ } else if (upstreamProxyAddress != null) {
+ // indicate that the proxy was bootstrapped with the default chained proxy manager, which allows changing the
+ // chained proxy after the proxy is started.
+ bootstrappedWithDefaultChainedProxy.set(true);
+
+ bootstrap.withChainProxyManager(new ChainedProxyManager() {
+ @Override
+ public void lookupChainedProxies(HttpRequest httpRequest, Queue chainedProxies) {
+ final InetSocketAddress upstreamProxy = upstreamProxyAddress;
+ if (upstreamProxy != null) {
+ chainedProxies.add(new ChainedProxyAdapter() {
+ @Override
+ public InetSocketAddress getChainedProxyAddress() {
+ return upstreamProxy;
+ }
+
+ @Override
+ public void filterRequest(HttpObject httpObject) {
+ String chainedProxyAuth = chainedProxyCredentials;
+ if (chainedProxyAuth != null) {
+ if (httpObject instanceof HttpRequest) {
+ HttpHeaders.addHeader((HttpRequest)httpObject, HttpHeaders.Names.PROXY_AUTHORIZATION, "Basic " + chainedProxyAuth);
+ }
+ }
+ }
+ });
+ }
+ }
+ });
+ }
+
+ if (threadPoolConfiguration != null) {
+ bootstrap.withThreadPoolConfiguration(threadPoolConfiguration);
+ }
+
+ proxyServer = bootstrap.start();
+ }
+
+ @Override
+ public boolean isStarted() {
+ return started.get();
+ }
+
+ @Override
+ public void start(int port) {
+ this.start(port, null, null);
+ }
+
+ @Override
+ public void start(int port, InetAddress bindAddress) {
+ this.start(port, bindAddress, null);
+
+ }
+
+ @Override
+ public void start() {
+ this.start(0);
+ }
+
+ @Override
+ public void stop() {
+ stop(true);
+ }
+
+ @Override
+ public void abort() {
+ stop(false);
+ }
+
+ protected void stop(boolean graceful) {
+ if (isStarted()) {
+ if (stopped.compareAndSet(false, true)) {
+ if (proxyServer != null) {
+ if (graceful) {
+ proxyServer.stop();
+ } else {
+ proxyServer.abort();
+ }
+ } else {
+ log.warn("Attempted to stop proxy server, but proxy was never successfully started.");
+ }
+ } else {
+ throw new IllegalStateException("Proxy server is already stopped. Cannot re-stop.");
+ }
+ } else {
+ throw new IllegalStateException("Proxy server has not been started");
+ }
+ }
+
+ @Override
+ public InetAddress getClientBindAddress() {
+ if (started.get()) {
+ return proxyServer.getListenAddress().getAddress();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public int getPort() {
+ if (started.get()) {
+ return proxyServer.getListenAddress().getPort();
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public InetAddress getServerBindAddress() {
+ return serverBindAddress;
+ }
+
+ @Override
+ public Har getHar() {
+ return har;
+ }
+
+ @Override
+ public Har newHar() {
+ return newHar(null);
+ }
+
+ @Override
+ public Har newHar(String initialPageRef) {
+ return newHar(initialPageRef, null);
+ }
+
+ @Override
+ public Har newHar(String initialPageRef, String initialPageTitle) {
+ Har oldHar = getHar();
+
+ addHarCaptureFilter();
+
+ harPageCount.set(0);
+
+ this.har = new Har(new HarLog(HAR_CREATOR_VERSION));
+
+ newPage(initialPageRef, initialPageTitle);
+
+ return oldHar;
+ }
+
+ @Override
+ public void setHarCaptureTypes(Set harCaptureSettings) {
+ if (harCaptureSettings == null || harCaptureSettings.isEmpty()) {
+ harCaptureTypes = EnumSet.noneOf(CaptureType.class);
+ } else {
+ harCaptureTypes = EnumSet.copyOf(harCaptureSettings);
+ }
+ }
+
+ @Override
+ public void setHarCaptureTypes(CaptureType... captureTypes) {
+ if (captureTypes == null) {
+ setHarCaptureTypes(EnumSet.noneOf(CaptureType.class));
+ } else {
+ setHarCaptureTypes(EnumSet.copyOf(Arrays.asList(captureTypes)));
+ }
+ }
+
+ @Override
+ public EnumSet getHarCaptureTypes() {
+ return EnumSet.copyOf(harCaptureTypes);
+ }
+
+ @Override
+ public void enableHarCaptureTypes(Set captureTypes) {
+ harCaptureTypes.addAll(captureTypes);
+ }
+
+ @Override
+ public void enableHarCaptureTypes(CaptureType... captureTypes) {
+ if (captureTypes == null) {
+ enableHarCaptureTypes(EnumSet.noneOf(CaptureType.class));
+ } else {
+ enableHarCaptureTypes(EnumSet.copyOf(Arrays.asList(captureTypes)));
+ }
+ }
+
+ @Override
+ public void disableHarCaptureTypes(Set captureTypes) {
+ harCaptureTypes.removeAll(captureTypes);
+
+ }
+
+ @Override
+ public void disableHarCaptureTypes(CaptureType... captureTypes) {
+ if (captureTypes == null) {
+ disableHarCaptureTypes(EnumSet.noneOf(CaptureType.class));
+ } else {
+ disableHarCaptureTypes(EnumSet.copyOf(Arrays.asList(captureTypes)));
+ }
+ }
+
+ @Override
+ public Har newPage() {
+ return newPage(null);
+ }
+
+ @Override
+ public Har newPage(String pageRef) {
+ return newPage(pageRef, null);
+ }
+
+ @Override
+ public Har newPage(String pageRef, String pageTitle) {
+ if (har == null) {
+ throw new IllegalStateException("No HAR exists for this proxy. Use newHar() to create a new HAR before calling newPage().");
+ }
+
+ Har endOfPageHar = null;
+
+ if (currentHarPage != null) {
+ String currentPageRef = currentHarPage.getId();
+
+ // end the previous page, so that page-wide timings are populated
+ endPage();
+
+ // the interface requires newPage() to return the Har as it was immediately after the previous page was ended.
+ endOfPageHar = BrowserMobProxyUtil.copyHarThroughPageRef(har, currentPageRef);
+ }
+
+ if (pageRef == null) {
+ pageRef = "Page " + harPageCount.getAndIncrement();
+ }
+
+ if (pageTitle == null) {
+ pageTitle = pageRef;
+ }
+
+ HarPage newPage = new HarPage(pageRef, pageTitle);
+ har.getLog().addPage(newPage);
+
+ currentHarPage = newPage;
+
+ return endOfPageHar;
+ }
+
+ @Override
+ public Har endHar() {
+ Har oldHar = getHar();
+
+ // end the page and populate timings
+ endPage();
+
+ this.har = null;
+
+ return oldHar;
+ }
+
+ @Override
+ public void setReadBandwidthLimit(long bytesPerSecond) {
+ this.readBandwidthLimitBps = bytesPerSecond;
+
+ if (isStarted()) {
+ proxyServer.setThrottle(this.readBandwidthLimitBps, this.writeBandwidthLimitBps);
+ }
+ }
+
+ @Override
+ public long getReadBandwidthLimit() {
+ return readBandwidthLimitBps;
+ }
+
+ @Override
+ public void setWriteBandwidthLimit(long bytesPerSecond) {
+ this.writeBandwidthLimitBps = bytesPerSecond;
+
+ if (isStarted()) {
+ proxyServer.setThrottle(this.readBandwidthLimitBps, this.writeBandwidthLimitBps);
+ }
+ }
+
+ @Override
+ public long getWriteBandwidthLimit() {
+ return writeBandwidthLimitBps;
+ }
+
+ public void endPage() {
+ if (har == null) {
+ throw new IllegalStateException("No HAR exists for this proxy. Use newHar() to create a new HAR.");
+ }
+
+ HarPage previousPage = this.currentHarPage;
+ this.currentHarPage = null;
+
+ if (previousPage == null) {
+ return;
+ }
+
+ previousPage.getPageTimings().setOnLoad(new Date().getTime() - previousPage.getStartedDateTime().getTime());
+ }
+
+ @Override
+ public void addHeaders(Map headers) {
+ ConcurrentMap newHeaders = new MapMaker().concurrencyLevel(1).makeMap();
+ newHeaders.putAll(headers);
+
+ this.additionalHeaders = newHeaders;
+ }
+
+ @Override
+ public void setLatency(long latency, TimeUnit timeUnit) {
+ this.latencyMs = (int) TimeUnit.MILLISECONDS.convert(latency, timeUnit);
+ }
+
+ @Override
+ public void autoAuthorization(String domain, String username, String password, AuthType authType) {
+ switch (authType) {
+ case BASIC:
+ // base64 encode the "username:password" string
+ String base64EncodedCredentials = BrowserMobHttpUtil.base64EncodeBasicCredentials(username, password);
+
+ basicAuthCredentials.put(domain, base64EncodedCredentials);
+ break;
+
+ default:
+ throw new UnsupportedOperationException("AuthType " + authType + " is not supported for HTTP Authorization");
+ }
+ }
+
+ @Override
+ public void stopAutoAuthorization(String domain) {
+ basicAuthCredentials.remove(domain);
+ }
+
+ @Override
+ public void chainedProxyAuthorization(String username, String password, AuthType authType) {
+ switch (authType) {
+ case BASIC:
+ chainedProxyCredentials = BrowserMobHttpUtil.base64EncodeBasicCredentials(username, password);
+ break;
+
+ default:
+ throw new UnsupportedOperationException("AuthType " + authType + " is not supported for Proxy Authorization");
+ }
+ }
+
+ @Override
+ public void setConnectTimeout(int connectTimeout, TimeUnit timeUnit) {
+ this.connectTimeoutMs = (int) TimeUnit.MILLISECONDS.convert(connectTimeout, timeUnit);
+
+ if (isStarted()) {
+ proxyServer.setConnectTimeout((int) TimeUnit.MILLISECONDS.convert(connectTimeout, timeUnit));
+ }
+ }
+
+ /**
+ * The LittleProxy implementation only allows idle connection timeouts to be specified in seconds. idleConnectionTimeouts greater than
+ * 0 but less than 1 second will be set to 1 second; otherwise, values will be truncated (i.e. 1500ms will become 1s).
+ */
+ @Override
+ public void setIdleConnectionTimeout(int idleConnectionTimeout, TimeUnit timeUnit) {
+ long timeout = TimeUnit.SECONDS.convert(idleConnectionTimeout, timeUnit);
+ if (timeout == 0 && idleConnectionTimeout > 0) {
+ this.idleConnectionTimeoutSec = 1;
+ } else {
+ this.idleConnectionTimeoutSec = (int) timeout;
+ }
+
+ if (isStarted()) {
+ proxyServer.setIdleConnectionTimeout(idleConnectionTimeoutSec);
+ }
+ }
+
+ @Override
+ public void setRequestTimeout(int requestTimeout, TimeUnit timeUnit) {
+ //TODO: implement Request Timeouts using LittleProxy. currently this only sets an idle connection timeout, if the idle connection
+ // timeout is higher than the specified requestTimeout.
+ if (idleConnectionTimeoutSec == 0 || idleConnectionTimeoutSec > TimeUnit.SECONDS.convert(requestTimeout, timeUnit)) {
+ setIdleConnectionTimeout(requestTimeout, timeUnit);
+ }
+ }
+
+ @Override
+ public void rewriteUrl(String pattern, String replace) {
+ rewriteRules.add(new RewriteRule(pattern, replace));
+ }
+
+ @Override
+ public void rewriteUrls(Map rewriteRules) {
+ List newRules = new ArrayList<>(rewriteRules.size());
+ for (Map.Entry rewriteRule : rewriteRules.entrySet()) {
+ RewriteRule newRule = new RewriteRule(rewriteRule.getKey(), rewriteRule.getValue());
+ newRules.add(newRule);
+ }
+
+ this.rewriteRules = new CopyOnWriteArrayList<>(newRules);
+ }
+
+ @Override
+ public void clearRewriteRules() {
+ rewriteRules.clear();
+ }
+
+ @Override
+ public void blacklistRequests(String pattern, int responseCode) {
+ blacklistEntries.add(new BlacklistEntry(pattern, responseCode));
+ }
+
+ @Override
+ public void blacklistRequests(String pattern, int responseCode, String method) {
+ blacklistEntries.add(new BlacklistEntry(pattern, responseCode, method));
+ }
+
+ @Override
+ public void setBlacklist(Collection blacklist) {
+ this.blacklistEntries = new CopyOnWriteArrayList<>(blacklist);
+ }
+
+ @Override
+ public Collection getBlacklist() {
+ return Collections.unmodifiableCollection(blacklistEntries);
+ }
+
+ @Override
+ public boolean isWhitelistEnabled() {
+ return whitelist.get().isEnabled();
+ }
+
+ @Override
+ public Collection getWhitelistUrls() {
+ ImmutableList.Builder builder = ImmutableList.builder();
+ for (Pattern pattern : whitelist.get().getPatterns()) {
+ builder.add(pattern.pattern());
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public int getWhitelistStatusCode() {
+ return whitelist.get().getStatusCode();
+ }
+
+ @Override
+ public void clearBlacklist() {
+ blacklistEntries.clear();
+ }
+
+ @Override
+ public void whitelistRequests(Collection urlPatterns, int statusCode) {
+ this.whitelist.set(new Whitelist(urlPatterns, statusCode));
+ }
+
+ @Override
+ public void addWhitelistPattern(String urlPattern) {
+ // to make sure this method is threadsafe, we need to guarantee that the "snapshot" of the whitelist taken at the beginning
+ // of the method has not been replaced by the time we have constructed a new whitelist at the end of the method
+ boolean whitelistUpdated = false;
+ while (!whitelistUpdated) {
+ Whitelist currentWhitelist = this.whitelist.get();
+ if (!currentWhitelist.isEnabled()) {
+ throw new IllegalStateException("Whitelist is disabled. Cannot add patterns to a disabled whitelist.");
+ }
+
+ // retrieve the response code and list of patterns from the current whitelist, the construct a new list of patterns that contains
+ // all of the old whitelist's patterns + this new pattern
+ int statusCode = currentWhitelist.getStatusCode();
+ List newPatterns = new ArrayList<>(currentWhitelist.getPatterns().size() + 1);
+ for (Pattern pattern : currentWhitelist.getPatterns()) {
+ newPatterns.add(pattern.pattern());
+ }
+ newPatterns.add(urlPattern);
+
+ // create a new (immutable) Whitelist object with the new pattern list and status code
+ Whitelist newWhitelist = new Whitelist(newPatterns, statusCode);
+
+ // replace the current whitelist with the new whitelist only if the current whitelist has not changed since we started
+ whitelistUpdated = this.whitelist.compareAndSet(currentWhitelist, newWhitelist);
+ }
+ }
+
+ /**
+ * Whitelist the specified request patterns, returning the specified responseCode for non-whitelisted
+ * requests.
+ *
+ * @param patterns regular expression strings matching URL patterns to whitelist. if empty or null,
+ * the whitelist will be enabled but will not match any URLs.
+ * @param responseCode the HTTP response code to return for non-whitelisted requests
+ */
+ public void whitelistRequests(String[] patterns, int responseCode) {
+ if (patterns == null || patterns.length == 0) {
+ this.enableEmptyWhitelist(responseCode);
+ } else {
+ this.whitelistRequests(Arrays.asList(patterns), responseCode);
+ }
+ }
+
+ @Override
+ public void enableEmptyWhitelist(int statusCode) {
+ whitelist.set(new Whitelist(statusCode));
+ }
+
+ @Override
+ public void disableWhitelist() {
+ whitelist.set(Whitelist.WHITELIST_DISABLED);
+ }
+
+ @Override
+ public void addHeader(String name, String value) {
+ additionalHeaders.put(name, value);
+ }
+
+ @Override
+ public void removeHeader(String name) {
+ additionalHeaders.remove(name);
+ }
+
+ @Override
+ public void removeAllHeaders() {
+ additionalHeaders.clear();
+ }
+
+ @Override
+ public Map getAllHeaders() {
+ return ImmutableMap.copyOf(additionalHeaders);
+ }
+
+ @Override
+ public void setHostNameResolver(AdvancedHostResolver resolver) {
+ delegatingResolver.setResolver(resolver);
+ }
+
+ @Override
+ public AdvancedHostResolver getHostNameResolver() {
+ return delegatingResolver.getResolver();
+ }
+
+ @Override
+ public boolean waitForQuiescence(long quietPeriod, long timeout, TimeUnit timeUnit) {
+ return activityMonitor.waitForQuiescence(quietPeriod, timeout, timeUnit);
+ }
+
+ /**
+ * Instructs this proxy to route traffic through an upstream proxy.
+ *
+ * Note: Using {@link #setChainedProxyManager(ChainedProxyManager)} will supersede any value set by this method. A chained
+ * proxy must be set before the proxy is started, though it can be changed after the proxy is started.
+ *
+ * @param chainedProxyAddress address of the upstream proxy
+ */
+ @Override
+ public void setChainedProxy(InetSocketAddress chainedProxyAddress) {
+ if (isStarted() && !bootstrappedWithDefaultChainedProxy.get()) {
+ throw new IllegalStateException("Cannot set a chained proxy after the proxy is started if the proxy was started without a chained proxy.");
+ }
+
+ upstreamProxyAddress = chainedProxyAddress;
+ }
+
+ @Override
+ public InetSocketAddress getChainedProxy() {
+ return upstreamProxyAddress;
+ }
+
+ /**
+ * Allows access to the LittleProxy {@link ChainedProxyManager} for fine-grained control of the chained proxies. To enable a single
+ * chained proxy, {@link BrowserMobProxy#setChainedProxy(InetSocketAddress)} is generally more convenient.
+ *
+ * Note: The chained proxy manager must be enabled before calling {@link #start()}.
+ *
+ * @param chainedProxyManager chained proxy manager to enable
+ */
+ public void setChainedProxyManager(ChainedProxyManager chainedProxyManager) {
+ if (isStarted()) {
+ throw new IllegalStateException("Cannot configure chained proxy manager after proxy has started.");
+ }
+
+ this.chainedProxyManager = chainedProxyManager;
+ }
+
+ /**
+ * Configures the Netty thread pool used by the LittleProxy back-end. See {@link ThreadPoolConfiguration} for details.
+ *
+ * @param threadPoolConfiguration thread pool configuration to use
+ */
+ public void setThreadPoolConfiguration(ThreadPoolConfiguration threadPoolConfiguration) {
+ if (isStarted()) {
+ throw new IllegalStateException("Cannot configure thread pool after proxy has started.");
+ }
+
+ this.threadPoolConfiguration = threadPoolConfiguration;
+ }
+
+ @Override
+ public void addFirstHttpFilterFactory(HttpFiltersSource filterFactory) {
+ filterFactories.add(0, filterFactory);
+ }
+
+ @Override
+ public void addLastHttpFilterFactory(HttpFiltersSource filterFactory) {
+ filterFactories.add(filterFactory);
+ }
+
+ /**
+ * Note: The current implementation of this method forces a maximum response size of 2 MiB. To adjust the maximum response size, or
+ * to disable aggregation (which disallows access to the {@link net.lightbody.bmp.util.HttpMessageContents}), you may add the filter source
+ * directly: addFirstHttpFilterFactory(new ResponseFilterAdapter.FilterSource(filter, bufferSizeInBytes));
+ */
+ @Override
+ public void addResponseFilter(ResponseFilter filter) {
+ addLastHttpFilterFactory(new ResponseFilterAdapter.FilterSource(filter));
+ }
+
+ /**
+ * Note: The current implementation of this method forces a maximum request size of 2 MiB. To adjust the maximum request size, or
+ * to disable aggregation (which disallows access to the {@link net.lightbody.bmp.util.HttpMessageContents}), you may add the filter source
+ * directly: addFirstHttpFilterFactory(new RequestFilterAdapter.FilterSource(filter, bufferSizeInBytes));
+ */
+ @Override
+ public void addRequestFilter(RequestFilter filter) {
+ addFirstHttpFilterFactory(new RequestFilterAdapter.FilterSource(filter));
+ }
+
+ @Override
+ public Map getRewriteRules() {
+ ImmutableMap.Builder builder = ImmutableMap.builder();
+ for (RewriteRule rewriteRule : rewriteRules) {
+ builder.put(rewriteRule.getPattern().pattern(), rewriteRule.getReplace());
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public void removeRewriteRule(String urlPattern) {
+ // normally removing elements from the list we are iterating over would not be possible, but since this is a CopyOnWriteArrayList
+ // the iterator it returns is a "snapshot" of the list that will not be affected by removal (and that does not support removal, either)
+ for (RewriteRule rewriteRule : rewriteRules) {
+ if (rewriteRule.getPattern().pattern().equals(urlPattern)) {
+ rewriteRules.remove(rewriteRule);
+ }
+ }
+ }
+
+ public boolean isStopped() {
+ return stopped.get();
+ }
+
+ public HarPage getCurrentHarPage() {
+ return currentHarPage;
+ }
+
+ public void addHttpFilterFactory(HttpFiltersSource filterFactory) {
+ filterFactories.add(filterFactory);
+ }
+
+ public List getFilterFactories() {
+ return filterFactories;
+ }
+
+ @Override
+ public void setMitmDisabled(boolean mitmDisabled) throws IllegalStateException {
+ if (isStarted()) {
+ throw new IllegalStateException("Cannot disable MITM after the proxy has been started");
+ }
+
+ this.mitmDisabled = mitmDisabled;
+ }
+
+ @Override
+ public void setMitmManager(MitmManager mitmManager) {
+ this.mitmManager = mitmManager;
+ }
+
+ @Override
+ public void setTrustAllServers(boolean trustAllServers) {
+ if (isStarted()) {
+ throw new IllegalStateException("Cannot disable upstream server verification after the proxy has been started");
+ }
+
+ if (trustAllServers) {
+ trustSource = null;
+ } else {
+ if (trustSource == null) {
+ trustSource = TrustSource.defaultTrustSource();
+ }
+ }
+ }
+
+ @Override
+ public void setTrustSource(TrustSource trustSource) {
+ if (isStarted()) {
+ throw new IllegalStateException("Cannot change TrustSource after proxy has been started");
+ }
+
+ this.trustSource = trustSource;
+ }
+
+ public boolean isMitmDisabled() {
+ return this.mitmDisabled;
+ }
+
+ public void setUseEcc(boolean useEcc) {
+ this.useEcc = useEcc;
+ }
+
+ /**
+ * Adds the basic browsermob-proxy filters, except for the relatively-expensive HAR capture filter.
+ */
+ protected void addBrowserMobFilters() {
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new ResolvedHostnameCacheFilter(originalRequest, ctx);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new RegisterRequestFilter(originalRequest, ctx, activityMonitor);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new HttpsOriginalHostCaptureFilter(originalRequest, ctx);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new BlacklistFilter(originalRequest, ctx, getBlacklist());
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ Whitelist currentWhitelist = whitelist.get();
+ return new WhitelistFilter(originalRequest, ctx, isWhitelistEnabled(), currentWhitelist.getStatusCode(), currentWhitelist.getPatterns());
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new AutoBasicAuthFilter(originalRequest, ctx, basicAuthCredentials);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new RewriteUrlFilter(originalRequest, ctx, rewriteRules);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new HttpsHostCaptureFilter(originalRequest, ctx);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest) {
+ return new AddHeadersFilter(originalRequest, additionalHeaders);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest) {
+ return new LatencyFilter(originalRequest, latencyMs);
+ }
+ });
+
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new UnregisterRequestFilter(originalRequest, ctx, activityMonitor);
+ }
+ });
+ }
+
+ private int getMaximumRequestBufferSize() {
+ int maxBufferSize = 0;
+ for (HttpFiltersSource source : filterFactories) {
+ int requestBufferSize = source.getMaximumRequestBufferSizeInBytes();
+ if (requestBufferSize > maxBufferSize) {
+ maxBufferSize = requestBufferSize;
+ }
+ }
+
+ return maxBufferSize;
+ }
+
+ private int getMaximumResponseBufferSize() {
+ int maxBufferSize = 0;
+ for (HttpFiltersSource source : filterFactories) {
+ int requestBufferSize = source.getMaximumResponseBufferSizeInBytes();
+ if (requestBufferSize > maxBufferSize) {
+ maxBufferSize = requestBufferSize;
+ }
+ }
+
+ return maxBufferSize;
+ }
+
+ /**
+ * Enables the HAR capture filter if it has not already been enabled. The filter will be added to the end of the filter chain.
+ * The HAR capture filter is relatively expensive, so this method is only called when a HAR is requested.
+ */
+ protected void addHarCaptureFilter() {
+ if (harCaptureFilterEnabled.compareAndSet(false, true)) {
+ // the HAR capture filter is (relatively) expensive, so only enable it when a HAR is being captured. furthermore,
+ // restricting the HAR capture filter to requests where the HAR exists, as well as excluding HTTP CONNECTs
+ // from the HAR capture filter, greatly simplifies the filter code.
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ Har har = getHar();
+ if (har != null && !ProxyUtils.isCONNECT(originalRequest)) {
+ return new HarCaptureFilter(originalRequest, ctx, har, getCurrentHarPage() == null ? null : getCurrentHarPage().getId(), getHarCaptureTypes());
+ } else {
+ return null;
+ }
+ }
+ });
+
+ // HTTP CONNECTs are a special case, since they require special timing and error handling
+ addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ Har har = getHar();
+ if (har != null && ProxyUtils.isCONNECT(originalRequest)) {
+ return new HttpConnectHarCaptureFilter(originalRequest, ctx, har, getCurrentHarPage() == null ? null : getCurrentHarPage().getId());
+ } else {
+ return null;
+ }
+ }
+ });
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/client/ClientUtil.java b/browsermob-core/src/main/java/net/lightbody/bmp/client/ClientUtil.java
new file mode 100644
index 000000000..9c8a151fd
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/client/ClientUtil.java
@@ -0,0 +1,120 @@
+package net.lightbody.bmp.client;
+
+import com.google.common.collect.ImmutableList;
+import net.lightbody.bmp.BrowserMobProxy;
+import net.lightbody.bmp.proxy.dns.AdvancedHostResolver;
+import net.lightbody.bmp.proxy.dns.ChainedHostResolver;
+import net.lightbody.bmp.proxy.dns.DnsJavaResolver;
+import net.lightbody.bmp.proxy.dns.NativeCacheManipulatingResolver;
+import net.lightbody.bmp.proxy.dns.NativeResolver;
+import org.openqa.selenium.Proxy;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+
+/**
+ * A utility class with convenience methods for clients using BrowserMob Proxy in embedded mode.
+ */
+public class ClientUtil {
+ /**
+ * Creates a {@link net.lightbody.bmp.proxy.dns.NativeCacheManipulatingResolver} instance that can be used when
+ * calling {@link net.lightbody.bmp.BrowserMobProxy#setHostNameResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
+ *
+ * @return a new NativeCacheManipulatingResolver
+ */
+ public static AdvancedHostResolver createNativeCacheManipulatingResolver() {
+ return new NativeCacheManipulatingResolver();
+ }
+
+ /**
+ * Creates a {@link net.lightbody.bmp.proxy.dns.NativeResolver} instance that does not support cache manipulation that can be used when
+ * calling {@link net.lightbody.bmp.BrowserMobProxy#setHostNameResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
+ *
+ * @return a new NativeResolver
+ */
+ public static AdvancedHostResolver createNativeResolver() {
+ return new NativeResolver();
+ }
+
+ /**
+ * Creates a {@link net.lightbody.bmp.proxy.dns.DnsJavaResolver} instance that can be used when
+ * calling {@link net.lightbody.bmp.BrowserMobProxy#setHostNameResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
+ *
+ * @return a new DnsJavaResolver
+ * @deprecated The dnsjava resolver has been deprecated in favor of the standard JVM resolver and will be removed in BMP >2.1.
+ */
+ public static AdvancedHostResolver createDnsJavaResolver() {
+ return new DnsJavaResolver();
+ }
+
+ /**
+ * Creates a {@link net.lightbody.bmp.proxy.dns.ChainedHostResolver} instance that first attempts to resolve a hostname using a
+ * {@link net.lightbody.bmp.proxy.dns.DnsJavaResolver}, then uses {@link net.lightbody.bmp.proxy.dns.NativeCacheManipulatingResolver}.
+ * Can be used when calling {@link net.lightbody.bmp.BrowserMobProxy#setHostNameResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
+ *
+ * @return a new ChainedHostResolver that resolves addresses first using a DnsJavaResolver, then using a NativeCacheManipulatingResolver
+ * @deprecated The dnsjava resolver has been deprecated in favor of the standard JVM resolver and will be removed in BMP >2.1.
+ */
+ public static AdvancedHostResolver createDnsJavaWithNativeFallbackResolver() {
+ return new ChainedHostResolver(ImmutableList.of(new DnsJavaResolver(), new NativeCacheManipulatingResolver()));
+ }
+
+ /**
+ * Creates a Selenium Proxy object from the BrowserMobProxy instance. The BrowserMobProxy must be started. Retrieves the address
+ * of the Proxy using {@link #getConnectableAddress()}.
+ *
+ * @param browserMobProxy started BrowserMobProxy instance to read connection information from
+ * @return a Selenium Proxy instance, configured to use the BrowserMobProxy instance as its proxy server
+ * @throws java.lang.IllegalStateException if the proxy has not been started.
+ */
+ public static org.openqa.selenium.Proxy createSeleniumProxy(BrowserMobProxy browserMobProxy) {
+ return createSeleniumProxy(browserMobProxy, getConnectableAddress());
+ }
+
+ /**
+ * Creates a Selenium Proxy object from the BrowserMobProxy instance, using the specified connectableAddress as the Selenium Proxy object's
+ * proxy address. Determines the port using {@link net.lightbody.bmp.BrowserMobProxy#getPort()}. The BrowserMobProxy must be started.
+ *
+ * @param browserMobProxy started BrowserMobProxy instance to read the port from
+ * @param connectableAddress the network address the Selenium Proxy will use to reach this BrowserMobProxy instance
+ * @return a Selenium Proxy instance, configured to use the BrowserMobProxy instance as its proxy server
+ * @throws java.lang.IllegalStateException if the proxy has not been started.
+ */
+ public static org.openqa.selenium.Proxy createSeleniumProxy(BrowserMobProxy browserMobProxy, InetAddress connectableAddress) {
+ return createSeleniumProxy(new InetSocketAddress(connectableAddress, browserMobProxy.getPort()));
+ }
+
+ /**
+ * Creates a Selenium Proxy object using the specified connectableAddressAndPort as the HTTP proxy server.
+ *
+ * @param connectableAddressAndPort the network address (or hostname) and port the Selenium Proxy will use to reach its
+ * proxy server (the InetSocketAddress may be unresolved).
+ * @return a Selenium Proxy instance, configured to use the specified address and port as its proxy server
+ */
+ public static org.openqa.selenium.Proxy createSeleniumProxy(InetSocketAddress connectableAddressAndPort) {
+ Proxy proxy = new Proxy();
+ proxy.setProxyType(Proxy.ProxyType.MANUAL);
+
+ String proxyStr = String.format("%s:%d", connectableAddressAndPort.getHostString(), connectableAddressAndPort.getPort());
+ proxy.setHttpProxy(proxyStr);
+ proxy.setSslProxy(proxyStr);
+
+ return proxy;
+ }
+
+ /**
+ * Attempts to retrieve a "connectable" address for this device that other devices on the network can use to connect to a local proxy.
+ * This is a "reasonable guess" that is suitable in many (but not all) common scenarios.
+ * TODO: define the algorithm used to discover a "connectable" local host
+ *
+ * @return a "reasonable guess" at an address that can be used by other machines on the network to reach this host
+ */
+ public static InetAddress getConnectableAddress() {
+ try {
+ return InetAddress.getLocalHost();
+ } catch (UnknownHostException e) {
+ throw new RuntimeException("Could not resolve localhost", e);
+ }
+ }
+}
diff --git a/src/main/java/org/browsermob/core/har/Har.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/Har.java
similarity index 60%
rename from src/main/java/org/browsermob/core/har/Har.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/Har.java
index b87d88f03..40e1987ad 100644
--- a/src/main/java/org/browsermob/core/har/Har.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/Har.java
@@ -1,6 +1,6 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.map.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.IOException;
@@ -8,7 +8,10 @@
import java.io.Writer;
public class Har {
- private HarLog log;
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private volatile HarLog log;
public Har() {
}
@@ -26,17 +29,14 @@ public void setLog(HarLog log) {
}
public void writeTo(Writer writer) throws IOException {
- ObjectMapper om = new ObjectMapper();
- om.writeValue(writer, this);
+ OBJECT_MAPPER.writeValue(writer, this);
}
public void writeTo(OutputStream os) throws IOException {
- ObjectMapper om = new ObjectMapper();
- om.writeValue(os, this);
+ OBJECT_MAPPER.writeValue(os, this);
}
public void writeTo(File file) throws IOException {
- ObjectMapper om = new ObjectMapper();
- om.writeValue(file, this);
+ OBJECT_MAPPER.writeValue(file, this);
}
}
diff --git a/src/main/java/org/browsermob/core/har/HarCache.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCache.java
similarity index 64%
rename from src/main/java/org/browsermob/core/har/HarCache.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCache.java
index ca9114e72..0a3594a33 100644
--- a/src/main/java/org/browsermob/core/har/HarCache.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCache.java
@@ -1,11 +1,11 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonInclude;
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarCache {
- private HarCacheStatus beforeRequest;
- private HarCacheStatus afterRequest;
+ private volatile HarCacheStatus beforeRequest;
+ private volatile HarCacheStatus afterRequest;
public HarCacheStatus getBeforeRequest() {
return beforeRequest;
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCacheStatus.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCacheStatus.java
new file mode 100644
index 000000000..7b8f5f347
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCacheStatus.java
@@ -0,0 +1,57 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.Date;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HarCacheStatus {
+ private volatile Date expires;
+ private volatile Date lastAccess;
+ private volatile String eTag;
+ private volatile int hitCount;
+ private volatile String comment = "";
+
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss")
+ public Date getExpires() {
+ return expires;
+ }
+
+ public void setExpires(Date expires) {
+ this.expires = expires;
+ }
+
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss")
+ public Date getLastAccess() {
+ return lastAccess;
+ }
+
+ public void setLastAccess(Date lastAccess) {
+ this.lastAccess = lastAccess;
+ }
+
+ public String geteTag() {
+ return eTag;
+ }
+
+ public void seteTag(String eTag) {
+ this.eTag = eTag;
+ }
+
+ public int getHitCount() {
+ return hitCount;
+ }
+
+ public void setHitCount(int hitCount) {
+ this.hitCount = hitCount;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarContent.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarContent.java
new file mode 100644
index 000000000..48b67583e
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarContent.java
@@ -0,0 +1,65 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HarContent {
+ private volatile long size;
+ private volatile Long compression;
+
+ // mimeType is required; though it shouldn't be set to null, if it is, it still needs to be included to comply with the HAR spec
+ @JsonInclude(JsonInclude.Include.ALWAYS)
+ private volatile String mimeType = "";
+
+ private volatile String text;
+ private volatile String encoding;
+ private volatile String comment = "";
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+
+ public Long getCompression() {
+ return compression;
+ }
+
+ public void setCompression(Long compression) {
+ this.compression = compression;
+ }
+
+ public String getMimeType() {
+ return mimeType;
+ }
+
+ public void setMimeType(String mimeType) {
+ this.mimeType = mimeType;
+ }
+
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ public String getEncoding() {
+ return encoding;
+ }
+
+ public void setEncoding(String encoding) {
+ this.encoding = encoding;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCookie.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCookie.java
new file mode 100644
index 000000000..b21b7cb34
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarCookie.java
@@ -0,0 +1,97 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.Date;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HarCookie {
+ private volatile String name;
+ private volatile String value;
+ private volatile String path;
+ private volatile String domain;
+ private volatile Date expires;
+ private volatile Boolean httpOnly;
+ private volatile Boolean secure;
+ private volatile String comment = "";
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public String getDomain() {
+ return domain;
+ }
+
+ public void setDomain(String domain) {
+ this.domain = domain;
+ }
+
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
+ public Date getExpires() {
+ return expires;
+ }
+
+ public void setExpires(Date expires) {
+ this.expires = expires;
+ }
+
+ public Boolean getHttpOnly() {
+ return httpOnly;
+ }
+
+ public void setHttpOnly(Boolean httpOnly) {
+ this.httpOnly = httpOnly;
+ }
+
+ public Boolean getSecure() {
+ return secure;
+ }
+
+ public void setSecure(Boolean secure) {
+ this.secure = secure;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ return "HarCookie{" +
+ "name='" + name + '\'' +
+ ", value='" + value + '\'' +
+ ", path='" + path + '\'' +
+ ", domain='" + domain + '\'' +
+ ", expires=" + expires +
+ ", httpOnly=" + httpOnly +
+ ", secure=" + secure +
+ ", comment='" + comment + '\'' +
+ '}';
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarEntry.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarEntry.java
new file mode 100644
index 000000000..1864a78c7
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarEntry.java
@@ -0,0 +1,160 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonAutoDetect
+public class HarEntry {
+ private volatile String pageref;
+ private volatile Date startedDateTime;
+ private volatile HarRequest request;
+ private volatile HarResponse response;
+ private volatile HarCache cache = new HarCache();
+ private volatile HarTimings timings = new HarTimings();
+ private volatile String serverIPAddress;
+ private volatile String connection;
+ private volatile String comment = "";
+
+ public HarEntry() {
+ }
+
+ public HarEntry(String pageref) {
+ this.pageref = pageref;
+ }
+
+ public String getPageref() {
+ return pageref;
+ }
+
+ public void setPageref(String pageref) {
+ this.pageref = pageref;
+ }
+
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
+ public Date getStartedDateTime() {
+ return startedDateTime;
+ }
+
+ public void setStartedDateTime(Date startedDateTime) {
+ this.startedDateTime = startedDateTime;
+ }
+
+ /**
+ * Retrieves the time for this HarEntry in milliseconds. To retrieve the time in another time unit, use {@link #getTime(java.util.concurrent.TimeUnit)}.
+ * Rather than storing the time directly, calculate the time from the HarTimings as required in the HAR spec.
+ * From https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html,
+ * section 4.2.16 timings:
+
+ Following must be true in case there are no -1 values (entry is an object in log.entries) :
+
+ entry.time == entry.timings.blocked + entry.timings.dns +
+ entry.timings.connect + entry.timings.send + entry.timings.wait +
+ entry.timings.receive;
+
+ * @return time for this HAR entry, in milliseconds
+ */
+ public long getTime() {
+ return getTime(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Retrieve the time for this HarEntry in the specified timeUnit. See {@link #getTime()} for details.
+ *
+ * @param timeUnit units of time to return
+ * @return time for this har entry
+ */
+ public long getTime(TimeUnit timeUnit) {
+ HarTimings timings = getTimings();
+ if (timings == null) {
+ return -1;
+ }
+
+ long timeNanos = 0;
+ if (timings.getBlocked(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getBlocked(TimeUnit.NANOSECONDS);
+ }
+
+ if (timings.getDns(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getDns(TimeUnit.NANOSECONDS);
+ }
+
+ if (timings.getConnect(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getConnect(TimeUnit.NANOSECONDS);
+ }
+
+ if (timings.getSend(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getSend(TimeUnit.NANOSECONDS);
+ }
+
+ if (timings.getWait(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getWait(TimeUnit.NANOSECONDS);
+ }
+
+ if (timings.getReceive(TimeUnit.NANOSECONDS) > 0) {
+ timeNanos += timings.getReceive(TimeUnit.NANOSECONDS);
+ }
+
+ return timeUnit.convert(timeNanos, TimeUnit.NANOSECONDS);
+ }
+
+ public HarRequest getRequest() {
+ return request;
+ }
+
+ public void setRequest(HarRequest request) {
+ this.request = request;
+ }
+
+ public HarResponse getResponse() {
+ return response;
+ }
+
+ public void setResponse(HarResponse response) {
+ this.response = response;
+ }
+
+ public HarCache getCache() {
+ return cache;
+ }
+
+ public void setCache(HarCache cache) {
+ this.cache = cache;
+ }
+
+ public HarTimings getTimings() {
+ return timings;
+ }
+
+ public void setTimings(HarTimings timings) {
+ this.timings = timings;
+ }
+
+ public String getServerIPAddress() {
+ return serverIPAddress;
+ }
+
+ public void setServerIPAddress(String serverIPAddress) {
+ this.serverIPAddress = serverIPAddress;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getConnection() {
+ return connection;
+ }
+
+ public void setConnection(String connection) {
+ this.connection = connection;
+ }
+}
diff --git a/src/main/java/org/browsermob/core/har/HarLog.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarLog.java
similarity index 51%
rename from src/main/java/org/browsermob/core/har/HarLog.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarLog.java
index e39352075..879e071f3 100644
--- a/src/main/java/org/browsermob/core/har/HarLog.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarLog.java
@@ -1,17 +1,18 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarLog {
- private String version = "1.2";
- private HarNameVersion creator;
- private HarNameVersion browser;
- private List pages = new CopyOnWriteArrayList();
- private List entries = new CopyOnWriteArrayList();
+ private final String version = "1.2";
+ private volatile HarNameVersion creator;
+ private volatile HarNameVersion browser;
+ private final List pages = new CopyOnWriteArrayList();
+ private final List entries = new CopyOnWriteArrayList();
+ private volatile String comment = "";
public HarLog() {
}
@@ -21,18 +22,10 @@ public HarLog(HarNameVersion creator) {
}
public void addPage(HarPage page) {
- if (pages == null) {
- pages = new CopyOnWriteArrayList();
- }
-
pages.add(page);
}
public void addEntry(HarEntry entry) {
- if (entries == null) {
- entries = new CopyOnWriteArrayList();
- }
-
entries.add(entry);
}
@@ -40,10 +33,6 @@ public String getVersion() {
return version;
}
- public void setVersion(String version) {
- this.version = version;
- }
-
public HarNameVersion getCreator() {
return creator;
}
@@ -64,15 +53,15 @@ public List getPages() {
return pages;
}
- public void setPages(List pages) {
- this.pages = pages;
- }
-
public List getEntries() {
return entries;
}
- public void setEntries(List entries) {
- this.entries = entries;
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
}
}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameValuePair.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameValuePair.java
new file mode 100644
index 000000000..b1723380c
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameValuePair.java
@@ -0,0 +1,43 @@
+package net.lightbody.bmp.core.har;
+
+public final class HarNameValuePair {
+ private final String name;
+ private final String value;
+
+ public HarNameValuePair(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public String toString() {
+ return name + "=" + value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ HarNameValuePair that = (HarNameValuePair) o;
+
+ if (name != null ? !name.equals(that.name) : that.name != null) return false;
+ if (value != null ? !value.equals(that.value) : that.value != null) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + (value != null ? value.hashCode() : 0);
+ return result;
+ }
+}
diff --git a/src/main/java/org/browsermob/core/har/HarNameVersion.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameVersion.java
similarity index 51%
rename from src/main/java/org/browsermob/core/har/HarNameVersion.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameVersion.java
index 6b7ae0752..b3d222bf7 100644
--- a/src/main/java/org/browsermob/core/har/HarNameVersion.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarNameVersion.java
@@ -1,11 +1,9 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
public class HarNameVersion {
- private String name;
- private String version;
-
- public HarNameVersion() {
- }
+ private final String name;
+ private final String version;
+ private volatile String comment = "";
public HarNameVersion(String name, String version) {
this.name = name;
@@ -16,15 +14,15 @@ public String getName() {
return name;
}
- public void setName(String name) {
- this.name = name;
- }
-
public String getVersion() {
return version;
}
- public void setVersion(String version) {
- this.version = version;
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
}
}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPage.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPage.java
new file mode 100644
index 000000000..7b63dd42e
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPage.java
@@ -0,0 +1,66 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.Date;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HarPage {
+ private volatile String id;
+ private volatile Date startedDateTime;
+ private volatile String title = "";
+ private final HarPageTimings pageTimings = new HarPageTimings();
+ private volatile String comment = "";
+
+ public HarPage() {
+ }
+
+ public HarPage(String id) {
+ this(id, "");
+ }
+
+ public HarPage(String id, String title) {
+ this.id = id;
+ this.title = title;
+ startedDateTime = new Date();
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
+ public Date getStartedDateTime() {
+ return startedDateTime;
+ }
+
+ public void setStartedDateTime(Date startedDateTime) {
+ this.startedDateTime = startedDateTime;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
+ public void setTitle(String title) {
+ this.title = title;
+ }
+
+ public HarPageTimings getPageTimings() {
+ return pageTimings;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+}
diff --git a/src/main/java/org/browsermob/core/har/HarPageTimings.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPageTimings.java
similarity index 57%
rename from src/main/java/org/browsermob/core/har/HarPageTimings.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPageTimings.java
index d87aad1c9..56d415e83 100644
--- a/src/main/java/org/browsermob/core/har/HarPageTimings.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPageTimings.java
@@ -1,11 +1,12 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.annotate.JsonWriteNullProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
-@JsonWriteNullProperties(value=false)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarPageTimings {
- private Long onContentLoad;
- private Long onLoad;
+ private volatile Long onContentLoad;
+ private volatile Long onLoad;
+ private volatile String comment = "";
public HarPageTimings() {
}
@@ -30,4 +31,13 @@ public Long getOnLoad() {
public void setOnLoad(Long onLoad) {
this.onLoad = onLoad;
}
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
}
diff --git a/src/main/java/org/browsermob/core/har/HarPostData.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostData.java
similarity index 53%
rename from src/main/java/org/browsermob/core/har/HarPostData.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostData.java
index 3d902ec28..a9f10d1a4 100644
--- a/src/main/java/org/browsermob/core/har/HarPostData.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostData.java
@@ -1,14 +1,15 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.annotate.JsonWriteNullProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
-@JsonWriteNullProperties(value=false)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarPostData {
- private String mimeType;
- private List params;
- private String text;
+ private volatile String mimeType;
+ private volatile List params;
+ private volatile String text;
+ private volatile String comment = "";
public String getMimeType() {
return mimeType;
@@ -33,4 +34,12 @@ public String getText() {
public void setText(String text) {
this.text = text;
}
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
}
diff --git a/src/main/java/org/browsermob/core/har/HarPostDataParam.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostDataParam.java
similarity index 62%
rename from src/main/java/org/browsermob/core/har/HarPostDataParam.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostDataParam.java
index fca534749..38c5f5980 100644
--- a/src/main/java/org/browsermob/core/har/HarPostDataParam.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarPostDataParam.java
@@ -1,13 +1,14 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.annotate.JsonWriteNullProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
-@JsonWriteNullProperties(value=false)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarPostDataParam {
- private String name;
- private String value;
- private String fileName;
- private String contentType;
+ private volatile String name;
+ private volatile String value;
+ private volatile String fileName;
+ private volatile String contentType;
+ private volatile String comment = "";
public HarPostDataParam() {
}
@@ -48,4 +49,12 @@ public String getContentType() {
public void setContentType(String contentType) {
this.contentType = contentType;
}
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
}
diff --git a/src/main/java/org/browsermob/core/har/HarRequest.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarRequest.java
similarity index 62%
rename from src/main/java/org/browsermob/core/har/HarRequest.java
rename to browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarRequest.java
index ff9d9a159..2cfe68957 100644
--- a/src/main/java/org/browsermob/core/har/HarRequest.java
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarRequest.java
@@ -1,21 +1,22 @@
-package org.browsermob.core.har;
+package net.lightbody.bmp.core.har;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class HarRequest {
- private String method;
- private String url;
- private String httpVersion;
- private List cookies = new CopyOnWriteArrayList();
- private List headers = new CopyOnWriteArrayList();
- private List queryString = new CopyOnWriteArrayList();
- private HarPostData postData;
- private long headersSize; // Odd grammar in spec
- private long bodySize;
+ private volatile String method;
+ private volatile String url;
+ private volatile String httpVersion;
+ private final List cookies = new CopyOnWriteArrayList();
+ private final List headers = new CopyOnWriteArrayList();
+ private final List queryString = new CopyOnWriteArrayList();
+ private volatile HarPostData postData;
+ private volatile long headersSize; // Odd grammar in spec
+ private volatile long bodySize;
+ private volatile String comment = "";
public HarRequest() {
}
@@ -54,26 +55,14 @@ public List getCookies() {
return cookies;
}
- public void setCookies(List cookies) {
- this.cookies = cookies;
- }
-
public List getHeaders() {
return headers;
}
- public void setHeaders(List headers) {
- this.headers = headers;
- }
-
public List getQueryString() {
return queryString;
}
- public void setQueryString(List queryString) {
- this.queryString = queryString;
- }
-
public HarPostData getPostData() {
return postData;
}
@@ -97,4 +86,13 @@ public long getBodySize() {
public void setBodySize(long bodySize) {
this.bodySize = bodySize;
}
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java
new file mode 100644
index 000000000..f82f248dd
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java
@@ -0,0 +1,117 @@
+package net.lightbody.bmp.core.har;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class HarResponse {
+ private volatile int status;
+ private volatile String statusText;
+ private volatile String httpVersion;
+ private final List cookies = new CopyOnWriteArrayList();
+ private final List headers = new CopyOnWriteArrayList();
+ private final HarContent content = new HarContent();
+ private volatile String redirectURL = "";
+
+ /* the values of headersSize and bodySize are set to -1 by default, in accordance with the HAR spec:
+ headersSize [number] - Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body. Set to -1 if the info is not available.
+ bodySize [number] - Size of the request body (POST data payload) in bytes. Set to -1 if the info is not available.
+ */
+ private volatile long headersSize = -1;
+ private volatile long bodySize = -1;
+ private volatile String comment = "";
+
+ /**
+ * A custom field indicating that an error occurred, such as DNS resolution failure.
+ */
+ @JsonProperty("_error")
+ private volatile String error;
+
+ public HarResponse() {
+ }
+
+ public HarResponse(int status, String statusText, String httpVersion) {
+ this.status = status;
+ this.statusText = statusText;
+ this.httpVersion = httpVersion;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getStatusText() {
+ return statusText;
+ }
+
+ public void setStatusText(String statusText) {
+ this.statusText = statusText;
+ }
+
+ public String getHttpVersion() {
+ return httpVersion;
+ }
+
+ public void setHttpVersion(String httpVersion) {
+ this.httpVersion = httpVersion;
+ }
+
+ public List getCookies() {
+ return cookies;
+ }
+
+ public List getHeaders() {
+ return headers;
+ }
+
+ public HarContent getContent() {
+ return content;
+ }
+
+ public String getRedirectURL() {
+ return redirectURL;
+ }
+
+ public void setRedirectURL(String redirectURL) {
+ this.redirectURL = redirectURL;
+ }
+
+ public long getHeadersSize() {
+ return headersSize;
+ }
+
+ public void setHeadersSize(long headersSize) {
+ this.headersSize = headersSize;
+ }
+
+ public long getBodySize() {
+ return bodySize;
+ }
+
+ public void setBodySize(long bodySize) {
+ this.bodySize = bodySize;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getError() {
+ return error;
+ }
+
+ public void setError(String error) {
+ this.error = error;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarTimings.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarTimings.java
new file mode 100644
index 000000000..efb8d8612
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarTimings.java
@@ -0,0 +1,177 @@
+package net.lightbody.bmp.core.har;
+
+import java.util.concurrent.TimeUnit;
+
+public class HarTimings {
+ // optional values are initialized to -1, which indicates they do not apply to the current request, according to the HAR spec
+ private volatile long blockedNanos = -1;
+ private volatile long dnsNanos = -1;
+ private volatile long connectNanos = -1;
+ private volatile long sendNanos;
+ private volatile long waitNanos;
+ private volatile long receiveNanos;
+ private volatile long sslNanos = -1;
+ private volatile String comment = "";
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ // the following getters and setters take a TimeUnit parameter, to allow finer precision control when no marshalling to JSON
+ public long getBlocked(TimeUnit timeUnit) {
+ if (blockedNanos == -1) {
+ return -1;
+ } else {
+ return timeUnit.convert(blockedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ public void setBlocked(long blocked, TimeUnit timeUnit) {
+ if (blocked == -1) {
+ this.blockedNanos = -1;
+ } else {
+ this.blockedNanos = TimeUnit.NANOSECONDS.convert(blocked, timeUnit);
+ }
+ }
+
+ public long getDns(TimeUnit timeUnit) {
+ if (dnsNanos == -1) {
+ return -1;
+ } else {
+ return timeUnit.convert(dnsNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ public void setDns(long dns, TimeUnit timeUnit) {
+ if (dns == -1) {
+ this.dnsNanos = -1;
+ } else{
+ this.dnsNanos = TimeUnit.NANOSECONDS.convert(dns, timeUnit);
+ }
+ }
+
+ public long getConnect(TimeUnit timeUnit) {
+ if (connectNanos == -1) {
+ return -1;
+ } else {
+ return timeUnit.convert(connectNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ public void setConnect(long connect, TimeUnit timeUnit) {
+ if (connect == -1) {
+ this.connectNanos = -1;
+ } else {
+ this.connectNanos = TimeUnit.NANOSECONDS.convert(connect, timeUnit);
+ }
+ }
+
+ /*
+ According to the HAR spec:
+ The send, wait and receive timings are not optional and must have non-negative values.
+ */
+ public long getSend(TimeUnit timeUnit) {
+ return timeUnit.convert(sendNanos, TimeUnit.NANOSECONDS);
+ }
+
+ public void setSend(long send, TimeUnit timeUnit) {
+ this.sendNanos = TimeUnit.NANOSECONDS.convert(send, timeUnit);
+ }
+
+ public long getWait(TimeUnit timeUnit) {
+ return timeUnit.convert(waitNanos, TimeUnit.NANOSECONDS);
+ }
+
+ public void setWait(long wait, TimeUnit timeUnit) {
+ this.waitNanos = TimeUnit.NANOSECONDS.convert(wait, timeUnit);
+ }
+
+ public long getReceive(TimeUnit timeUnit) {
+ return timeUnit.convert(receiveNanos, TimeUnit.NANOSECONDS);
+ }
+
+ public void setReceive(long receive, TimeUnit timeUnit) {
+ this.receiveNanos = TimeUnit.NANOSECONDS.convert(receive, timeUnit);
+ }
+
+ public long getSsl(TimeUnit timeUnit) {
+ if (sslNanos == -1) {
+ return -1;
+ } else {
+ return timeUnit.convert(sslNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ public void setSsl(long ssl, TimeUnit timeUnit) {
+ if (ssl == -1) {
+ this.sslNanos = -1;
+ } else {
+ this.sslNanos = TimeUnit.NANOSECONDS.convert(ssl, timeUnit);
+ }
+ }
+
+ // the following getters and setters assume TimeUnit.MILLISECOND precision. this allows jackson to generate ms values (in accordance
+ // with the HAR spec), and also preserves compatibility with the legacy methods. optional methods are also declared as Long instead of
+ // long (even though they always have values), to preserve compatibility. in general, the getters/setters which take TimeUnits
+ // should always be preferred.
+ public Long getBlocked() {
+ return getBlocked(TimeUnit.MILLISECONDS);
+ }
+
+ public void setBlocked(long blocked) {
+ setBlocked(blocked, TimeUnit.MILLISECONDS);
+ }
+
+ public Long getDns() {
+ return getDns(TimeUnit.MILLISECONDS);
+ }
+
+ public void setDns(long dns) {
+ setDns(dns, TimeUnit.MILLISECONDS);
+ }
+
+ public Long getConnect() {
+ return getConnect(TimeUnit.MILLISECONDS);
+ }
+
+ public void setConnect(long connect) {
+ setConnect(connect, TimeUnit.MILLISECONDS);
+ }
+
+ public long getSend() {
+ return getSend(TimeUnit.MILLISECONDS);
+ }
+
+ public void setSend(long send) {
+ setSend(send, TimeUnit.MILLISECONDS);
+ }
+
+ public long getWait() {
+ return getWait(TimeUnit.MILLISECONDS);
+ }
+
+ public void setWait(long wait) {
+ setWait(wait, TimeUnit.MILLISECONDS);
+ }
+
+ public long getReceive() {
+ return getReceive(TimeUnit.MILLISECONDS);
+ }
+
+ public void setReceive(long receive) {
+ setReceive(receive, TimeUnit.MILLISECONDS);
+ }
+
+ public Long getSsl() {
+ return getSsl(TimeUnit.MILLISECONDS);
+ }
+
+ public void setSsl(long ssl) {
+ setSsl(ssl, TimeUnit.MILLISECONDS);
+ }
+
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/exception/DecompressionException.java b/browsermob-core/src/main/java/net/lightbody/bmp/exception/DecompressionException.java
new file mode 100644
index 000000000..a77e1a68b
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/exception/DecompressionException.java
@@ -0,0 +1,23 @@
+package net.lightbody.bmp.exception;
+
+/**
+ * Indicates that an error occurred while decompressing content.
+ */
+public class DecompressionException extends RuntimeException {
+ private static final long serialVersionUID = 8666473793514307564L;
+
+ public DecompressionException() {
+ }
+
+ public DecompressionException(String message) {
+ super(message);
+ }
+
+ public DecompressionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DecompressionException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/exception/UnsupportedCharsetException.java b/browsermob-core/src/main/java/net/lightbody/bmp/exception/UnsupportedCharsetException.java
new file mode 100644
index 000000000..2336dbbf8
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/exception/UnsupportedCharsetException.java
@@ -0,0 +1,23 @@
+package net.lightbody.bmp.exception;
+
+/**
+ * A checked exception wrapper for {@link java.nio.charset.UnsupportedCharsetException}. This exception is checked to prevent
+ * situations where an unsupported character set in e.g. a Content-Type header causes the proxy to fail completely, rather
+ * than fallback to some suitable default behavior, such as not parsing the text contents of a message.
+ */
+public class UnsupportedCharsetException extends Exception {
+ public UnsupportedCharsetException(java.nio.charset.UnsupportedCharsetException e) {
+ super(e);
+
+ if (e == null) {
+ throw new IllegalArgumentException("net.lightbody.bmp.exception.UnsupportedCharsetException must be initialized with a non-null instance of java.nio.charset.UnsupportedCharsetException");
+ }
+ }
+
+ /**
+ * @return the underlying {@link java.nio.charset.UnsupportedCharsetException} that this exception wraps.
+ */
+ public java.nio.charset.UnsupportedCharsetException getUnsupportedCharsetExceptionCause() {
+ return (java.nio.charset.UnsupportedCharsetException) this.getCause();
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/AddHeadersFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/AddHeadersFilter.java
new file mode 100644
index 000000000..5b584ecfd
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/AddHeadersFilter.java
@@ -0,0 +1,40 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Adds the headers specified in the constructor to this request. The filter does not make a defensive copy of the map, so there is no guarantee
+ * that the map at the time of construction will contain the same values when the filter is actually invoked, if the map is modified concurrently.
+ */
+public class AddHeadersFilter extends HttpFiltersAdapter {
+ private final Map additionalHeaders;
+
+ public AddHeadersFilter(HttpRequest originalRequest, Map additionalHeaders) {
+ super(originalRequest);
+
+ if (additionalHeaders != null) {
+ this.additionalHeaders = additionalHeaders;
+ } else {
+ this.additionalHeaders = Collections.emptyMap();
+ }
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ for (Map.Entry header : additionalHeaders.entrySet()) {
+ httpRequest.headers().add(header.getKey(), header.getValue());
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/AutoBasicAuthFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/AutoBasicAuthFilter.java
new file mode 100644
index 000000000..758f9bfaa
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/AutoBasicAuthFilter.java
@@ -0,0 +1,54 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import org.littleshoot.proxy.impl.ProxyUtils;
+
+import java.util.Map;
+
+/**
+ * A filter that adds Basic authentication information to non-CONNECT requests. Takes a map of domain names to base64-encoded
+ * Basic auth credentials as a constructor parameter. If a key in the map matches the hostname of a filtered request, an Authorization
+ * header will be added to the request.
+ *
+ * The Authorization header itself is specified in RFC 7235, section 4.2: https://tools.ietf.org/html/rfc7235#section-4.2
+ * The Basic authentication scheme is specified in RFC 2617, section 2: https://tools.ietf.org/html/rfc2617#section-2
+ */
+public class AutoBasicAuthFilter extends HttpsAwareFiltersAdapter {
+ private final Map credentialsByHostname;
+
+ public AutoBasicAuthFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Map credentialsByHostname) {
+ super(originalRequest, ctx);
+
+ this.credentialsByHostname = credentialsByHostname;
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (credentialsByHostname.isEmpty()) {
+ return null;
+ }
+
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ // providing authorization during a CONNECT is generally not useful
+ if (ProxyUtils.isCONNECT(httpRequest)) {
+ return null;
+ }
+
+ String hostname = getHost(httpRequest);
+
+ // if there is an entry in the credentials map matching this hostname, add the credentials to the request
+ String base64CredentialsForHostname = credentialsByHostname.get(hostname);
+ if (base64CredentialsForHostname != null) {
+ httpRequest.headers().add(HttpHeaders.Names.AUTHORIZATION, "Basic " + base64CredentialsForHostname);
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/BlacklistFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/BlacklistFilter.java
new file mode 100644
index 000000000..b267673e1
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/BlacklistFilter.java
@@ -0,0 +1,58 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import net.lightbody.bmp.proxy.BlacklistEntry;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Applies blacklist entries to this request. The filter does not make a defensive copy of the blacklist entries, so there is no guarantee
+ * that the blacklist at the time of construction will contain the same values when the filter is actually invoked, if the entries are modified concurrently.
+ */
+public class BlacklistFilter extends HttpsAwareFiltersAdapter {
+ private final Collection blacklistedUrls;
+
+ public BlacklistFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Collection blacklistedUrls) {
+ super(originalRequest, ctx);
+
+ if (blacklistedUrls != null) {
+ this.blacklistedUrls = blacklistedUrls;
+ } else {
+ this.blacklistedUrls = Collections.emptyList();
+ }
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ String url = getFullUrl(httpRequest);
+
+ for (BlacklistEntry entry : blacklistedUrls) {
+ if (HttpMethod.CONNECT.equals(httpRequest.getMethod()) && entry.getHttpMethodPattern() == null) {
+ // do not allow CONNECTs to be blacklisted unless a method pattern is explicitly specified
+ continue;
+ }
+
+ if (entry.matches(url, httpRequest.getMethod().name())) {
+ HttpResponseStatus status = HttpResponseStatus.valueOf(entry.getStatusCode());
+ HttpResponse resp = new DefaultFullHttpResponse(httpRequest.getProtocolVersion(), status);
+ HttpHeaders.setContentLength(resp, 0L);
+
+ return resp;
+ }
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java
new file mode 100644
index 000000000..6ddf1331f
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java
@@ -0,0 +1,313 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import net.lightbody.bmp.BrowserMobProxyServer;
+import org.littleshoot.proxy.HttpFilters;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.littleshoot.proxy.HttpFiltersSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The filter "driver" that delegates to all chained filters specified by the proxy server.
+ */
+public class BrowserMobHttpFilterChain extends HttpFiltersAdapter {
+ private static final Logger log = LoggerFactory.getLogger(BrowserMobHttpFilterChain.class);
+
+ private final BrowserMobProxyServer proxyServer;
+
+ private final List filters;
+
+ public BrowserMobHttpFilterChain(BrowserMobProxyServer proxyServer, HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+
+ this.proxyServer = proxyServer;
+
+ if (proxyServer.getFilterFactories() != null) {
+ filters = new ArrayList<>(proxyServer.getFilterFactories().size());
+
+ // instantiate all HttpFilters using the proxy's filter factories
+ for (HttpFiltersSource filterFactory : proxyServer.getFilterFactories()) {
+ HttpFilters filter = filterFactory.filterRequest(originalRequest, ctx);
+ // allow filter factories to avoid adding a filter on a per-request basis by returning a null
+ // HttpFilters instance
+ if (filter != null) {
+ filters.add(filter);
+ }
+ }
+ } else {
+ filters = Collections.emptyList();
+ }
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (proxyServer.isStopped()) {
+ log.warn("Aborting request to {} because proxy is stopped", originalRequest.getUri());
+ HttpResponse abortedResponse = new DefaultFullHttpResponse(originalRequest.getProtocolVersion(), HttpResponseStatus.SERVICE_UNAVAILABLE);
+ HttpHeaders.setContentLength(abortedResponse, 0L);
+ return abortedResponse;
+ }
+
+ for (HttpFilters filter : filters) {
+ try {
+ HttpResponse filterResponse = filter.clientToProxyRequest(httpObject);
+ if (filterResponse != null) {
+ // if we are short-circuiting the response to an HttpRequest, update ModifiedRequestAwareFilter instances
+ // with this (possibly) modified HttpRequest before returning the short-circuit response
+ if (httpObject instanceof HttpRequest) {
+ updateFiltersWithModifiedResponse((HttpRequest) httpObject);
+ }
+
+ return filterResponse;
+ }
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ // if this httpObject is the HTTP request, set the modified request object on all ModifiedRequestAwareFilter
+ // instances, so they have access to all modifications the request filters made while filtering
+ if (httpObject instanceof HttpRequest) {
+ updateFiltersWithModifiedResponse((HttpRequest) httpObject);
+ }
+
+ return null;
+ }
+
+ @Override
+ public HttpResponse proxyToServerRequest(HttpObject httpObject) {
+ for (HttpFilters filter : filters) {
+ try {
+ HttpResponse filterResponse = filter.proxyToServerRequest(httpObject);
+ if (filterResponse != null) {
+ return filterResponse;
+ }
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public void proxyToServerRequestSending() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerRequestSending();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+
+ @Override
+ public HttpObject serverToProxyResponse(HttpObject httpObject) {
+ HttpObject processedHttpObject = httpObject;
+
+ for (HttpFilters filter : filters) {
+ try {
+ processedHttpObject = filter.serverToProxyResponse(processedHttpObject);
+ if (processedHttpObject == null) {
+ return null;
+ }
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ return processedHttpObject;
+ }
+
+ @Override
+ public void serverToProxyResponseTimedOut() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.serverToProxyResponseTimedOut();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void serverToProxyResponseReceiving() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.serverToProxyResponseReceiving();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHostAndPort) {
+ InetSocketAddress overrideAddress = null;
+ String newServerHostAndPort = resolvingServerHostAndPort;
+
+ for (HttpFilters filter : filters) {
+ try {
+ InetSocketAddress filterResult = filter.proxyToServerResolutionStarted(newServerHostAndPort);
+ if (filterResult != null) {
+ overrideAddress = filterResult;
+ newServerHostAndPort = filterResult.getHostString() + ":" + filterResult.getPort();
+ }
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ return overrideAddress;
+ }
+
+ @Override
+ public void proxyToServerResolutionFailed(String hostAndPort) {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerResolutionFailed(hostAndPort);
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerResolutionSucceeded(serverHostAndPort, resolvedRemoteAddress);
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ super.proxyToServerResolutionSucceeded(serverHostAndPort, resolvedRemoteAddress);
+ }
+
+ @Override
+ public void proxyToServerConnectionStarted() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerConnectionStarted();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionSSLHandshakeStarted() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerConnectionSSLHandshakeStarted();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionFailed() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerConnectionFailed();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerConnectionSucceeded(serverCtx);
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void proxyToServerRequestSent() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerRequestSent();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public void serverToProxyResponseReceived() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.serverToProxyResponseReceived();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ @Override
+ public HttpObject proxyToClientResponse(HttpObject httpObject) {
+ HttpObject processedHttpObject = httpObject;
+ for (HttpFilters filter : filters) {
+ try {
+ processedHttpObject = filter.proxyToClientResponse(processedHttpObject);
+ if (processedHttpObject == null) {
+ return null;
+ }
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+
+ return processedHttpObject;
+ }
+
+ @Override
+ public void proxyToServerConnectionQueued() {
+ for (HttpFilters filter : filters) {
+ try {
+ filter.proxyToServerConnectionQueued();
+ } catch (RuntimeException e) {
+ log.warn("Filter in filter chain threw exception. Filter method may have been aborted.", e);
+ }
+ }
+ }
+
+ /**
+ * Updates {@link ModifiedRequestAwareFilter} filters with the final, modified request after all request filters have
+ * processed the request.
+ *
+ * @param modifiedRequest the modified HttpRequest after all filters have finished processing it
+ */
+ private void updateFiltersWithModifiedResponse(HttpRequest modifiedRequest) {
+ for (HttpFilters filter : filters) {
+ if (filter instanceof ModifiedRequestAwareFilter) {
+ ModifiedRequestAwareFilter requestCaptureFilter = (ModifiedRequestAwareFilter) filter;
+ try {
+ requestCaptureFilter.setModifiedHttpRequest(modifiedRequest);
+ } catch (RuntimeException e) {
+ log.warn("ModifiedRequestAwareFilter in filter chain threw exception while setting modified HTTP request.", e);
+ }
+ }
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ClientRequestCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ClientRequestCaptureFilter.java
new file mode 100644
index 000000000..d6bd1b58c
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ClientRequestCaptureFilter.java
@@ -0,0 +1,93 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * This filter captures requests from the client (headers and content).
+ *
+ * The filter can be used in one of three ways: (1) directly, by adding the filter to the filter chain; (2) by subclassing
+ * the filter and overriding its filter methods; or (3) by invoking the filter directly from within another filter (see
+ * {@link net.lightbody.bmp.filters.HarCaptureFilter} for an example of the latter).
+ */
+public class ClientRequestCaptureFilter extends HttpFiltersAdapter {
+ /**
+ * Populated by clientToProxyRequest() when processing the HttpRequest object. Unlike originalRequest,
+ * this represents the "real" request that is being sent to the server, including headers.
+ */
+ private volatile HttpRequest httpRequest;
+
+ /**
+ * Populated by clientToProxyRequest() when processing the HttpContent objects. If the request is chunked,
+ * it will be populated across multiple calls to clientToProxyRequest().
+ */
+ private final ByteArrayOutputStream requestContents = new ByteArrayOutputStream();
+
+ /**
+ * Populated by clientToProxyRequest() when processing the LastHttpContent.
+ */
+ private volatile HttpHeaders trailingHeaders;
+
+ public ClientRequestCaptureFilter(HttpRequest originalRequest) {
+ super(originalRequest);
+ }
+
+ public ClientRequestCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ this.httpRequest = (HttpRequest) httpObject;
+ }
+
+ if (httpObject instanceof HttpContent) {
+ HttpContent httpContent = (HttpContent) httpObject;
+
+ storeRequestContent(httpContent);
+
+ if (httpContent instanceof LastHttpContent) {
+ LastHttpContent lastHttpContent = (LastHttpContent) httpContent;
+ trailingHeaders = lastHttpContent .trailingHeaders();
+ }
+ }
+
+ return null;
+ }
+
+ protected void storeRequestContent(HttpContent httpContent) {
+ ByteBuf bufferedContent = httpContent.content();
+ byte[] content = BrowserMobHttpUtil.extractReadableBytes(bufferedContent);
+
+ try {
+ requestContents.write(content);
+ } catch (IOException e) {
+ // can't happen
+ }
+ }
+
+ public HttpRequest getHttpRequest() {
+ return httpRequest;
+ }
+
+ public byte[] getFullRequestContents() {
+ return requestContents.toByteArray();
+ }
+
+ public HttpHeaders getTrailingHeaders() {
+ return trailingHeaders;
+ }
+
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java
new file mode 100644
index 000000000..ca8c044c2
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java
@@ -0,0 +1,766 @@
+package net.lightbody.bmp.filters;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.BaseEncoding;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
+import io.netty.handler.codec.http.cookie.Cookie;
+import io.netty.handler.codec.http.cookie.ServerCookieDecoder;
+import net.lightbody.bmp.core.har.Har;
+import net.lightbody.bmp.core.har.HarCookie;
+import net.lightbody.bmp.core.har.HarEntry;
+import net.lightbody.bmp.core.har.HarNameValuePair;
+import net.lightbody.bmp.core.har.HarPostData;
+import net.lightbody.bmp.core.har.HarPostDataParam;
+import net.lightbody.bmp.core.har.HarRequest;
+import net.lightbody.bmp.core.har.HarResponse;
+import net.lightbody.bmp.exception.UnsupportedCharsetException;
+import net.lightbody.bmp.filters.support.HttpConnectTiming;
+import net.lightbody.bmp.filters.util.HarCaptureUtil;
+import net.lightbody.bmp.proxy.CaptureType;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.impl.ProxyUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class HarCaptureFilter extends HttpsAwareFiltersAdapter {
+ private static final Logger log = LoggerFactory.getLogger(HarCaptureFilter.class);
+
+ /**
+ * The currently active HAR at the time the current request is received.
+ */
+ private final Har har;
+
+ /**
+ * The harEntry is created when this filter is constructed and is shared by both the clientToProxyRequest
+ * and serverToProxyResponse methods. It is added to the HarLog when the request is received from the client.
+ */
+ private final HarEntry harEntry;
+
+ /**
+ * The requestCaptureFilter captures all request content, including headers, trailing headers, and content. The HarCaptureFilter
+ * delegates to it when the clientToProxyRequest() callback is invoked. If this request does not need content capture, the
+ * ClientRequestCaptureFilter filter will not be instantiated and will not capture content.
+ */
+ private final ClientRequestCaptureFilter requestCaptureFilter;
+
+ /**
+ * Like requestCaptureFilter above, HarCaptureFilter delegates to responseCaptureFilter to capture response contents. If content capture
+ * is not required for this request, the filter will not be instantiated or invoked.
+ */
+ private final ServerResponseCaptureFilter responseCaptureFilter;
+
+ /**
+ * The CaptureType data types to capture in this request.
+ */
+ private final EnumSet dataToCapture;
+
+ /**
+ * Populated by proxyToServerResolutionStarted when DNS resolution starts. If any previous filters already resolved the address, their resolution time
+ * will not be included in this time.
+ */
+ private volatile long dnsResolutionStartedNanos;
+
+ private volatile long connectionQueuedNanos;
+ private volatile long connectionStartedNanos;
+
+ private volatile long sendStartedNanos;
+ private volatile long sendFinishedNanos;
+
+ private volatile long responseReceiveStartedNanos;
+
+ /**
+ * The address of the client making the request. Captured in the constructor and used when calculating and capturing ssl handshake and connect
+ * timing information for SSL connections.
+ */
+ private final InetSocketAddress clientAddress;
+
+ /**
+ * Request body size is determined by the actual size of the data the client sends. The filter does not use the Content-Length header to determine request size.
+ */
+ private final AtomicInteger requestBodySize = new AtomicInteger(0);
+
+ /**
+ * Response body size is determined by the actual size of the data the server sends.
+ */
+ private final AtomicInteger responseBodySize = new AtomicInteger(0);
+
+ /**
+ * The "real" original request, as captured by the {@link #clientToProxyRequest(io.netty.handler.codec.http.HttpObject)} method.
+ */
+ private volatile HttpRequest capturedOriginalRequest;
+
+ /**
+ * True if this filter instance processed a {@link #proxyToServerResolutionSucceeded(String, java.net.InetSocketAddress)} call, indicating
+ * that the hostname was resolved and populated in the HAR (if this is not a CONNECT).
+ */
+ private volatile boolean addressResolved = false;
+
+ /**
+ * Create a new instance of the HarCaptureFilter that will capture request and response information. If no har is specified in the
+ * constructor, this filter will do nothing.
+ *
+ * Regardless of the CaptureTypes specified in dataToCapture, the HarCaptureFilter will always capture:
+ *
+ *
Request and response sizes
+ *
HTTP request and status lines
+ *
Page timing information
+ *
+ *
+ * @param originalRequest the original HttpRequest from the HttpFiltersSource factory
+ * @param har a reference to the ProxyServer's current HAR file at the time this request is received (can be null if HAR capture is not required)
+ * @param currentPageRef the ProxyServer's currentPageRef at the time this request is received from the client
+ * @param dataToCapture the data types to capture for this request. null or empty set indicates only basic information will be
+ * captured (see {@link net.lightbody.bmp.proxy.CaptureType} for information on data collected for each CaptureType)
+ */
+ public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Har har, String currentPageRef, Set dataToCapture) {
+ super(originalRequest, ctx);
+
+ if (har == null) {
+ throw new IllegalStateException("Attempted har capture when har is null");
+ }
+
+ if (ProxyUtils.isCONNECT(originalRequest)) {
+ throw new IllegalStateException("Attempted har capture for HTTP CONNECT request");
+ }
+
+ this.clientAddress = (InetSocketAddress) ctx.channel().remoteAddress();
+
+ if (dataToCapture != null && !dataToCapture.isEmpty()) {
+ this.dataToCapture = EnumSet.copyOf(dataToCapture);
+ } else {
+ this.dataToCapture = EnumSet.noneOf(CaptureType.class);
+ }
+
+ // we may need to capture both the request and the response, so set up the request/response filters and delegate to them when
+ // the corresponding filter methods are invoked. to save time and memory, only set up the capturing filters when
+ // we actually need to capture the data.
+ if (this.dataToCapture.contains(CaptureType.REQUEST_CONTENT) || this.dataToCapture.contains(CaptureType.REQUEST_BINARY_CONTENT)) {
+ requestCaptureFilter = new ClientRequestCaptureFilter(originalRequest);
+ } else {
+ requestCaptureFilter = null;
+ }
+
+ if (this.dataToCapture.contains(CaptureType.RESPONSE_CONTENT) || this.dataToCapture.contains(CaptureType.RESPONSE_BINARY_CONTENT)) {
+ responseCaptureFilter = new ServerResponseCaptureFilter(originalRequest, true);
+ } else {
+ responseCaptureFilter = null;
+ }
+
+ this.har = har;
+
+ this.harEntry = new HarEntry(currentPageRef);
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ // if a ServerResponseCaptureFilter is configured, delegate to it to collect the client request. if it is not
+ // configured, we still need to capture basic information (timings, possibly client headers, etc.), just not content.
+ if (requestCaptureFilter != null) {
+ requestCaptureFilter.clientToProxyRequest(httpObject);
+ }
+
+ if (httpObject instanceof HttpRequest) {
+ // link the object up now, before we make the request, so that if we get cut off (ie: favicon.ico request and browser shuts down)
+ // we still have the attempt associated, even if we never got a response
+ harEntry.setStartedDateTime(new Date());
+ har.getLog().addEntry(harEntry);
+
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+ this.capturedOriginalRequest = httpRequest;
+
+ // associate this request's HarRequest object with the har entry
+ HarRequest request = createHarRequestForHttpRequest(httpRequest);
+ harEntry.setRequest(request);
+
+ // create a "no response received" HarResponse, in case the connection is interrupted, terminated, or the response is not received
+ // for any other reason. having a "default" HarResponse prevents us from generating an invalid HAR.
+ HarResponse defaultHarResponse = HarCaptureUtil.createHarResponseForFailure();
+ defaultHarResponse.setError(HarCaptureUtil.getNoResponseReceivedErrorMessage());
+ harEntry.setResponse(defaultHarResponse);
+
+ captureQueryParameters(httpRequest);
+ // not capturing user agent: in many cases, it doesn't make sense to capture at the HarLog level, since the proxy could be
+ // serving requests from many different clients with various user agents. clients can turn on the REQUEST_HEADERS capture type
+ // in order to capture the User-Agent header, if desired.
+ captureRequestHeaderSize(httpRequest);
+
+ if (dataToCapture.contains(CaptureType.REQUEST_COOKIES)) {
+ captureRequestCookies(httpRequest);
+ }
+
+ if (dataToCapture.contains(CaptureType.REQUEST_HEADERS)) {
+ captureRequestHeaders(httpRequest);
+ }
+
+ // The HTTP CONNECT to the proxy server establishes the SSL connection to the remote server, but the
+ // HTTP CONNECT is not recorded in a separate HarEntry (except in case of error). Instead, the ssl and
+ // connect times are recorded in the first request between the client and remote server after the HTTP CONNECT.
+ captureConnectTiming();
+ }
+
+ if (httpObject instanceof HttpContent) {
+ HttpContent httpContent = (HttpContent) httpObject;
+
+ captureRequestSize(httpContent);
+ }
+
+ if (httpObject instanceof LastHttpContent) {
+ LastHttpContent lastHttpContent = (LastHttpContent) httpObject;
+ if (dataToCapture.contains(CaptureType.REQUEST_HEADERS)) {
+ captureTrailingHeaders(lastHttpContent);
+ }
+
+ if (dataToCapture.contains(CaptureType.REQUEST_CONTENT)) {
+ captureRequestContent(requestCaptureFilter.getHttpRequest(), requestCaptureFilter.getFullRequestContents());
+ }
+
+ harEntry.getRequest().setBodySize(requestBodySize.get());
+ }
+
+ return null;
+ }
+
+ @Override
+ public HttpObject serverToProxyResponse(HttpObject httpObject) {
+ // if a ServerResponseCaptureFilter is configured, delegate to it to collect the server's response. if it is not
+ // configured, we still need to capture basic information (timings, HTTP status, etc.), just not content.
+ if (responseCaptureFilter != null) {
+ responseCaptureFilter.serverToProxyResponse(httpObject);
+ }
+
+ if (httpObject instanceof HttpResponse) {
+ HttpResponse httpResponse = (HttpResponse) httpObject;
+
+ captureResponse(httpResponse);
+ }
+
+ if (httpObject instanceof HttpContent) {
+ HttpContent httpContent = (HttpContent) httpObject;
+
+ captureResponseSize(httpContent);
+ }
+
+ if (httpObject instanceof LastHttpContent) {
+ if (dataToCapture.contains(CaptureType.RESPONSE_CONTENT)) {
+ captureResponseContent(responseCaptureFilter.getHttpResponse(), responseCaptureFilter.getFullResponseContents());
+ }
+
+ harEntry.getResponse().setBodySize(responseBodySize.get());
+ }
+
+ return super.serverToProxyResponse(httpObject);
+ }
+
+ @Override
+ public void serverToProxyResponseTimedOut() {
+ // replace any existing HarResponse that was created if the server sent a partial response
+ HarResponse response = HarCaptureUtil.createHarResponseForFailure();
+ harEntry.setResponse(response);
+
+ response.setError(HarCaptureUtil.getResponseTimedOutErrorMessage());
+
+
+ // include this timeout time in the HarTimings object
+ long timeoutTimestampNanos = System.nanoTime();
+
+ // if the proxy started to send the request but has not yet finished, we are currently "sending"
+ if (sendStartedNanos > 0L && sendFinishedNanos == 0L) {
+ harEntry.getTimings().setSend(timeoutTimestampNanos - sendStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ // if the entire request was sent but the proxy has not begun receiving the response, we are currently "waiting"
+ else if (sendFinishedNanos > 0L && responseReceiveStartedNanos == 0L) {
+ harEntry.getTimings().setWait(timeoutTimestampNanos - sendFinishedNanos, TimeUnit.NANOSECONDS);
+ }
+ // if the proxy has already begun to receive the response, we are currenting "receiving"
+ else if (responseReceiveStartedNanos > 0L) {
+ harEntry.getTimings().setReceive(timeoutTimestampNanos - responseReceiveStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ /**
+ * Creates a HarRequest object using the method, url, and HTTP version of the specified request.
+ *
+ * @param httpRequest HTTP request on which the HarRequest will be based
+ * @return a new HarRequest object
+ */
+ private HarRequest createHarRequestForHttpRequest(HttpRequest httpRequest) {
+ // the HAR spec defines the request.url field as:
+ // url [string] - Absolute URL of the request (fragments are not included).
+ // the URI on the httpRequest may only identify the path of the resource, so find the full URL.
+ // the full URL consists of the scheme + host + port (if non-standard) + path + query params + fragment.
+ String url = getFullUrl(httpRequest);
+
+ return new HarRequest(httpRequest.getMethod().toString(), url, httpRequest.getProtocolVersion().text());
+ }
+
+ //TODO: add unit tests for these utility-like capture() methods
+
+ protected void captureQueryParameters(HttpRequest httpRequest) {
+ // capture query parameters. it is safe to assume the query string is UTF-8, since it "should" be in US-ASCII (a subset of UTF-8),
+ // but sometimes does include UTF-8 characters.
+ QueryStringDecoder queryStringDecoder = new QueryStringDecoder(httpRequest.getUri(), StandardCharsets.UTF_8);
+
+ try {
+ for (Map.Entry> entry : queryStringDecoder.parameters().entrySet()) {
+ for (String value : entry.getValue()) {
+ harEntry.getRequest().getQueryString().add(new HarNameValuePair(entry.getKey(), value));
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ // QueryStringDecoder will throw an IllegalArgumentException if it cannot interpret a query string. rather than cause the entire request to
+ // fail by propagating the exception, simply skip the query parameter capture.
+ harEntry.setComment("Unable to decode query parameters on URI: " + httpRequest.getUri());
+ log.info("Unable to decode query parameters on URI: " + httpRequest.getUri(), e);
+ }
+ }
+
+ protected void captureRequestHeaderSize(HttpRequest httpRequest) {
+ String requestLine = httpRequest.getMethod().toString() + ' ' + httpRequest.getUri() + ' ' + httpRequest.getProtocolVersion().toString();
+ // +2 => CRLF after status line, +4 => header/data separation
+ long requestHeadersSize = requestLine.length() + 6;
+
+ HttpHeaders headers = httpRequest.headers();
+ requestHeadersSize += BrowserMobHttpUtil.getHeaderSize(headers);
+
+ harEntry.getRequest().setHeadersSize(requestHeadersSize);
+ }
+
+ protected void captureRequestCookies(HttpRequest httpRequest) {
+ String cookieHeader = httpRequest.headers().get(HttpHeaders.Names.COOKIE);
+ if (cookieHeader == null) {
+ return;
+ }
+
+ Set cookies = ServerCookieDecoder.LAX.decode(cookieHeader);
+
+ for (Cookie cookie : cookies) {
+ HarCookie harCookie = new HarCookie();
+
+ harCookie.setName(cookie.name());
+ harCookie.setValue(cookie.value());
+
+ harEntry.getRequest().getCookies().add(harCookie);
+ }
+ }
+
+ protected void captureRequestHeaders(HttpRequest httpRequest) {
+ HttpHeaders headers = httpRequest.headers();
+
+ captureHeaders(headers);
+ }
+
+ protected void captureTrailingHeaders(LastHttpContent lastHttpContent) {
+ HttpHeaders headers = lastHttpContent.trailingHeaders();
+
+ captureHeaders(headers);
+ }
+
+ protected void captureHeaders(HttpHeaders headers) {
+ for (Map.Entry header : headers.entries()) {
+ harEntry.getRequest().getHeaders().add(new HarNameValuePair(header.getKey(), header.getValue()));
+ }
+ }
+
+ protected void captureRequestContent(HttpRequest httpRequest, byte[] fullMessage) {
+ if (fullMessage.length == 0) {
+ return;
+ }
+
+ String contentType = HttpHeaders.getHeader(httpRequest, HttpHeaders.Names.CONTENT_TYPE);
+ if (contentType == null) {
+ log.warn("No content type specified in request to {}. Content will be treated as {}", httpRequest.getUri(), BrowserMobHttpUtil.UNKNOWN_CONTENT_TYPE);
+ contentType = BrowserMobHttpUtil.UNKNOWN_CONTENT_TYPE;
+ }
+
+ HarPostData postData = new HarPostData();
+ harEntry.getRequest().setPostData(postData);
+
+ postData.setMimeType(contentType);
+
+ boolean urlEncoded;
+ if (contentType.startsWith(HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED)) {
+ urlEncoded = true;
+ } else {
+ urlEncoded = false;
+ }
+
+ Charset charset;
+ try {
+ charset = BrowserMobHttpUtil.readCharsetInContentTypeHeader(contentType);
+ } catch (UnsupportedCharsetException e) {
+ log.warn("Found unsupported character set in Content-Type header '{}' in HTTP request to {}. Content will not be captured in HAR.", contentType, httpRequest.getUri(), e);
+ return;
+ }
+
+ if (charset == null) {
+ // no charset specified, so use the default -- but log a message since this might not encode the data correctly
+ charset = BrowserMobHttpUtil.DEFAULT_HTTP_CHARSET;
+ log.debug("No charset specified; using charset {} to decode contents to {}", charset, httpRequest.getUri());
+ }
+
+ if (urlEncoded) {
+ String textContents = BrowserMobHttpUtil.getContentAsString(fullMessage, charset);
+
+ QueryStringDecoder queryStringDecoder = new QueryStringDecoder(textContents, charset, false);
+
+ ImmutableList.Builder paramBuilder = ImmutableList.builder();
+
+ for (Map.Entry> entry : queryStringDecoder.parameters().entrySet()) {
+ for (String value : entry.getValue()) {
+ paramBuilder.add(new HarPostDataParam(entry.getKey(), value));
+ }
+ }
+
+ harEntry.getRequest().getPostData().setParams(paramBuilder.build());
+ } else {
+ //TODO: implement capture of files and multipart form data
+
+ // not URL encoded, so let's grab the body of the POST and capture that
+ String postBody = BrowserMobHttpUtil.getContentAsString(fullMessage, charset);
+ harEntry.getRequest().getPostData().setText(postBody);
+ }
+ }
+
+ protected void captureResponseContent(HttpResponse httpResponse, byte[] fullMessage) {
+ // force binary if the content encoding is not supported
+ boolean forceBinary = false;
+
+ String contentType = HttpHeaders.getHeader(httpResponse, HttpHeaders.Names.CONTENT_TYPE);
+ if (contentType == null) {
+ log.warn("No content type specified in response from {}. Content will be treated as {}", originalRequest.getUri(), BrowserMobHttpUtil.UNKNOWN_CONTENT_TYPE);
+ contentType = BrowserMobHttpUtil.UNKNOWN_CONTENT_TYPE;
+ }
+
+ if (responseCaptureFilter.isResponseCompressed() && !responseCaptureFilter.isDecompressionSuccessful()) {
+ log.warn("Unable to decompress content with encoding: {}. Contents will be encoded as base64 binary data.", responseCaptureFilter.getContentEncoding());
+
+ forceBinary = true;
+ }
+
+ Charset charset;
+ try {
+ charset = BrowserMobHttpUtil.readCharsetInContentTypeHeader(contentType);
+ } catch (UnsupportedCharsetException e) {
+ log.warn("Found unsupported character set in Content-Type header '{}' in HTTP response from {}. Content will not be captured in HAR.", contentType, originalRequest.getUri(), e);
+ return;
+ }
+
+ if (charset == null) {
+ // no charset specified, so use the default -- but log a message since this might not encode the data correctly
+ charset = BrowserMobHttpUtil.DEFAULT_HTTP_CHARSET;
+ log.debug("No charset specified; using charset {} to decode contents from {}", charset, originalRequest.getUri());
+ }
+
+ if (!forceBinary && BrowserMobHttpUtil.hasTextualContent(contentType)) {
+ String text = BrowserMobHttpUtil.getContentAsString(fullMessage, charset);
+ harEntry.getResponse().getContent().setText(text);
+ } else if (dataToCapture.contains(CaptureType.RESPONSE_BINARY_CONTENT)) {
+ harEntry.getResponse().getContent().setText(BaseEncoding.base64().encode(fullMessage));
+ harEntry.getResponse().getContent().setEncoding("base64");
+ }
+
+ harEntry.getResponse().getContent().setSize(fullMessage.length);
+ }
+
+ protected void captureResponse(HttpResponse httpResponse) {
+ HarResponse response = new HarResponse(httpResponse.getStatus().code(), httpResponse.getStatus().reasonPhrase(), httpResponse.getProtocolVersion().text());
+ harEntry.setResponse(response);
+
+ captureResponseHeaderSize(httpResponse);
+
+ captureResponseMimeType(httpResponse);
+
+ if (dataToCapture.contains(CaptureType.RESPONSE_COOKIES)) {
+ captureResponseCookies(httpResponse);
+ }
+
+ if (dataToCapture.contains(CaptureType.RESPONSE_HEADERS)) {
+ captureResponseHeaders(httpResponse);
+ }
+
+ if (BrowserMobHttpUtil.isRedirect(httpResponse)) {
+ captureRedirectUrl(httpResponse);
+ }
+ }
+
+ protected void captureResponseMimeType(HttpResponse httpResponse) {
+ String contentType = HttpHeaders.getHeader(httpResponse, HttpHeaders.Names.CONTENT_TYPE);
+ // don't set the mimeType to null, since mimeType is a required field
+ if (contentType != null) {
+ harEntry.getResponse().getContent().setMimeType(contentType);
+ }
+ }
+
+ protected void captureResponseCookies(HttpResponse httpResponse) {
+ List setCookieHeaders = httpResponse.headers().getAll(HttpHeaders.Names.SET_COOKIE);
+ if (setCookieHeaders == null) {
+ return;
+ }
+
+ for (String setCookieHeader : setCookieHeaders) {
+ Cookie cookie = ClientCookieDecoder.LAX.decode(setCookieHeader);
+ if (cookie == null) {
+ return;
+ }
+
+ HarCookie harCookie = new HarCookie();
+
+ harCookie.setName(cookie.name());
+ harCookie.setValue(cookie.value());
+ // comment is no longer supported in the netty ClientCookieDecoder
+ harCookie.setDomain(cookie.domain());
+ harCookie.setHttpOnly(cookie.isHttpOnly());
+ harCookie.setPath(cookie.path());
+ harCookie.setSecure(cookie.isSecure());
+ if (cookie.maxAge() > 0) {
+ // use a Calendar with the current timestamp + maxAge seconds. the locale of the calendar is irrelevant,
+ // since we are dealing with timestamps.
+ Calendar expires = Calendar.getInstance();
+ // zero out the milliseconds, since maxAge is in seconds
+ expires.set(Calendar.MILLISECOND, 0);
+ // we can't use Calendar.add, since that only takes ints. TimeUnit.convert handles second->millisecond
+ // overflow reasonably well by returning the result as Long.MAX_VALUE.
+ expires.setTimeInMillis(expires.getTimeInMillis() + TimeUnit.MILLISECONDS.convert(cookie.maxAge(), TimeUnit.SECONDS));
+
+ harCookie.setExpires(expires.getTime());
+ }
+
+ harEntry.getResponse().getCookies().add(harCookie);
+ }
+ }
+
+ protected void captureResponseHeaderSize(HttpResponse httpResponse) {
+ String statusLine = httpResponse.getProtocolVersion().toString() + ' ' + httpResponse.getStatus().toString();
+ // +2 => CRLF after status line, +4 => header/data separation
+ long responseHeadersSize = statusLine.length() + 6;
+ HttpHeaders headers = httpResponse.headers();
+ responseHeadersSize += BrowserMobHttpUtil.getHeaderSize(headers);
+
+ harEntry.getResponse().setHeadersSize(responseHeadersSize);
+ }
+
+ protected void captureResponseHeaders(HttpResponse httpResponse) {
+ HttpHeaders headers = httpResponse.headers();
+ for (Map.Entry header : headers.entries()) {
+ harEntry.getResponse().getHeaders().add(new HarNameValuePair(header.getKey(), header.getValue()));
+ }
+ }
+
+ protected void captureRedirectUrl(HttpResponse httpResponse) {
+ String locationHeaderValue = HttpHeaders.getHeader(httpResponse, HttpHeaders.Names.LOCATION);
+ if (locationHeaderValue != null) {
+ harEntry.getResponse().setRedirectURL(locationHeaderValue);
+ }
+ }
+
+ /**
+ * Adds the size of this httpContent to the requestBodySize.
+ *
+ * @param httpContent HttpContent to size
+ */
+ protected void captureRequestSize(HttpContent httpContent) {
+ ByteBuf bufferedContent = httpContent.content();
+ int contentSize = bufferedContent.readableBytes();
+ requestBodySize.addAndGet(contentSize);
+ }
+
+ /**
+ * Adds the size of this httpContent to the responseBodySize.
+ *
+ * @param httpContent HttpContent to size
+ */
+ protected void captureResponseSize(HttpContent httpContent) {
+ ByteBuf bufferedContent = httpContent.content();
+ int contentSize = bufferedContent.readableBytes();
+ responseBodySize.addAndGet(contentSize);
+ }
+
+ /**
+ * Populates ssl and connect timing info in the HAR if an entry for this client and server exist in the cache.
+ */
+ protected void captureConnectTiming() {
+ HttpConnectTiming httpConnectTiming = HttpConnectHarCaptureFilter.consumeConnectTimingForConnection(clientAddress);
+ if (httpConnectTiming != null) {
+ harEntry.getTimings().setSsl(httpConnectTiming.getSslHandshakeTimeNanos(), TimeUnit.NANOSECONDS);
+ harEntry.getTimings().setConnect(httpConnectTiming.getConnectTimeNanos(), TimeUnit.NANOSECONDS);
+ harEntry.getTimings().setBlocked(httpConnectTiming.getBlockedTimeNanos(), TimeUnit.NANOSECONDS);
+ harEntry.getTimings().setDns(httpConnectTiming.getDnsTimeNanos(), TimeUnit.NANOSECONDS);
+ }
+ }
+
+ /**
+ * Populates the serverIpAddress field of the harEntry using the internal hostname->IP address cache.
+ *
+ * @param httpRequest HTTP request to take the hostname from
+ */
+ protected void populateAddressFromCache(HttpRequest httpRequest) {
+ String serverHost = getHost(httpRequest);
+
+ if (serverHost != null && !serverHost.isEmpty()) {
+ String resolvedAddress = ResolvedHostnameCacheFilter.getPreviouslyResolvedAddressForHost(serverHost);
+ if (resolvedAddress != null) {
+ harEntry.setServerIPAddress(resolvedAddress);
+ } else {
+ // the resolvedAddress may be null if the ResolvedHostnameCacheFilter has expired the entry (which is unlikely),
+ // or in the far more common case that the proxy is using a chained proxy to connect to connect to the
+ // remote host. since the chained proxy handles IP address resolution, the IP address in the HAR must be blank.
+ log.trace("Unable to find cached IP address for host: {}. IP address in HAR entry will be blank.", serverHost);
+ }
+ } else {
+ log.warn("Unable to identify host from request uri: {}", httpRequest.getUri());
+ }
+ }
+
+ @Override
+ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHostAndPort) {
+ dnsResolutionStartedNanos = System.nanoTime();
+
+ // resolution started means the connection is no longer queued, so populate 'blocked' time
+ if (connectionQueuedNanos > 0L) {
+ harEntry.getTimings().setBlocked(dnsResolutionStartedNanos - connectionQueuedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setBlocked(0L, TimeUnit.NANOSECONDS);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void proxyToServerResolutionFailed(String hostAndPort) {
+ HarResponse response = HarCaptureUtil.createHarResponseForFailure();
+ harEntry.setResponse(response);
+
+ response.setError(HarCaptureUtil.getResolutionFailedErrorMessage(hostAndPort));
+
+ // record the amount of time we attempted to resolve the hostname in the HarTimings object
+ if (dnsResolutionStartedNanos > 0L) {
+ harEntry.getTimings().setDns(System.nanoTime() - dnsResolutionStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) {
+ long dnsResolutionFinishedNanos = System.nanoTime();
+
+ if (dnsResolutionStartedNanos > 0L) {
+ harEntry.getTimings().setDns(dnsResolutionFinishedNanos - dnsResolutionStartedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setDns(0L, TimeUnit.NANOSECONDS);
+ }
+
+ // the address *should* always be resolved at this point
+ InetAddress resolvedAddress = resolvedRemoteAddress.getAddress();
+ if (resolvedAddress != null) {
+ addressResolved = true;
+
+ harEntry.setServerIPAddress(resolvedAddress.getHostAddress());
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionQueued() {
+ this.connectionQueuedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void proxyToServerConnectionStarted() {
+ this.connectionStartedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void proxyToServerConnectionFailed() {
+ HarResponse response = HarCaptureUtil.createHarResponseForFailure();
+ harEntry.setResponse(response);
+
+ response.setError(HarCaptureUtil.getConnectionFailedErrorMessage());
+
+ // record the amount of time we attempted to connect in the HarTimings object
+ if (connectionStartedNanos > 0L) {
+ harEntry.getTimings().setConnect(System.nanoTime() - connectionStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) {
+ long connectionSucceededTimeNanos = System.nanoTime();
+
+ // make sure the previous timestamp was captured, to avoid setting an absurd value in the har (see serverToProxyResponseReceiving())
+ if (connectionStartedNanos > 0L) {
+ harEntry.getTimings().setConnect(connectionSucceededTimeNanos - connectionStartedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setConnect(0L, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void proxyToServerRequestSending() {
+ this.sendStartedNanos = System.nanoTime();
+
+ // if the hostname was not resolved (and thus the IP address populated in the har) during this request, populate the IP address from the cache
+ if (!addressResolved) {
+ populateAddressFromCache(capturedOriginalRequest);
+ }
+ }
+
+ @Override
+ public void proxyToServerRequestSent() {
+ this.sendFinishedNanos = System.nanoTime();
+
+ // make sure the previous timestamp was captured, to avoid setting an absurd value in the har (see serverToProxyResponseReceiving())
+ if (sendStartedNanos > 0L) {
+ harEntry.getTimings().setSend(sendFinishedNanos - sendStartedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setSend(0L, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void serverToProxyResponseReceiving() {
+ this.responseReceiveStartedNanos = System.nanoTime();
+
+ // started to receive response, so populate the 'wait' time. if we started receiving a response from the server before we finished
+ // sending (for example, the server replied with a 404 while we were uploading a large file), there was no wait time, so
+ // make sure the wait is set to 0.
+ if (sendFinishedNanos > 0L && sendFinishedNanos < responseReceiveStartedNanos) {
+ harEntry.getTimings().setWait(responseReceiveStartedNanos - sendFinishedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setWait(0L, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void serverToProxyResponseReceived() {
+ long responseReceivedNanos = System.nanoTime();
+
+ // like the wait time, the receive time requires that the serverToProxyResponseReceiving() method be called before this method is invoked.
+ // typically that should happen, but it has been reported (https://github.com/lightbody/browsermob-proxy/issues/288) that it
+ // sometimes does not. therefore, to be safe, make sure responseReceiveStartedNanos is populated before setting the receive time.
+ if (responseReceiveStartedNanos > 0L) {
+ harEntry.getTimings().setReceive(responseReceivedNanos - responseReceiveStartedNanos, TimeUnit.NANOSECONDS);
+ } else {
+ harEntry.getTimings().setReceive(0L, TimeUnit.NANOSECONDS);
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpConnectHarCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpConnectHarCaptureFilter.java
new file mode 100644
index 000000000..1717370a2
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpConnectHarCaptureFilter.java
@@ -0,0 +1,393 @@
+package net.lightbody.bmp.filters;
+
+import com.google.common.cache.CacheBuilder;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.core.har.Har;
+import net.lightbody.bmp.core.har.HarEntry;
+import net.lightbody.bmp.core.har.HarRequest;
+import net.lightbody.bmp.core.har.HarResponse;
+import net.lightbody.bmp.core.har.HarTimings;
+import net.lightbody.bmp.filters.support.HttpConnectTiming;
+import net.lightbody.bmp.filters.util.HarCaptureUtil;
+import net.lightbody.bmp.util.HttpUtil;
+import org.littleshoot.proxy.impl.ProxyUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Date;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This filter captures HAR data for HTTP CONNECT requests. CONNECTs are "meta" requests that must be made before HTTPS
+ * requests, but are not populated as separate requests in the HAR. Most information from HTTP CONNECTs (such as SSL
+ * handshake time, dns resolution time, etc.) is populated in the HAR entry for the first "true" request following the
+ * CONNECT. This filter captures the timing-related information and makes it available to subsequent filters through
+ * static methods. This filter also handles HTTP CONNECT errors and creates HAR entries for those errors, since there
+ * would otherwise not be any record in the HAR of the error (if the CONNECT fails, there will be no subsequent "real"
+ * request in which to record the error).
+ *
+ */
+public class HttpConnectHarCaptureFilter extends HttpsAwareFiltersAdapter implements ModifiedRequestAwareFilter {
+ private static final Logger log = LoggerFactory.getLogger(HttpConnectHarCaptureFilter.class);
+
+ /**
+ * The currently active HAR at the time the current request is received.
+ */
+ private final Har har;
+
+ /**
+ * The currently active page ref at the time the current request is received.
+ */
+ private final String currentPageRef;
+
+ /**
+ * The time this CONNECT began. Used to populate the HAR entry in case of failure.
+ */
+ private volatile Date requestStartTime;
+
+ /**
+ * True if this filter instance processed a {@link #proxyToServerResolutionSucceeded(String, java.net.InetSocketAddress)} call, indicating
+ * that the hostname was resolved and populated in the HAR (if this is not a CONNECT).
+ */
+// private volatile boolean addressResolved = false;
+ private volatile InetAddress resolvedAddress;
+
+ /**
+ * Populated by proxyToServerResolutionStarted when DNS resolution starts. If any previous filters already resolved the address, their resolution time
+ * will not be included in this time. See {@link HarCaptureFilter#dnsResolutionStartedNanos}.
+ */
+ private volatile long dnsResolutionStartedNanos;
+
+ private volatile long dnsResolutionFinishedNanos;
+
+ private volatile long connectionQueuedNanos;
+ private volatile long connectionStartedNanos;
+ private volatile long connectionSucceededTimeNanos;
+ private volatile long sendStartedNanos;
+ private volatile long sendFinishedNanos;
+
+ private volatile long responseReceiveStartedNanos;
+ private volatile long sslHandshakeStartedNanos;
+
+ /**
+ * The address of the client making the request. Captured in the constructor and used when calculating and capturing ssl handshake and connect
+ * timing information for SSL connections.
+ */
+ private final InetSocketAddress clientAddress;
+
+ /**
+ * Stores HTTP CONNECT timing information for this request, if it is an HTTP CONNECT.
+ */
+ private final HttpConnectTiming httpConnectTiming;
+
+ /**
+ * The maximum amount of time to save timing information between an HTTP CONNECT and the subsequent HTTP request. Typically this is done
+ * immediately, but if for some reason it is not (e.g. due to a client crash or dropped connection), the timing information will be
+ * kept for this long before being evicted to prevent a memory leak. If a subsequent request does come through after eviction, it will still
+ * be recorded, but the timing information will not be populated in the HAR.
+ */
+ private static final int HTTP_CONNECT_TIMING_EVICTION_SECONDS = 60;
+
+ /**
+ * Concurrency of the httpConnectTiming map. Should be approximately equal to the maximum number of simultaneous connection
+ * attempts (but not necessarily simultaneous connections). A lower value will inhibit performance.
+ * TODO: tune this value for a large number of concurrent requests. develop a non-cache-based mechanism of passing ssl timings to subsequent requests.
+ */
+ private static final int HTTP_CONNECT_TIMING_CONCURRENCY_LEVEL = 50;
+
+ /**
+ * Stores SSL connection timing information from HTTP CONNNECT requests. This timing information is stored in the first HTTP request
+ * after the CONNECT, not in the CONNECT itself, so it needs to be stored across requests.
+ *
+ * This is the only state stored across multiple requests.
+ */
+ private static final ConcurrentMap httpConnectTimes =
+ CacheBuilder.newBuilder()
+ .expireAfterWrite(HTTP_CONNECT_TIMING_EVICTION_SECONDS, TimeUnit.SECONDS)
+ .concurrencyLevel(HTTP_CONNECT_TIMING_CONCURRENCY_LEVEL)
+ .build()
+ .asMap();
+
+ private volatile HttpRequest modifiedHttpRequest;
+
+ public HttpConnectHarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Har har, String currentPageRef) {
+ super(originalRequest, ctx);
+
+ if (har == null) {
+ throw new IllegalStateException("Attempted har capture when har is null");
+ }
+
+ if (!ProxyUtils.isCONNECT(originalRequest)) {
+ throw new IllegalStateException("Attempted HTTP CONNECT har capture on non-HTTP CONNECT request");
+ }
+
+ this.har = har;
+ this.currentPageRef = currentPageRef;
+
+ this.clientAddress = (InetSocketAddress) ctx.channel().remoteAddress();
+
+ // create and cache an HTTP CONNECT timing object to capture timing-related information
+ this.httpConnectTiming = new HttpConnectTiming();
+ httpConnectTimes.put(clientAddress, httpConnectTiming);
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ // store the CONNECT start time in case of failure, so we can populate the HarEntry with it
+ requestStartTime = new Date();
+ }
+
+ return null;
+ }
+
+ @Override
+ public void proxyToServerResolutionFailed(String hostAndPort) {
+ // since this is a CONNECT, which is not handled by the HarCaptureFilter, we need to create and populate the
+ // entire HarEntry and add it to this har.
+ HarEntry harEntry = createHarEntryForFailedCONNECT(HarCaptureUtil.getResolutionFailedErrorMessage(hostAndPort));
+ har.getLog().addEntry(harEntry);
+
+ // record the amount of time we attempted to resolve the hostname in the HarTimings object
+ if (dnsResolutionStartedNanos > 0L) {
+ harEntry.getTimings().setDns(System.nanoTime() - dnsResolutionStartedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ httpConnectTimes.remove(clientAddress);
+ }
+
+ @Override
+ public void proxyToServerConnectionFailed() {
+ // since this is a CONNECT, which is not handled by the HarCaptureFilter, we need to create and populate the
+ // entire HarEntry and add it to this har.
+ HarEntry harEntry = createHarEntryForFailedCONNECT(HarCaptureUtil.getConnectionFailedErrorMessage());
+ har.getLog().addEntry(harEntry);
+
+ // record the amount of time we attempted to connect in the HarTimings object
+ if (connectionStartedNanos > 0L) {
+ harEntry.getTimings().setConnect(System.nanoTime() - connectionStartedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ httpConnectTimes.remove(clientAddress);
+ }
+
+ @Override
+ public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) {
+ this.connectionSucceededTimeNanos = System.nanoTime();
+
+ if (connectionStartedNanos > 0L) {
+ httpConnectTiming.setConnectTimeNanos(connectionSucceededTimeNanos - connectionStartedNanos);
+ } else {
+ httpConnectTiming.setConnectTimeNanos(0L);
+ }
+
+ if (sslHandshakeStartedNanos > 0L) {
+ httpConnectTiming.setSslHandshakeTimeNanos(connectionSucceededTimeNanos - sslHandshakeStartedNanos);
+ } else {
+ httpConnectTiming.setSslHandshakeTimeNanos(0L);
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionSSLHandshakeStarted() {
+ this.sslHandshakeStartedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void serverToProxyResponseTimedOut() {
+ HarEntry harEntry = createHarEntryForFailedCONNECT(HarCaptureUtil.getResponseTimedOutErrorMessage());
+ har.getLog().addEntry(harEntry);
+
+ // include this timeout time in the HarTimings object
+ long timeoutTimestampNanos = System.nanoTime();
+
+ // if the proxy started to send the request but has not yet finished, we are currently "sending"
+ if (sendStartedNanos > 0L && sendFinishedNanos == 0L) {
+ harEntry.getTimings().setSend(timeoutTimestampNanos - sendStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ // if the entire request was sent but the proxy has not begun receiving the response, we are currently "waiting"
+ else if (sendFinishedNanos > 0L && responseReceiveStartedNanos == 0L) {
+ harEntry.getTimings().setWait(timeoutTimestampNanos - sendFinishedNanos, TimeUnit.NANOSECONDS);
+ }
+ // if the proxy has already begun to receive the response, we are currenting "receiving"
+ else if (responseReceiveStartedNanos > 0L) {
+ harEntry.getTimings().setReceive(timeoutTimestampNanos - responseReceiveStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ @Override
+ public void proxyToServerConnectionQueued() {
+ this.connectionQueuedNanos = System.nanoTime();
+ }
+
+
+ @Override
+ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHostAndPort) {
+ dnsResolutionStartedNanos = System.nanoTime();
+
+ if (connectionQueuedNanos > 0L) {
+ httpConnectTiming.setBlockedTimeNanos(dnsResolutionStartedNanos - connectionQueuedNanos);
+ } else {
+ httpConnectTiming.setBlockedTimeNanos(0L);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) {
+ this.dnsResolutionFinishedNanos = System.nanoTime();
+
+ if (dnsResolutionStartedNanos > 0L) {
+ httpConnectTiming.setDnsTimeNanos(dnsResolutionFinishedNanos - dnsResolutionStartedNanos);
+ } else {
+ httpConnectTiming.setDnsTimeNanos(0L);
+ }
+
+ // the address *should* always be resolved at this point
+ this.resolvedAddress = resolvedRemoteAddress.getAddress();
+ }
+
+ @Override
+ public void proxyToServerConnectionStarted() {
+ this.connectionStartedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void proxyToServerRequestSending() {
+ this.sendStartedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void proxyToServerRequestSent() {
+ this.sendFinishedNanos = System.nanoTime();
+ }
+
+ @Override
+ public void serverToProxyResponseReceiving() {
+ this.responseReceiveStartedNanos = System.nanoTime();
+ }
+
+ /**
+ * Populates timing information in the specified harEntry for failed rquests. Populates as much timing information
+ * as possible, up to the point of failure.
+ *
+ * @param harEntry HAR entry to populate timing information in
+ */
+ private void populateTimingsForFailedCONNECT(HarEntry harEntry) {
+ HarTimings timings = harEntry.getTimings();
+
+ if (connectionQueuedNanos > 0L && dnsResolutionStartedNanos > 0L) {
+ timings.setBlocked(dnsResolutionStartedNanos - connectionQueuedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ if (dnsResolutionStartedNanos > 0L && dnsResolutionFinishedNanos > 0L) {
+ timings.setDns(dnsResolutionFinishedNanos - dnsResolutionStartedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ if (connectionStartedNanos > 0L && connectionSucceededTimeNanos > 0L) {
+ timings.setConnect(connectionSucceededTimeNanos - connectionStartedNanos, TimeUnit.NANOSECONDS);
+
+ if (sslHandshakeStartedNanos > 0L) {
+ timings.setSsl(connectionSucceededTimeNanos - this.sslHandshakeStartedNanos, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ if (sendStartedNanos > 0L && sendFinishedNanos >= 0L) {
+ timings.setSend(sendFinishedNanos - sendStartedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ if (sendFinishedNanos > 0L && responseReceiveStartedNanos >= 0L) {
+ timings.setWait(responseReceiveStartedNanos - sendFinishedNanos, TimeUnit.NANOSECONDS);
+ }
+
+ // since this method is for HTTP CONNECT failures only, we can't populate a "received" time, since that would
+ // require the CONNECT to be successful, in which case this method wouldn't be called.
+ }
+
+ /**
+ * Creates a {@link HarEntry} for a failed CONNECT request. Initializes and populates the entry, including the
+ * {@link HarRequest}, {@link HarResponse}, and {@link HarTimings}. (Note: only successful timing information is
+ * populated in the timings object; the calling method must populate the timing information for the final, failed
+ * step. For example, if DNS resolution failed, this method will populate the network 'blocked' time, but not the DNS
+ * time.) Populates the specified errorMessage in the {@link HarResponse}'s error field.
+ *
+ * @param errorMessage error message to place in the har response
+ * @return a new HAR entry
+ */
+ private HarEntry createHarEntryForFailedCONNECT(String errorMessage) {
+ HarEntry harEntry = new HarEntry(currentPageRef);
+ harEntry.setStartedDateTime(requestStartTime);
+
+ HarRequest request = createRequestForFailedConnect(originalRequest);
+ harEntry.setRequest(request);
+
+ HarResponse response = HarCaptureUtil.createHarResponseForFailure();
+ harEntry.setResponse(response);
+
+ response.setError(errorMessage);
+
+ populateTimingsForFailedCONNECT(harEntry);
+
+ populateServerIpAddress(harEntry);
+
+
+ return harEntry;
+ }
+
+ private void populateServerIpAddress(HarEntry harEntry) {
+ // populate the server IP address if it was resolved as part of this request. otherwise, populate the IP address from the cache.
+ if (resolvedAddress != null) {
+ harEntry.setServerIPAddress(resolvedAddress.getHostAddress());
+ } else {
+ String serverHost = HttpUtil.getHostFromRequest(modifiedHttpRequest);
+ if (serverHost != null && !serverHost.isEmpty()) {
+ String resolvedAddress = ResolvedHostnameCacheFilter.getPreviouslyResolvedAddressForHost(serverHost);
+ if (resolvedAddress != null) {
+ harEntry.setServerIPAddress(resolvedAddress);
+ } else {
+ // the resolvedAddress may be null if the ResolvedHostnameCacheFilter has expired the entry (which is unlikely),
+ // or in the far more common case that the proxy is using a chained proxy to connect to connect to the
+ // remote host. since the chained proxy handles IP address resolution, the IP address in the HAR must be blank.
+ log.trace("Unable to find cached IP address for host: {}. IP address in HAR entry will be blank.", serverHost);
+ }
+ } else {
+ log.warn("Unable to identify host from request uri: {}", modifiedHttpRequest.getUri());
+ }
+ }
+ }
+
+ /**
+ * Creates a new {@link HarRequest} object for this failed HTTP CONNECT. Does not populate fields within the request,
+ * such as the error message.
+ *
+ * @param httpConnectRequest the HTTP CONNECT request that failed
+ * @return a new HAR request object
+ */
+ private HarRequest createRequestForFailedConnect(HttpRequest httpConnectRequest) {
+ String url = getFullUrl(httpConnectRequest);
+
+ return new HarRequest(httpConnectRequest.getMethod().toString(), url, httpConnectRequest.getProtocolVersion().text());
+ }
+
+ /**
+ * Retrieves and removes (thus "consumes") the SSL timing information from the connection cache for the specified address.
+ *
+ * @param clientAddress the address of the client connection that established the HTTP tunnel
+ * @return the timing information for the tunnel previously established from the clientAddress
+ */
+ public static HttpConnectTiming consumeConnectTimingForConnection(InetSocketAddress clientAddress) {
+ return httpConnectTimes.remove(clientAddress);
+ }
+
+ @Override
+ public void setModifiedHttpRequest(HttpRequest modifiedHttpRequest) {
+ this.modifiedHttpRequest = modifiedHttpRequest;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java
new file mode 100644
index 000000000..e727f2b00
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java
@@ -0,0 +1,162 @@
+package net.lightbody.bmp.filters;
+
+import com.google.common.net.HostAndPort;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.util.Attribute;
+import io.netty.util.AttributeKey;
+import net.lightbody.bmp.util.HttpUtil;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.littleshoot.proxy.impl.ProxyUtils;
+
+/**
+ * The HttpsAwareFiltersAdapter exposes the original host and the "real" host (after filter modifications) to filters for HTTPS
+ * requets. HTTPS requests do not normally contain the host in the URI, and the Host header may be missing or spoofed.
+ *
+ * Note: The {@link #getHttpsRequestHostAndPort()} and {@link #getHttpsOriginalRequestHostAndPort()} methods can only be
+ * called when the request is an HTTPS request. Otherwise they will throw an IllegalStateException.
+ */
+public class HttpsAwareFiltersAdapter extends HttpFiltersAdapter {
+ public static final String IS_HTTPS_ATTRIBUTE_NAME = "isHttps";
+ public static final String HOST_ATTRIBUTE_NAME = "host";
+ public static final String ORIGINAL_HOST_ATTRIBUTE_NAME = "originalHost";
+
+ public HttpsAwareFiltersAdapter(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+ }
+
+ /**
+ * Returns true if this is an HTTPS request.
+ *
+ * @return true if https, false if http
+ */
+ public boolean isHttps() {
+ Attribute isHttpsAttr = ctx.attr(AttributeKey.valueOf(IS_HTTPS_ATTRIBUTE_NAME));
+
+ Boolean isHttps = isHttpsAttr.get();
+ if (isHttps == null) {
+ return false;
+ } else {
+ return isHttps;
+ }
+ }
+
+ /**
+ * Returns the full, absolute URL of the specified request for both HTTP and HTTPS URLs. The request may reflect
+ * modifications from this or other filters. This filter instance must be currently handling the specified request;
+ * otherwise the results are undefined.
+ *
+ * @param modifiedRequest a possibly-modified version of the request currently being processed
+ * @return the full URL of the request, including scheme, host, port, path, and query parameters
+ */
+ public String getFullUrl(HttpRequest modifiedRequest) {
+ // special case: for HTTPS requests, the full URL is scheme (https://) + the URI of this request
+ if (ProxyUtils.isCONNECT(modifiedRequest)) {
+ // CONNECT requests contain the default port, even if it isn't specified on the request.
+ String hostNoDefaultPort = BrowserMobHttpUtil.removeMatchingPort(modifiedRequest.getUri(), 443);
+ return "https://" + hostNoDefaultPort;
+ }
+
+ // To get the full URL, we need to retrieve the Scheme, Host + Port, Path, and Query Params from the request.
+ // If the request URI starts with http:// or https://, it is already a full URL and can be returned directly.
+ if (HttpUtil.startsWithHttpOrHttps(modifiedRequest.getUri())) {
+ return modifiedRequest.getUri();
+ }
+
+ // The URI did not include the scheme and host, so examine the request to obtain them:
+ // Scheme: the scheme (HTTP/HTTPS) are based on the type of connection, obtained from isHttps()
+ // Host and Port: available for HTTP and HTTPS requests using the getHostAndPort() helper method.
+ // Path + Query Params: since the request URI doesn't start with the scheme, we can safely assume that the URI
+ // contains only the path and query params.
+ String hostAndPort = getHostAndPort(modifiedRequest);
+ String path = modifiedRequest.getUri();
+ String url;
+ if (isHttps()) {
+ url = "https://" + hostAndPort + path;
+ } else {
+ url = "http://" + hostAndPort + path;
+ }
+ return url;
+ }
+
+ /**
+ * Returns the full, absolute URL of the original request from the client for both HTTP and HTTPS URLs. The URL
+ * will not reflect modifications from this or other filters.
+ *
+ * @return the full URL of the original request, including scheme, host, port, path, and query parameters
+ */
+ public String getOriginalUrl() {
+ return getFullUrl(originalRequest);
+ }
+
+ /**
+ * Returns the hostname (but not the port) the specified request for both HTTP and HTTPS requests. The request may reflect
+ * modifications from this or other filters. This filter instance must be currently handling the specified request;
+ * otherwise the results are undefined.
+ *
+ * @param modifiedRequest a possibly-modified version of the request currently being processed
+ * @return hostname of the specified request, without the port
+ */
+ public String getHost(HttpRequest modifiedRequest) {
+ String serverHost;
+ if (isHttps()) {
+ HostAndPort hostAndPort = HostAndPort.fromString(getHttpsRequestHostAndPort());
+ serverHost = hostAndPort.getHost();
+ } else {
+ serverHost = HttpUtil.getHostFromRequest(modifiedRequest);
+ }
+ return serverHost;
+ }
+
+ /**
+ * Returns the host and port of the specified request for both HTTP and HTTPS requests. The request may reflect
+ * modifications from this or other filters. This filter instance must be currently handling the specified request;
+ * otherwise the results are undefined.
+ *
+ * @param modifiedRequest a possibly-modified version of the request currently being processed
+ * @return host and port of the specified request
+ */
+ public String getHostAndPort(HttpRequest modifiedRequest) {
+ // For HTTP requests, the host and port can be read from the request itself using the URI and/or
+ // Host header. for HTTPS requests, the host and port are not available in the request. by using the
+ // getHttpsRequestHostAndPort() helper method, we can retrieve the host and port for HTTPS requests.
+ if (isHttps()) {
+ return getHttpsRequestHostAndPort();
+ } else {
+ return HttpUtil.getHostAndPortFromRequest(modifiedRequest);
+ }
+ }
+
+ /**
+ * Returns the host and port of this HTTPS request, including any modifications by other filters.
+ *
+ * @return host and port of this HTTPS request
+ * @throws IllegalStateException if this is not an HTTPS request
+ */
+ private String getHttpsRequestHostAndPort() throws IllegalStateException {
+ if (!isHttps()) {
+ throw new IllegalStateException("Request is not HTTPS. Cannot get host and port on non-HTTPS request using this method.");
+ }
+
+ Attribute hostnameAttr = ctx.attr(AttributeKey.valueOf(HOST_ATTRIBUTE_NAME));
+ return hostnameAttr.get();
+ }
+
+ /**
+ * Returns the original host and port of this HTTPS request, as sent by the client. Does not reflect any modifications
+ * by other filters.
+ * TODO: evaluate this (unused) method and its capture mechanism in HttpsOriginalHostCaptureFilter; remove if not useful.
+ *
+ * @return host and port of this HTTPS request
+ * @throws IllegalStateException if this is not an HTTPS request
+ */
+ private String getHttpsOriginalRequestHostAndPort() throws IllegalStateException {
+ if (!isHttps()) {
+ throw new IllegalStateException("Request is not HTTPS. Cannot get original host and port on non-HTTPS request using this method.");
+ }
+
+ Attribute hostnameAttr = ctx.attr(AttributeKey.valueOf(ORIGINAL_HOST_ATTRIBUTE_NAME));
+ return hostnameAttr.get();
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsHostCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsHostCaptureFilter.java
new file mode 100644
index 000000000..f2a52a014
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsHostCaptureFilter.java
@@ -0,0 +1,43 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.util.Attribute;
+import io.netty.util.AttributeKey;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.littleshoot.proxy.impl.ProxyUtils;
+
+/**
+ * Captures the host for HTTPS requests and stores the value in the ChannelHandlerContext for use by {@link HttpsAwareFiltersAdapter}
+ * filters. This filter reads the host from the HttpRequest during the HTTP CONNECT call, and therefore MUST be invoked
+ * after any other filters which modify the host.
+ * Note: If the request uses the default HTTPS port (443), it will be removed from the hostname captured by this filter.
+ */
+public class HttpsHostCaptureFilter extends HttpFiltersAdapter {
+ public HttpsHostCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ if (ProxyUtils.isCONNECT(httpRequest)) {
+ Attribute hostname = ctx.attr(AttributeKey.valueOf(HttpsAwareFiltersAdapter.HOST_ATTRIBUTE_NAME));
+ String hostAndPort = httpRequest.getUri();
+
+ // CONNECT requests contain the port, even when using the default port. a sensible default is to remove the
+ // default port, since in most cases it is not explicitly specified and its presence (in a HAR file, for example)
+ // would be unexpected.
+ String hostNoDefaultPort = BrowserMobHttpUtil.removeMatchingPort(hostAndPort, 443);
+ hostname.set(hostNoDefaultPort);
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsOriginalHostCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsOriginalHostCaptureFilter.java
new file mode 100644
index 000000000..4a6894c4d
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsOriginalHostCaptureFilter.java
@@ -0,0 +1,34 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.util.Attribute;
+import io.netty.util.AttributeKey;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.littleshoot.proxy.impl.ProxyUtils;
+
+/**
+ * Captures the original host for HTTPS requests and stores the value in the ChannelHandlerContext for use by {@link HttpsAwareFiltersAdapter}
+ * filters. This filter sets the isHttps attribute on the ChannelHandlerContext during the HTTP CONNECT and therefore MUST be invoked before
+ * any other filters calling any of the methods in {@link HttpsAwareFiltersAdapter}.
+ * This filter extends {@link HttpsHostCaptureFilter} and so also sets the host attribute on the channel for use by filters
+ * that modify the original host during the CONNECT. If the hostname is modified by filters, it will be overwritten when the {@link HttpsHostCaptureFilter}
+ * is processed later in the filter chain.
+ */
+public class HttpsOriginalHostCaptureFilter extends HttpsHostCaptureFilter {
+ public HttpsOriginalHostCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+
+ // if this is an HTTP CONNECT, set the isHttps attribute on the ChannelHandlerConect and capture the hostname from the original request.
+ // capturing the original host (and the remapped/modified host in clientToProxyRequest() below) guarantees that we will
+ // have the "true" host, rather than relying on the Host header in subsequent requests (which may be absent or spoofed by malicious clients).
+ if (ProxyUtils.isCONNECT(originalRequest)) {
+ Attribute originalHostAttr = ctx.attr(AttributeKey.valueOf(HttpsAwareFiltersAdapter.ORIGINAL_HOST_ATTRIBUTE_NAME));
+ String hostAndPort = originalRequest.getUri();
+ originalHostAttr.set(hostAndPort);
+
+ Attribute isHttpsAttr = ctx.attr(AttributeKey.valueOf(HttpsAwareFiltersAdapter.IS_HTTPS_ATTRIBUTE_NAME));
+ isHttpsAttr.set(true);
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/LatencyFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/LatencyFilter.java
new file mode 100644
index 000000000..10f90dffc
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/LatencyFilter.java
@@ -0,0 +1,43 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Adds latency to a response before sending it to the client. This filter always adds the specified latency, even if the latency
+ * between the proxy and the remote server already exceeds this value.
+ */
+public class LatencyFilter extends HttpFiltersAdapter {
+ private static final Logger log = LoggerFactory.getLogger(HttpFiltersAdapter.class);
+
+ private final int latencyMs;
+
+ public LatencyFilter(HttpRequest originalRequest, int latencyMs) {
+ super(originalRequest);
+
+ this.latencyMs = latencyMs;
+ }
+
+ @Override
+ public HttpObject proxyToClientResponse(HttpObject httpObject) {
+ if (httpObject instanceof HttpResponse) {
+ if (latencyMs > 0) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(latencyMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ log.warn("Interrupted while adding latency to response", e);
+ }
+ }
+ }
+
+ return super.proxyToClientResponse(httpObject);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ModifiedRequestAwareFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ModifiedRequestAwareFilter.java
new file mode 100644
index 000000000..c117c0b56
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ModifiedRequestAwareFilter.java
@@ -0,0 +1,19 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.handler.codec.http.HttpRequest;
+
+/**
+ * Indicates that a filter wishes to capture the final HttpRequest that is sent to the server, reflecting all
+ * modifications from request filters. {@link BrowserMobHttpFilterChain#clientToProxyRequest(io.netty.handler.codec.http.HttpObject)}
+ * will invoke the {@link #setModifiedHttpRequest(HttpRequest)} method after all filters have processed the initial
+ * {@link HttpRequest} object.
+ */
+public interface ModifiedRequestAwareFilter {
+ /**
+ * Notifies implementing classes of the modified HttpRequest that will be sent to the server, reflecting all
+ * modifications from filters.
+ *
+ * @param modifiedHttpRequest the modified HttpRequest sent to the server
+ */
+ void setModifiedHttpRequest(HttpRequest modifiedHttpRequest);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/RegisterRequestFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RegisterRequestFilter.java
new file mode 100644
index 000000000..32dc1285b
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RegisterRequestFilter.java
@@ -0,0 +1,30 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.proxy.ActivityMonitor;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+
+/**
+ * Registers this request with the {@link net.lightbody.bmp.proxy.ActivityMonitor} when the HttpRequest is received from the client.
+ */
+public class RegisterRequestFilter extends HttpFiltersAdapter {
+ private final ActivityMonitor activityMonitor;
+
+ public RegisterRequestFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, ActivityMonitor activityMonitor) {
+ super(originalRequest, ctx);
+
+ this.activityMonitor = activityMonitor;
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ activityMonitor.requestStarted();
+ }
+
+ return super.clientToProxyRequest(httpObject);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilter.java
new file mode 100644
index 000000000..4718d2b92
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilter.java
@@ -0,0 +1,24 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.util.HttpMessageContents;
+import net.lightbody.bmp.util.HttpMessageInfo;
+
+/**
+ * A functional interface to simplify modification and manipulation of requests.
+ */
+public interface RequestFilter {
+ /**
+ * Implement this method to filter an HTTP request. The HTTP method, URI, headers, etc. are available in the {@code request} parameter,
+ * while the contents of the message are available in the {@code contents} parameter. The request can be modified directly, while the
+ * contents may be modified using the {@link HttpMessageContents#setTextContents(String)} or {@link HttpMessageContents#setBinaryContents(byte[])}
+ * methods. The request can be "short-circuited" by returning a non-null value.
+ *
+ * @param request The request object, including method, URI, headers, etc. Modifications to the request object will be reflected in the request sent to the server.
+ * @param contents The request contents.
+ * @param messageInfo Additional information relating to the HTTP message.
+ * @return if the return value is non-null, the proxy will suppress the request and send the specified response to the client immediately
+ */
+ HttpResponse filterRequest(HttpRequest request, HttpMessageContents contents, HttpMessageInfo messageInfo);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilterAdapter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilterAdapter.java
new file mode 100644
index 000000000..2139ceb54
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RequestFilterAdapter.java
@@ -0,0 +1,99 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpMessage;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.util.HttpMessageContents;
+import net.lightbody.bmp.util.HttpMessageInfo;
+import org.littleshoot.proxy.HttpFilters;
+import org.littleshoot.proxy.HttpFiltersSourceAdapter;
+
+/**
+ * A filter adapter for {@link RequestFilter} implementations. Executes the filter when the {@link HttpFilters#clientToProxyRequest(HttpObject)}
+ * method is invoked.
+ */
+public class RequestFilterAdapter extends HttpsAwareFiltersAdapter {
+ private final RequestFilter requestFilter;
+
+ public RequestFilterAdapter(HttpRequest originalRequest, ChannelHandlerContext ctx, RequestFilter requestFilter) {
+ super(originalRequest, ctx);
+
+ this.requestFilter = requestFilter;
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ // only filter when the original HttpRequest comes through. the RequestFilterAdapter is not designed to filter
+ // any subsequent HttpContents.
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ HttpMessageContents contents;
+ if (httpObject instanceof FullHttpMessage) {
+ FullHttpMessage httpContent = (FullHttpMessage) httpObject;
+ contents = new HttpMessageContents(httpContent);
+ } else {
+ // the HTTP object is not a FullHttpMessage, which means that message contents are not available on this request and cannot be modified.
+ contents = null;
+ }
+
+ HttpMessageInfo messageInfo = new HttpMessageInfo(originalRequest, ctx, isHttps(), getFullUrl(httpRequest), getOriginalUrl());
+
+ HttpResponse response = requestFilter.filterRequest(httpRequest, contents, messageInfo);
+ if (response != null) {
+ return response;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * A {@link HttpFiltersSourceAdapter} for {@link RequestFilterAdapter}s. By default, this FilterSource enables HTTP message aggregation
+ * and sets a maximum request buffer size of 2 MiB.
+ */
+ public static class FilterSource extends HttpFiltersSourceAdapter {
+ private static final int DEFAULT_MAXIMUM_REQUEST_BUFFER_SIZE = 2097152;
+
+ private final RequestFilter filter;
+ private final int maximumRequestBufferSizeInBytes;
+
+ /**
+ * Creates a new filter source that will invoke the specified filter and uses the {@link #DEFAULT_MAXIMUM_REQUEST_BUFFER_SIZE} as
+ * the maximum buffer size.
+ *
+ * @param filter RequestFilter to invoke
+ */
+ public FilterSource(RequestFilter filter) {
+ this.filter = filter;
+ this.maximumRequestBufferSizeInBytes = DEFAULT_MAXIMUM_REQUEST_BUFFER_SIZE;
+ }
+
+ /**
+ * Creates a new filter source that will invoke the specified filter and uses the maximumRequestBufferSizeInBytes as the maximum
+ * buffer size. Set maximumRequestBufferSizeInBytes to 0 to disable aggregation. If message aggregation is disabled,
+ * the {@link HttpMessageContents} will not be available for modification. (Note: HTTP message aggregation will
+ * be enabled if any filter has a maximum request or response buffer size greater than 0. See
+ * {@link org.littleshoot.proxy.HttpFiltersSource#getMaximumRequestBufferSizeInBytes()} for details.)
+ *
+ * @param filter RequestFilter to invoke
+ * @param maximumRequestBufferSizeInBytes maximum buffer size when aggregating Requests for filtering
+ */
+ public FilterSource(RequestFilter filter, int maximumRequestBufferSizeInBytes) {
+ this.filter = filter;
+ this.maximumRequestBufferSizeInBytes = maximumRequestBufferSizeInBytes;
+ }
+
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new RequestFilterAdapter(originalRequest, ctx, filter);
+ }
+
+ @Override
+ public int getMaximumRequestBufferSizeInBytes() {
+ return maximumRequestBufferSizeInBytes;
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java
new file mode 100644
index 000000000..0cc4dcaa5
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java
@@ -0,0 +1,73 @@
+package net.lightbody.bmp.filters;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.net.HostAndPort;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpRequest;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Caches hostname resolutions reported by the {@link org.littleshoot.proxy.HttpFilters#proxyToServerResolutionSucceeded(String, InetSocketAddress)}
+ * filter method. Allows access to the resolved IP address on subsequent requests, when the address is not re-resolved because
+ * the connection has already been established.
+ */
+public class ResolvedHostnameCacheFilter extends HttpFiltersAdapter {
+ /**
+ * The maximum amount of time to save host name resolution information. This is done in order to populate the server IP address field in the
+ * har. Unfortunately there is not currently any way to determine the remote IP address of a keep-alive connection in a filter, so caching the
+ * resolved hostnames gives a generally-reasonable best guess.
+ */
+ private static final int RESOLVED_ADDRESSES_EVICTION_SECONDS = 600;
+
+ /**
+ * Concurrency of the resolvedAddresses map. Should be approximately equal to the maximum number of simultaneous connection
+ * attempts (but not necessarily simultaneous connections). A lower value will inhibit performance.
+ */
+ private static final int RESOLVED_ADDRESSES_CONCURRENCY_LEVEL = 50;
+
+ /**
+ * A {@code Map} that provides a reasonable estimate of the upstream server's IP address for keep-alive connections.
+ * The expiration time is renewed after each access, rather than after each write, so if the connection is consistently kept alive and used,
+ * the cached IP address will not be evicted.
+ */
+ private static final Cache resolvedAddresses =
+ CacheBuilder.newBuilder()
+ .expireAfterAccess(RESOLVED_ADDRESSES_EVICTION_SECONDS, TimeUnit.SECONDS)
+ .concurrencyLevel(RESOLVED_ADDRESSES_CONCURRENCY_LEVEL)
+ .build();
+
+ public ResolvedHostnameCacheFilter(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ super(originalRequest, ctx);
+ }
+
+ @Override
+ public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) {
+ // the address *should* always be resolved at this point
+ InetAddress resolvedAddress = resolvedRemoteAddress.getAddress();
+
+ if (resolvedAddress != null) {
+ // place the resolved host into the hostname cache, so subsequent requests will be able to identify the IP address
+ HostAndPort parsedHostAndPort = HostAndPort.fromString(serverHostAndPort);
+ String host = parsedHostAndPort.getHost();
+
+ if (host != null && !host.isEmpty()) {
+ resolvedAddresses.put(host, resolvedAddress.getHostAddress());
+ }
+ }
+ }
+
+ /**
+ * Returns the (cached) address that was previously resolved for the specified host.
+ *
+ * @param host hostname that was previously resolved (without a port)
+ * @return the resolved IP address for the host, or null if the resolved address is not in the cache
+ */
+ public static String getPreviouslyResolvedAddressForHost(String host) {
+ return resolvedAddresses.getIfPresent(host);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilter.java
new file mode 100644
index 000000000..14ed7d4c2
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilter.java
@@ -0,0 +1,22 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.util.HttpMessageContents;
+import net.lightbody.bmp.util.HttpMessageInfo;
+
+/**
+ * A functional interface to simplify modification and manipulation of responses.
+ */
+public interface ResponseFilter {
+ /**
+ * Implement this method to filter an HTTP response. The URI, headers, status line, etc. are available in the {@code response} parameter,
+ * while the contents of the message are available in the {@code contents} parameter. The response can be modified directly, while the
+ * contents may be modified using the {@link HttpMessageContents#setTextContents(String)} or {@link HttpMessageContents#setBinaryContents(byte[])}
+ * methods.
+ *
+ * @param response The response object, including URI, headers, status line, etc. Modifications to the response object will be reflected in the client response.
+ * @param contents The response contents.
+ * @param messageInfo Additional information relating to the HTTP message.
+ */
+ void filterResponse(HttpResponse response, HttpMessageContents contents, HttpMessageInfo messageInfo);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilterAdapter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilterAdapter.java
new file mode 100644
index 000000000..7edc34dd1
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResponseFilterAdapter.java
@@ -0,0 +1,106 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpMessage;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.util.HttpMessageContents;
+import net.lightbody.bmp.util.HttpMessageInfo;
+import org.littleshoot.proxy.HttpFilters;
+import org.littleshoot.proxy.HttpFiltersSourceAdapter;
+
+/**
+ * A filter adapter for {@link ResponseFilter} implementations. Executes the filter when the {@link HttpFilters#serverToProxyResponse(HttpObject)}
+ * method is invoked.
+ */
+public class ResponseFilterAdapter extends HttpsAwareFiltersAdapter implements ModifiedRequestAwareFilter {
+ private final ResponseFilter responseFilter;
+
+ /**
+ * The final HttpRequest sent to the server, reflecting all modifications from request filters.
+ */
+ private HttpRequest modifiedHttpRequest;
+
+ public ResponseFilterAdapter(HttpRequest originalRequest, ChannelHandlerContext ctx, ResponseFilter responseFilter) {
+ super(originalRequest, ctx);
+
+ this.responseFilter = responseFilter;
+ }
+
+ @Override
+ public HttpObject serverToProxyResponse(HttpObject httpObject) {
+ // only filter when the original HttpResponse comes through. the ResponseFilterAdapter is not designed to filter
+ // any subsequent HttpContents.
+ if (httpObject instanceof HttpResponse) {
+ HttpResponse httpResponse = (HttpResponse) httpObject;
+
+ HttpMessageContents contents;
+ if (httpObject instanceof FullHttpMessage) {
+ FullHttpMessage httpContent = (FullHttpMessage) httpObject;
+ contents = new HttpMessageContents(httpContent);
+ } else {
+ // the HTTP object is not a FullHttpMessage, which means that message contents will not be available on this response and cannot be modified.
+ contents = null;
+ }
+
+ HttpMessageInfo messageInfo = new HttpMessageInfo(originalRequest, ctx, isHttps(), getFullUrl(modifiedHttpRequest), getOriginalUrl());
+
+ responseFilter.filterResponse(httpResponse, contents, messageInfo);
+ }
+
+ return super.serverToProxyResponse(httpObject);
+ }
+
+ @Override
+ public void setModifiedHttpRequest(HttpRequest modifiedHttpRequest) {
+ this.modifiedHttpRequest = modifiedHttpRequest;
+ }
+
+ /**
+ * A {@link HttpFiltersSourceAdapter} for {@link ResponseFilterAdapter}s. By default, this FilterSource enables HTTP message aggregation
+ * and sets a maximum response buffer size of 2 MiB.
+ */
+ public static class FilterSource extends HttpFiltersSourceAdapter {
+ private static final int DEFAULT_MAXIMUM_RESPONSE_BUFFER_SIZE = 2097152;
+
+ private final ResponseFilter filter;
+ private final int maximumResponseBufferSizeInBytes;
+
+ /**
+ * Creates a new filter source that will invoke the specified filter and uses the {@link #DEFAULT_MAXIMUM_RESPONSE_BUFFER_SIZE} as
+ * the maximum buffer size.
+ *
+ * @param filter ResponseFilter to invoke
+ */
+ public FilterSource(ResponseFilter filter) {
+ this.filter = filter;
+ this.maximumResponseBufferSizeInBytes = DEFAULT_MAXIMUM_RESPONSE_BUFFER_SIZE;
+ }
+
+ /**
+ * Creates a new filter source that will invoke the specified filter and uses the maximumResponseBufferSizeInBytes as the maximum
+ * buffer size. Set maximumResponseBufferSizeInBytes to 0 to disable aggregation. If message aggregation is disabled,
+ * the {@link HttpMessageContents} will not be available for modification. (Note: HTTP message aggregation will
+ * be enabled if any filter has a maximum request or response buffer size greater than 0. See
+ * {@link org.littleshoot.proxy.HttpFiltersSource#getMaximumResponseBufferSizeInBytes()} for details.)
+ *
+ * @param filter ResponseFilter to invoke
+ * @param maximumResponseBufferSizeInBytes maximum buffer size when aggregating responses for filtering
+ */
+ public FilterSource(ResponseFilter filter, int maximumResponseBufferSizeInBytes) {
+ this.filter = filter;
+ this.maximumResponseBufferSizeInBytes = maximumResponseBufferSizeInBytes;
+ }
+
+ @Override
+ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+ return new ResponseFilterAdapter(originalRequest, ctx, filter);
+ }
+
+ @Override
+ public int getMaximumResponseBufferSizeInBytes() {
+ return maximumResponseBufferSizeInBytes;
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/RewriteUrlFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RewriteUrlFilter.java
new file mode 100644
index 000000000..ef0e223c0
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/RewriteUrlFilter.java
@@ -0,0 +1,130 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.util.HttpUtil;
+import net.lightbody.bmp.proxy.RewriteRule;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.impl.ProxyUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.regex.Matcher;
+
+/**
+ * Applies rewrite rules to the specified request. If a rewrite rule matches, the request's URI will be overwritten with the rewritten URI.
+ * The filter does not make a defensive copy of the rewrite rule collection, so there is no guarantee
+ * that the collection at the time of construction will contain the same values when the filter is actually invoked, if the collection is
+ * modified concurrently.
+ */
+public class RewriteUrlFilter extends HttpsAwareFiltersAdapter {
+ private static final Logger log = LoggerFactory.getLogger(RewriteUrlFilter.class);
+
+ private final Collection rewriteRules;
+
+ public RewriteUrlFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Collection rewriterules) {
+ super(originalRequest, ctx);
+
+ if (rewriterules != null) {
+ this.rewriteRules = rewriterules;
+ } else {
+ this.rewriteRules = Collections.emptyList();
+ }
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ // if this is a CONNECT request, don't bother applying the rewrite rules, since CONNECT rewriting is not supported
+ if (ProxyUtils.isCONNECT(httpRequest)) {
+ return null;
+ }
+
+ String originalUrl = getFullUrl(httpRequest);
+ String rewrittenUrl = originalUrl;
+
+ boolean rewroteUri = false;
+ for (RewriteRule rule : rewriteRules) {
+ Matcher matcher = rule.getPattern().matcher(rewrittenUrl);
+ if (matcher.matches()) {
+ rewrittenUrl = matcher.replaceAll(rule.getReplace());
+ rewroteUri = true;
+ }
+ }
+
+ if (rewroteUri) {
+ // if the URI in the request contains the scheme, host, and port, the request's URI can be replaced
+ // with the rewritten URI. if not (for example, on HTTPS requests), strip the scheme, host, and port from
+ // the rewritten URL before replacing the URI on the request.
+ String uriFromRequest = httpRequest.getUri();
+ if (HttpUtil.startsWithHttpOrHttps(uriFromRequest)) {
+ httpRequest.setUri(rewrittenUrl);
+ } else {
+ try {
+ String resource = BrowserMobHttpUtil.getRawPathAndParamsFromUri(rewrittenUrl);
+ httpRequest.setUri(resource);
+ } catch (URISyntaxException e) {
+ // the rewritten URL couldn't be parsed, possibly due to the rewrite rule mangling the URL. log
+ // a warning message and replace the resource on the request with the full, rewritten URL.
+ log.warn("Unable to determine path from rewritten URL. Request URL will be set to the full rewritten URL instead of the resource's path.\n\tOriginal URL: {}\n\tRewritten URL: {}",
+ originalUrl,
+ rewrittenUrl,
+ e);
+
+ httpRequest.setUri(rewrittenUrl);
+ }
+ }
+
+ // determine if the hostname and/or port has been changed by the rewrite rule. if so, update the Host
+ // header for HTTP requests. for HTTPS requests, log a warning, since hostname and port cannot be changed
+ // by rewrite rules.
+
+ String originalHostAndPort = null;
+ try {
+ originalHostAndPort = HttpUtil.getHostAndPortFromUri(originalUrl);
+ } catch (URISyntaxException e) {
+ // for some reason we couldn't determine the original host and port from the original URL. log a warning,
+ // and allow the Host header to be forcibly updated to the rewritten host and port.
+ log.warn("Unable to determine host and port from original URL. Host header will be set to rewritten URL's host and port.\n\tOriginal URL: {}\n\tRewritten URL: {}",
+ originalUrl,
+ rewrittenUrl,
+ e);
+ }
+
+ String modifiedHostAndPort = null;
+ try {
+ modifiedHostAndPort = HttpUtil.getHostAndPortFromUri(rewrittenUrl);
+ } catch (URISyntaxException e) {
+ log.warn("Unable to determine host and port from rewritten URL. Host header will not be updated.\n\tOriginal URL: {}\n\tRewritten URL: {}",
+ originalUrl,
+ rewrittenUrl,
+ e);
+ }
+
+ // if the modifiedHostAndPort was parsed successfully and is different from the originalHostAndPort, update the Host header
+ if (modifiedHostAndPort != null && !modifiedHostAndPort.equals(originalHostAndPort)) {
+ if (isHttps()) {
+ // for HTTPS requests we cannot modify the host and port, since we are always reusing a persistent connection.
+ log.warn("Cannot rewrite the host or port of an HTTPS connection.\n\tHost and port from original request: {}\n\tRewritten host and port: {}",
+ originalHostAndPort, modifiedHostAndPort);
+ } else {
+ // only modify the Host header if it already exists
+ if (httpRequest.headers().contains(HttpHeaders.Names.HOST)) {
+ HttpHeaders.setHost(httpRequest, modifiedHostAndPort);
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ServerResponseCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ServerResponseCaptureFilter.java
new file mode 100644
index 000000000..d69ad8f76
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ServerResponseCaptureFilter.java
@@ -0,0 +1,215 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * This filter captures responses from the server (headers and content). The filter can also decompress contents if desired.
+ *
+ * The filter can be used in one of three ways: (1) directly, by adding the filter to the filter chain; (2) by subclassing
+ * the filter and overriding its filter methods; or (3) by invoking the filter directly from within another filter (see
+ * {@link net.lightbody.bmp.filters.HarCaptureFilter} for an example of the latter).
+ */
+public class ServerResponseCaptureFilter extends HttpFiltersAdapter {
+ private static final Logger log = LoggerFactory.getLogger(ServerResponseCaptureFilter.class);
+
+ /**
+ * Populated by serverToProxyResponse() when processing the HttpResponse object
+ */
+ private volatile HttpResponse httpResponse;
+
+ /**
+ * Populated by serverToProxyResponse() as it receives HttpContent responses. If the response is chunked, it will
+ * be populated across multiple calls to proxyToServerResponse().
+ */
+ private final ByteArrayOutputStream rawResponseContents = new ByteArrayOutputStream();
+
+ /**
+ * Populated when processing the LastHttpContent. If the response is compressed and decompression is requested,
+ * this contains the entire decompressed response. Otherwise it contains the raw response.
+ */
+ private volatile byte[] fullResponseContents;
+
+ /**
+ * Populated by serverToProxyResponse() when it processes the LastHttpContent object.
+ */
+ private volatile HttpHeaders trailingHeaders;
+
+ /**
+ * Set to true when processing the LastHttpContent if the server indicates there is a content encoding.
+ */
+ private volatile boolean responseCompressed;
+
+ /**
+ * Set to true when processing the LastHttpContent if decompression was requested and successful.
+ */
+ private volatile boolean decompressionSuccessful;
+
+ /**
+ * Populated when processing the LastHttpContent.
+ */
+ private volatile String contentEncoding;
+
+ /**
+ * User option indicating compressed content should be uncompressed.
+ */
+ private final boolean decompressEncodedContent;
+
+ public ServerResponseCaptureFilter(HttpRequest originalRequest, boolean decompressEncodedContent) {
+ super(originalRequest);
+
+ this.decompressEncodedContent = decompressEncodedContent;
+ }
+
+ public ServerResponseCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, boolean decompressEncodedContent) {
+ super(originalRequest, ctx);
+
+ this.decompressEncodedContent = decompressEncodedContent;
+ }
+
+ @Override
+ public HttpObject serverToProxyResponse(HttpObject httpObject) {
+ if (httpObject instanceof HttpResponse) {
+ httpResponse = (HttpResponse) httpObject;
+ captureContentEncoding(httpResponse);
+ }
+
+ if (httpObject instanceof HttpContent) {
+ HttpContent httpContent = (HttpContent) httpObject;
+
+ storeResponseContent(httpContent);
+
+ if (httpContent instanceof LastHttpContent) {
+ LastHttpContent lastContent = (LastHttpContent) httpContent;
+ captureTrailingHeaders(lastContent);
+
+ captureFullResponseContents();
+ }
+ }
+
+ return super.serverToProxyResponse(httpObject);
+ }
+
+ protected void captureFullResponseContents() {
+ // start by setting fullResponseContent to the raw, (possibly) compressed byte stream. replace it
+ // with the decompressed bytes if decompression is successful.
+ fullResponseContents = getRawResponseContents();
+
+ // if the content is compressed, we need to decompress it. but don't use
+ // the netty HttpContentCompressor/Decompressor in the pipeline because we don't actually want it to
+ // change the message sent to the client
+ if (contentEncoding != null) {
+ responseCompressed = true;
+
+ if (decompressEncodedContent) {
+ decompressContents();
+ } else {
+ // will not decompress response
+ }
+ } else {
+ // no compression
+ responseCompressed = false;
+ }
+ }
+
+ protected void decompressContents() {
+ if (contentEncoding.equals(HttpHeaders.Values.GZIP)) {
+ try {
+ fullResponseContents = BrowserMobHttpUtil.decompressContents(getRawResponseContents());
+ decompressionSuccessful = true;
+ } catch (RuntimeException e) {
+ log.warn("Failed to decompress response with encoding type " + contentEncoding + " when decoding request from " + originalRequest.getUri(), e);
+ }
+ } else {
+ log.warn("Cannot decode unsupported content encoding type {}", contentEncoding);
+ }
+ }
+
+ protected void captureContentEncoding(HttpResponse httpResponse) {
+ contentEncoding = HttpHeaders.getHeader(httpResponse, HttpHeaders.Names.CONTENT_ENCODING);
+ }
+
+ protected void captureTrailingHeaders(LastHttpContent lastContent) {
+ trailingHeaders = lastContent.trailingHeaders();
+
+ // technically, the Content-Encoding header can be in a trailing header, although this is excruciatingly uncommon
+ if (trailingHeaders != null) {
+ String trailingContentEncoding = trailingHeaders.get(HttpHeaders.Names.CONTENT_ENCODING);
+ if (trailingContentEncoding != null) {
+ contentEncoding = trailingContentEncoding;
+ }
+ }
+
+ }
+
+ protected void storeResponseContent(HttpContent httpContent) {
+ ByteBuf bufferedContent = httpContent.content();
+ byte[] content = BrowserMobHttpUtil.extractReadableBytes(bufferedContent);
+
+ try {
+ rawResponseContents.write(content);
+ } catch (IOException e) {
+ // can't happen
+ }
+ }
+
+ public HttpResponse getHttpResponse() {
+ return httpResponse;
+ }
+
+ /**
+ * Returns the contents of the entire response. If the contents were compressed, decompressEncodedContent is true, and
+ * decompression was successful, this method returns the decompressed contents.
+ *
+ * @return entire response contents, decompressed if possible
+ */
+ public byte[] getFullResponseContents() {
+ return fullResponseContents;
+ }
+
+ /**
+ * Returns the raw contents of the entire response, without decompression.
+ *
+ * @return entire response contents, without decompression
+ */
+ public byte[] getRawResponseContents() {
+ return rawResponseContents.toByteArray();
+ }
+
+ public HttpHeaders getTrailingHeaders() {
+ return trailingHeaders;
+ }
+
+ public boolean isResponseCompressed() {
+ return responseCompressed;
+ }
+
+ /**
+ * @return true if decompression is both enabled and successful
+ */
+ public boolean isDecompressionSuccessful() {
+ if (!decompressEncodedContent) {
+ return false;
+ }
+
+ return decompressionSuccessful;
+ }
+
+ public String getContentEncoding() {
+ return contentEncoding;
+ }
+
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/UnregisterRequestFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/UnregisterRequestFilter.java
new file mode 100644
index 000000000..1ffff0f72
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/UnregisterRequestFilter.java
@@ -0,0 +1,30 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.LastHttpContent;
+import net.lightbody.bmp.proxy.ActivityMonitor;
+import org.littleshoot.proxy.HttpFiltersAdapter;
+
+/**
+ * Unregisters this request with the {@link net.lightbody.bmp.proxy.ActivityMonitor} when the LastHttpContent is sent to the client.
+ */
+public class UnregisterRequestFilter extends HttpFiltersAdapter {
+ private final ActivityMonitor activityMonitor;
+
+ public UnregisterRequestFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, ActivityMonitor activityMonitor) {
+ super(originalRequest, ctx);
+
+ this.activityMonitor = activityMonitor;
+ }
+
+ @Override
+ public HttpObject proxyToClientResponse(HttpObject httpObject) {
+ if (httpObject instanceof LastHttpContent) {
+ activityMonitor.requestFinished();
+ }
+
+ return super.proxyToClientResponse(httpObject);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/WhitelistFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/WhitelistFilter.java
new file mode 100644
index 000000000..ad8fbf6f6
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/WhitelistFilter.java
@@ -0,0 +1,75 @@
+package net.lightbody.bmp.filters;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import org.littleshoot.proxy.impl.ProxyUtils;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.regex.Pattern;
+
+/**
+ * Checks this request against the whitelist, and returns the modified response if the request is not in the whitelist. The filter does not
+ * make a defensive copy of the whitelist URLs, so there is no guarantee that the whitelist URLs at the time of construction will contain the
+ * same values when the filter is actually invoked, if the URL collection is modified concurrently.
+ */
+public class WhitelistFilter extends HttpsAwareFiltersAdapter {
+ private final boolean whitelistEnabled;
+ private final int whitelistResponseCode;
+ private final Collection whitelistUrls;
+
+ public WhitelistFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, boolean whitelistEnabled,int whitelistResponseCode,
+ Collection whitelistUrls) {
+ super(originalRequest, ctx);
+
+ this.whitelistEnabled = whitelistEnabled;
+ this.whitelistResponseCode = whitelistResponseCode;
+ if (whitelistUrls != null) {
+ this.whitelistUrls = whitelistUrls;
+ } else {
+ this.whitelistUrls = Collections.emptyList();
+ }
+ }
+
+ @Override
+ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
+ if (!whitelistEnabled) {
+ return null;
+ }
+
+ if (httpObject instanceof HttpRequest) {
+ HttpRequest httpRequest = (HttpRequest) httpObject;
+
+ // do not allow HTTP CONNECTs to be short-circuited
+ if (ProxyUtils.isCONNECT(httpRequest)) {
+ return null;
+ }
+
+ boolean urlWhitelisted = false;
+
+ String url = getFullUrl(httpRequest);
+
+ for (Pattern pattern : whitelistUrls) {
+ if (pattern.matcher(url).matches()) {
+ urlWhitelisted = true;
+ break;
+ }
+ }
+
+ if (!urlWhitelisted) {
+ HttpResponseStatus status = HttpResponseStatus.valueOf(whitelistResponseCode);
+ HttpResponse resp = new DefaultFullHttpResponse(httpRequest.getProtocolVersion(), status);
+ HttpHeaders.setContentLength(resp, 0L);
+
+ return resp;
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/support/HttpConnectTiming.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/support/HttpConnectTiming.java
new file mode 100644
index 000000000..c18070f7b
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/support/HttpConnectTiming.java
@@ -0,0 +1,51 @@
+package net.lightbody.bmp.filters.support;
+
+/**
+ * Holds the connection-related timing information from an HTTP CONNECT request, so it can be added to the HAR timings for the first
+ * "real" request to the same host. The HTTP CONNECT and the "real" HTTP requests are processed in different HarCaptureFilter instances.
+ *
+ * Note: The connect time must include the ssl time. According to the HAR spec at https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.htm:
+
+ ssl [number, optional] (new in 1.2) - Time required for SSL/TLS negotiation. If this field is defined then the time is also
+ included in the connect field (to ensure backward compatibility with HAR 1.1). Use -1 if the timing does not apply to the
+ current request.
+
+ */
+public class HttpConnectTiming {
+ private volatile long blockedTimeNanos = -1;
+ private volatile long dnsTimeNanos = -1;
+ private volatile long connectTimeNanos = -1;
+ private volatile long sslHandshakeTimeNanos = -1;
+
+ public void setConnectTimeNanos(long connectTimeNanos) {
+ this.connectTimeNanos = connectTimeNanos;
+ }
+
+ public void setSslHandshakeTimeNanos(long sslHandshakeTimeNanos) {
+ this.sslHandshakeTimeNanos = sslHandshakeTimeNanos;
+ }
+
+ public void setBlockedTimeNanos(long blockedTimeNanos) {
+ this.blockedTimeNanos = blockedTimeNanos;
+ }
+
+ public void setDnsTimeNanos(long dnsTimeNanos) {
+ this.dnsTimeNanos = dnsTimeNanos;
+ }
+
+ public long getConnectTimeNanos() {
+ return connectTimeNanos;
+ }
+
+ public long getSslHandshakeTimeNanos() {
+ return sslHandshakeTimeNanos;
+ }
+
+ public long getBlockedTimeNanos() {
+ return blockedTimeNanos;
+ }
+
+ public long getDnsTimeNanos() {
+ return dnsTimeNanos;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/util/HarCaptureUtil.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/util/HarCaptureUtil.java
new file mode 100644
index 000000000..1ac47f93f
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/util/HarCaptureUtil.java
@@ -0,0 +1,96 @@
+package net.lightbody.bmp.filters.util;
+
+import net.lightbody.bmp.core.har.HarResponse;
+
+/**
+ * Static utility methods for {@link net.lightbody.bmp.filters.HarCaptureFilter} and {@link net.lightbody.bmp.filters.HttpConnectHarCaptureFilter}.
+ */
+public class HarCaptureUtil {
+ /**
+ * The HTTP version string in the {@link HarResponse} for failed requests.
+ */
+ public static final String HTTP_VERSION_STRING_FOR_FAILURE = "unknown";
+
+ /**
+ * The HTTP status code in the {@link HarResponse} for failed requests.
+ */
+ public static final int HTTP_STATUS_CODE_FOR_FAILURE = 0;
+
+ /**
+ * The HTTP status text/reason phrase in the {@link HarResponse} for failed requests.
+ */
+ public static final String HTTP_REASON_PHRASE_FOR_FAILURE = "";
+
+ /**
+ * The error message that will be populated in the _error field of the {@link HarResponse} due to a name
+ * lookup failure.
+ */
+ private static final String RESOLUTION_FAILED_ERROR_MESSAGE = "Unable to resolve host: ";
+
+ /**
+ * The error message that will be populated in the _error field of the {@link HarResponse} due to a
+ * connection failure.
+ */
+ private static final String CONNECTION_FAILED_ERROR_MESSAGE = "Unable to connect to host";
+
+ /**
+ * The error message that will be populated in the _error field of the {@link HarResponse} when the proxy fails to
+ * receive a response in a timely manner.
+ */
+ private static final String RESPONSE_TIMED_OUT_ERROR_MESSAGE = "Response timed out";
+
+ /**
+ * The error message that will be populated in the _error field of the {@link HarResponse} when no response is received
+ * from the server for any reason other than a server response timeout.
+ */
+ private static final String NO_RESPONSE_RECEIVED_ERROR_MESSAGE = "No response received";
+
+ /**
+ * Creates a HarResponse object for failed requests. Normally the HarResponse is populated when the response is received
+ * from the server, but if the request fails due to a name resolution issue, connection problem, timeout, etc., no
+ * HarResponse would otherwise be created.
+ *
+ * @return a new HarResponse object with invalid HTTP status code (0) and version string ("unknown")
+ */
+ public static HarResponse createHarResponseForFailure() {
+ return new HarResponse(HTTP_STATUS_CODE_FOR_FAILURE, HTTP_REASON_PHRASE_FOR_FAILURE, HTTP_VERSION_STRING_FOR_FAILURE);
+ }
+
+ /**
+ * Returns the error message for the HAR response when DNS resolution fails.
+ *
+ * @param hostAndPort the host and port of the address lookup that failed
+ * @return the resolution failed error message
+ */
+ public static String getResolutionFailedErrorMessage(String hostAndPort) {
+ return RESOLUTION_FAILED_ERROR_MESSAGE + hostAndPort;
+ }
+
+ /**
+ * Returns the error message for the HAR response when the connection fails.
+ *
+ * @return the connection failed error message
+ */
+ public static String getConnectionFailedErrorMessage() {
+ return CONNECTION_FAILED_ERROR_MESSAGE;
+ }
+
+ /**
+ * Returns the error message for the HAR response when the response from the server times out.
+ *
+ * @return the response timed out error message
+ */
+ public static String getResponseTimedOutErrorMessage() {
+ return RESPONSE_TIMED_OUT_ERROR_MESSAGE;
+ }
+
+ /**
+ * Returns the error message for the HAR response when no response was received from the server (e.g. when the
+ * browser is closed).
+ *
+ * @return the no response received error message
+ */
+ public static String getNoResponseReceivedErrorMessage() {
+ return NO_RESPONSE_RECEIVED_ERROR_MESSAGE;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/ActivityMonitor.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/ActivityMonitor.java
new file mode 100644
index 000000000..7b6fba109
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/ActivityMonitor.java
@@ -0,0 +1,124 @@
+package net.lightbody.bmp.proxy;
+
+import com.google.common.util.concurrent.Monitor;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Tracks active and total requests on a proxy, and provides a mechanism to wait for active requests to finish.
+ * See {@link net.lightbody.bmp.proxy.ActivityMonitor#waitForQuiescence(long, long, java.util.concurrent.TimeUnit)}.
+ */
+public class ActivityMonitor {
+ private final AtomicInteger activeRequests = new AtomicInteger(0);
+ private final AtomicInteger totalRequests = new AtomicInteger(0);
+
+ private final AtomicLong lastRequestFinishedNanos = new AtomicLong(System.nanoTime());
+
+ private final Monitor monitor = new Monitor();
+
+ private final Monitor.Guard requestNotActive = new Monitor.Guard(monitor) {
+ @Override
+ public boolean isSatisfied() {
+ return activeRequests.get() == 0;
+ }
+ };
+
+ private final Monitor.Guard requestActive = new Monitor.Guard(monitor) {
+ @Override
+ public boolean isSatisfied() {
+ return activeRequests.get() > 0;
+ }
+ };
+
+ public void requestStarted() {
+ int previousCount = activeRequests.getAndIncrement();
+ totalRequests.incrementAndGet();
+ if (previousCount == 0) {
+ // previously there were no active requests, but now there are -- signal to any waitForQuiescence threads that they need to
+ // begin waiting again
+ monitor.enter();
+ monitor.leave();
+ }
+ }
+
+ public void requestFinished() {
+ int newCount = activeRequests.decrementAndGet();
+ lastRequestFinishedNanos.set(System.nanoTime());
+
+ if (newCount == 0) {
+ // there are no active requests, so signal to any waitForQuiescence threads that they can begin waiting for their quietPeriod
+ monitor.enter();
+ monitor.leave();
+ }
+ }
+
+ public int getActiveRequests() {
+ return activeRequests.get();
+ }
+
+ public int getTotalRequests() {
+ return totalRequests.get();
+ }
+
+ public boolean waitForQuiescence(long quietPeriod, long timeout, TimeUnit timeUnit) {
+ // the minRequestFinishTime is the earliest possible time the current or last request "could" finish. if there is no active
+ // request, this is simply the lastRequestFinishedNanos time. if there is an active request, it is "now". this helps avoid waiting
+ // for quiescence if there is an active request and the timeout is less than the quietPeriod.
+ long minRequestFinishTime;
+ if (activeRequests.get() == 0) {
+ if (timeUnit.convert(System.nanoTime() - lastRequestFinishedNanos.get(), TimeUnit.NANOSECONDS) >= quietPeriod) {
+ return true;
+ } else {
+ minRequestFinishTime = lastRequestFinishedNanos.get();
+ }
+ } else {
+ minRequestFinishTime = System.nanoTime();
+ }
+
+ // record the maximum time we can wait until (the current time + the timeout), which will allow us to avoid waiting for
+ // quiescence if it is not possible to satisfy the quietPeriod before the waitUntil time elapses
+ long waitUntil = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, timeUnit);
+
+ while (minRequestFinishTime + TimeUnit.NANOSECONDS.convert(quietPeriod, timeUnit) <= waitUntil) {
+ // the maximum amount of time we can wait for active requests to finish that will still allow us to wait for quiescence
+ // for the quietPeriod.
+ long maxWaitTimeForActiveRequests = waitUntil - System.nanoTime() - TimeUnit.NANOSECONDS.convert(quietPeriod, timeUnit);
+
+ // wait for active requests to finish
+ boolean success = monitor.enterWhenUninterruptibly(requestNotActive, maxWaitTimeForActiveRequests, TimeUnit.NANOSECONDS);
+
+ if (!success) {
+ // timed out waiting for active requests to finish
+ return false;
+ }
+
+ monitor.leave();
+
+ // the time needed to monitor for new active requests is whenever the last request finished + the quiet period. this may be less
+ // than the actual quiet period if no requests were active when entering waitForQuiescence, but the quietPeriod has not yet elapsed
+ // since the last request.
+ long waitForNewRequests = lastRequestFinishedNanos.get() - System.nanoTime() + TimeUnit.NANOSECONDS.convert(quietPeriod, timeUnit);
+
+ // if the quietPeriod has already elapsed since the last request, no need to wait any longer
+ if (waitForNewRequests < 0) {
+ return true;
+ }
+
+ // wait for new requests to come in. if a new request comes in, the loop will restart, waiting for active requests to complete.
+ boolean requestsActive = monitor.enterWhenUninterruptibly(requestActive, waitForNewRequests, TimeUnit.NANOSECONDS);
+
+ if (requestsActive) {
+ // a request became active, so we need to wait for all requests to finish again
+ monitor.leave();
+
+ continue;
+ } else {
+ return true;
+ }
+ }
+
+ return false;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/BlacklistEntry.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/BlacklistEntry.java
new file mode 100644
index 000000000..671fff9c7
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/BlacklistEntry.java
@@ -0,0 +1,92 @@
+package net.lightbody.bmp.proxy;
+
+import java.util.regex.Pattern;
+
+/**
+ * An entry in the Blacklist, consisting of a regular expression to match the URL, an HTTP status code, and a regular expression
+ * to match the HTTP method.
+ */
+public class BlacklistEntry {
+ private final Pattern urlPattern;
+ private final int statusCode;
+ private final Pattern httpMethodPattern;
+
+ /**
+ * Creates a new BlacklistEntry with no HTTP method matching (i.e. all methods will match).
+ *
+ * @param urlPattern URL pattern to blacklist
+ * @param statusCode HTTP status code to return for blacklisted URL
+ */
+ public BlacklistEntry(String urlPattern, int statusCode) {
+ this(urlPattern, statusCode, null);
+ }
+
+ /**
+ * Creates a new BlacklistEntry which will match both a URL and an HTTP method
+ *
+ * @param urlPattern URL pattern to blacklist
+ * @param statusCode status code to return for blacklisted URL
+ * @param httpMethodPattern HTTP method to match (e.g. GET, PUT, PATCH, etc.)
+ */
+ public BlacklistEntry(String urlPattern, int statusCode, String httpMethodPattern) {
+ this.urlPattern = Pattern.compile(urlPattern);
+ this.statusCode = statusCode;
+ if (httpMethodPattern == null || httpMethodPattern.isEmpty()) {
+ this.httpMethodPattern = null;
+ } else {
+ this.httpMethodPattern = Pattern.compile(httpMethodPattern);
+ }
+ }
+
+ /**
+ * Determines if this BlacklistEntry matches the given URL. Attempts to match both the URL and the
+ * HTTP method.
+ *
+ * @param url possibly-blacklisted URL
+ * @param httpMethod HTTP method this URL is being accessed with
+ * @return true if the URL matches this BlacklistEntry
+ */
+ public boolean matches(String url, String httpMethod) {
+ if (httpMethodPattern != null) {
+ return urlPattern.matcher(url).matches() && httpMethodPattern.matcher(httpMethod).matches();
+ } else {
+ return urlPattern.matcher(url).matches();
+ }
+ }
+
+ public Pattern getUrlPattern() {
+ return urlPattern;
+ }
+
+ public int getStatusCode() {
+ return statusCode;
+ }
+
+ public Pattern getHttpMethodPattern() {
+ return httpMethodPattern;
+ }
+
+ @Deprecated
+ /**
+ * @deprecated use {@link #getUrlPattern()}
+ */
+ public Pattern getPattern() {
+ return getUrlPattern();
+ }
+
+ @Deprecated
+ /**
+ * @deprecated use {@link #getStatusCode()}
+ */
+ public int getResponseCode() {
+ return getStatusCode();
+ }
+
+ @Deprecated
+ /**
+ * @deprecated use {@link #getHttpMethodPattern()}
+ */
+ public Pattern getMethod() {
+ return getHttpMethodPattern();
+ }
+}
\ No newline at end of file
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/CaptureType.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/CaptureType.java
new file mode 100644
index 000000000..cf3b98ce6
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/CaptureType.java
@@ -0,0 +1,113 @@
+package net.lightbody.bmp.proxy;
+
+import java.util.EnumSet;
+
+/**
+ * Data types that the proxy can capture. Data types are organized into two broad categories, REQUEST_* and
+ * RESPONSE_*, corresponding to client requests and server responses.
+ */
+public enum CaptureType {
+ /**
+ * HTTP request headers, including trailing headers.
+ */
+ REQUEST_HEADERS,
+
+ /**
+ * HTTP Cookies sent with the request.
+ */
+ REQUEST_COOKIES,
+
+ /**
+ * Non-binary HTTP request content, such as post data or other text-based request payload.
+ * See {@link net.lightbody.bmp.util.BrowserMobHttpUtil#hasTextualContent(String)} for a list of Content-Types that
+ * are considered non-binary.
+ *
+ */
+ REQUEST_CONTENT,
+
+ /**
+ * Binary HTTP request content, such as file uploads, or any unrecognized request payload.
+ */
+ REQUEST_BINARY_CONTENT,
+
+ /**
+ * HTTP response headers, including trailing headers.
+ */
+ RESPONSE_HEADERS,
+
+ /**
+ * Set-Cookie headers sent with the response.
+ */
+ RESPONSE_COOKIES,
+
+ /**
+ * Non-binary HTTP response content (typically, HTTP body content).
+ * See {@link net.lightbody.bmp.util.BrowserMobHttpUtil#hasTextualContent(String)} for a list of Content-Types that
+ * are considered non-binary.
+ */
+ RESPONSE_CONTENT,
+
+ /**
+ * Binary HTTP response content, such as image files, or any unrecognized response payload.
+ */
+ RESPONSE_BINARY_CONTENT;
+
+ // the following groups of capture types are private so that clients do not accidentally modify these sets (EnumSets are not immutable)
+ private static final EnumSet REQUEST_CAPTURE_TYPES = EnumSet.of(REQUEST_HEADERS, REQUEST_CONTENT, REQUEST_BINARY_CONTENT, REQUEST_COOKIES);
+ private static final EnumSet RESPONSE_CAPTURE_TYPES = EnumSet.of(RESPONSE_HEADERS, RESPONSE_CONTENT, RESPONSE_BINARY_CONTENT, RESPONSE_COOKIES);
+ private static final EnumSet HEADER_CAPTURE_TYPES = EnumSet.of(REQUEST_HEADERS, RESPONSE_HEADERS);
+ private static final EnumSet NON_BINARY_CONTENT_CAPTURE_TYPES = EnumSet.of(REQUEST_CONTENT, RESPONSE_CONTENT);
+ private static final EnumSet BINARY_CONTENT_CAPTURE_TYPES = EnumSet.of(REQUEST_BINARY_CONTENT, RESPONSE_BINARY_CONTENT);
+ private static final EnumSet ALL_CONTENT_CAPTURE_TYPES = EnumSet.of(REQUEST_CONTENT, RESPONSE_CONTENT, REQUEST_BINARY_CONTENT, RESPONSE_BINARY_CONTENT);
+ private static final EnumSet COOKIE_CAPTURE_TYPES = EnumSet.of(REQUEST_COOKIES, RESPONSE_COOKIES);
+
+ /**
+ * @return Set of CaptureTypes for requests.
+ */
+ public static EnumSet getRequestCaptureTypes() {
+ return EnumSet.copyOf(REQUEST_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for responses.
+ */
+ public static EnumSet getResponseCaptureTypes() {
+ return EnumSet.copyOf(RESPONSE_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for headers.
+ */
+ public static EnumSet getHeaderCaptureTypes() {
+ return EnumSet.copyOf(HEADER_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for non-binary content.
+ */
+ public static EnumSet getNonBinaryContentCaptureTypes() {
+ return EnumSet.copyOf(NON_BINARY_CONTENT_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for binary content.
+ */
+ public static EnumSet getBinaryContentCaptureTypes() {
+ return EnumSet.copyOf(BINARY_CONTENT_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for both binary and non-binary content.
+ */
+ public static EnumSet getAllContentCaptureTypes() {
+ return EnumSet.copyOf(ALL_CONTENT_CAPTURE_TYPES);
+ }
+
+ /**
+ * @return Set of CaptureTypes for cookies.
+ */
+ public static EnumSet getCookieCaptureTypes() {
+ return EnumSet.copyOf(COOKIE_CAPTURE_TYPES);
+ }
+
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/RewriteRule.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/RewriteRule.java
new file mode 100644
index 000000000..c5ec97e3f
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/RewriteRule.java
@@ -0,0 +1,44 @@
+package net.lightbody.bmp.proxy;
+
+import java.util.regex.Pattern;
+
+/**
+ * Container for a URL rewrite rule pattern and replacement string.
+ */
+public class RewriteRule {
+ private final Pattern pattern;
+ private final String replace;
+
+ public RewriteRule(String pattern, String replace) {
+ this.pattern = Pattern.compile(pattern);
+ this.replace = replace;
+ }
+
+ public Pattern getPattern() {
+ return pattern;
+ }
+
+ public String getReplace() {
+ return replace;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RewriteRule that = (RewriteRule) o;
+
+ if (!pattern.equals(that.pattern)) return false;
+ if (!replace.equals(that.replace)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = pattern.hashCode();
+ result = 31 * result + replace.hashCode();
+ return result;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/Whitelist.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/Whitelist.java
new file mode 100644
index 000000000..ea015ee27
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/Whitelist.java
@@ -0,0 +1,129 @@
+package net.lightbody.bmp.proxy;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A URL whitelist. This object is immutable and the list of matching patterns and the HTTP status code is unmodifiable
+ * after creation. Enabling, disabling, or modifying the whitelist can be safely and easily accomplished by updating the
+ * whitelist reference to a new whitelist.
+ */
+public class Whitelist {
+ private final List patterns;
+ private final int statusCode;
+ private final boolean enabled;
+
+ /**
+ * A disabled Whitelist.
+ */
+ public static final Whitelist WHITELIST_DISABLED = new Whitelist();
+
+ /**
+ * Creates an empty, disabled Whitelist.
+ */
+ public Whitelist() {
+ this.patterns = Collections.emptyList();
+ this.statusCode = -1;
+ this.enabled = false;
+ }
+
+ /**
+ * Creates an empty, enabled whitelist with the specified response code.
+ *
+ * @param statusCode the response code that the (enabled) Whitelist will return for all URLs.
+ */
+ public Whitelist(int statusCode) {
+ this.patterns = Collections.emptyList();
+ this.statusCode = statusCode;
+ this.enabled = true;
+ }
+
+ /**
+ * @deprecated use {@link #Whitelist(java.util.Collection, int)}
+ */
+ @Deprecated
+ public Whitelist(String[] patterns, int statusCode) {
+ this(patterns == null ? null : Arrays.asList(patterns), statusCode);
+ }
+
+ /**
+ * Creates a whitelist for the specified patterns, returning the given statusCode when a URL does not match one of the patterns.
+ * A null or empty collection will result in an empty whitelist.
+ *
+ * @param patterns URL-matching regular expression patterns to whitelist
+ * @param statusCode the HTTP status code to return when a request URL matches a whitelist pattern
+ */
+ public Whitelist(Collection patterns, int statusCode) {
+ if (patterns == null || patterns.isEmpty()) {
+ this.patterns = Collections.emptyList();
+ } else {
+ ImmutableList.Builder builder = ImmutableList.builder();
+ for (String pattern : patterns) {
+ builder.add(Pattern.compile(pattern));
+ }
+
+ this.patterns = builder.build();
+ }
+
+ this.statusCode = statusCode;
+
+ this.enabled = true;
+ }
+
+ /**
+ * @return true if this whitelist is enabled, otherwise false
+ */
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ /**
+ * @return regular expression patterns describing the URLs that should be whitelisted, or an empty collection if the whitelist is disabled
+ */
+ public Collection getPatterns() {
+ return this.patterns;
+ }
+
+ /**
+ * @return HTTP status code returned by the whitelist, or -1 if the whitelist is disabled
+ */
+ public int getStatusCode() {
+ return statusCode;
+ }
+
+ /**
+ * @deprecated use {@link #getStatusCode()}
+ */
+ @Deprecated
+ public int getResponseCode() {
+ return getStatusCode();
+ }
+
+ /**
+ * Returns true if the specified URL matches a whitelisted URL regular expression. If the whitelist is disabled, this
+ * method always returns false.
+ *
+ * @param url URL to match against the whitelist
+ * @return true if the whitelist is enabled and the URL matched an entry in the whitelist, otherwise false
+ */
+ public boolean matches(String url) {
+ if (!enabled) {
+ return false;
+ }
+
+ for (Pattern pattern : getPatterns()) {
+ Matcher matcher = pattern.matcher(url);
+ if (matcher.matches()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/auth/AuthType.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/auth/AuthType.java
new file mode 100644
index 000000000..b5cf93738
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/auth/AuthType.java
@@ -0,0 +1,10 @@
+package net.lightbody.bmp.proxy.auth;
+
+/**
+ * Authentication types support by BrowserMobProxy.
+ */
+public enum AuthType {
+ BASIC,
+ // TODO: determine if we can actually do NTLM authentication
+ NTLM
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AbstractHostNameRemapper.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AbstractHostNameRemapper.java
new file mode 100644
index 000000000..253b5f45d
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AbstractHostNameRemapper.java
@@ -0,0 +1,135 @@
+package net.lightbody.bmp.proxy.dns;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Base class that provides host name remapping capabilities for AdvancedHostResolvers. Subclasses must implement {@link #resolveRemapped(String)}
+ * instead of {@link net.lightbody.bmp.proxy.dns.HostResolver#resolve(String)}, which takes the remapped host as the input parameter.
+ */
+public abstract class AbstractHostNameRemapper implements AdvancedHostResolver {
+ /**
+ * Host name remappings, maintained as a reference to an ImmutableMap. The ImmutableMap type is specified explicitly because ImmutableMap
+ * guarantees the iteration order of the map's entries. Specifying ImmutableMap also makes clear that the underlying map will never change,
+ * and that any modifications to the host name remappings will result in an entirely new map.
+ *
+ * The current implementation does not actually use any of the special features of AtomicReference, but it does rely on synchronizing on
+ * the AtomicReference when performing write operations. It could be replaced by a volatile reference to a Map and separate lock object.
+ */
+ private final AtomicReference> remappedHostNames = new AtomicReference<>(ImmutableMap.of());
+
+ @Override
+ public void remapHosts(Map hostRemappings) {
+ synchronized (remappedHostNames) {
+ ImmutableMap newRemappings = ImmutableMap.copyOf(hostRemappings);
+
+ remappedHostNames.set(newRemappings);
+ }
+ }
+
+ @Override
+ public void remapHost(String originalHost, String remappedHost) {
+ synchronized (remappedHostNames) {
+ Map currentHostRemappings = remappedHostNames.get();
+
+ // use a LinkedHashMap to build the new remapping, to avoid duplicate key issues if the originalHost is already in the map
+ Map builderMap = Maps.newLinkedHashMap(currentHostRemappings);
+ builderMap.remove(originalHost);
+ builderMap.put(originalHost, remappedHost);
+
+ ImmutableMap newRemappings = ImmutableMap.copyOf(builderMap);
+
+ remappedHostNames.set(newRemappings);
+ }
+ }
+
+ @Override
+ public void removeHostRemapping(String originalHost) {
+ synchronized (remappedHostNames) {
+ Map currentHostRemappings = remappedHostNames.get();
+ if (currentHostRemappings.containsKey(originalHost)) {
+ // use a LinkedHashMap to build the new remapping, to take advantage of the remove() method
+ Map builderMap = Maps.newLinkedHashMap(currentHostRemappings);
+ builderMap.remove(originalHost);
+
+ ImmutableMap newRemappings = ImmutableMap.copyOf(builderMap);
+
+ remappedHostNames.set(newRemappings);
+ }
+ }
+ }
+
+ @Override
+ public void clearHostRemappings() {
+ synchronized (remappedHostNames) {
+ remappedHostNames.set(ImmutableMap.of());
+ }
+ }
+
+ @Override
+ public Map getHostRemappings() {
+ return remappedHostNames.get();
+ }
+
+ @Override
+ public Collection getOriginalHostnames(String remappedHost) {
+ //TODO: implement this using a reverse mapping multimap that is guarded by the same lock as remappedHostNames, since this method will likely be called
+ // very often when forging certificates
+ List originalHostnames = new ArrayList<>();
+
+ Map currentRemappings = remappedHostNames.get();
+ for (Map.Entry entry : currentRemappings.entrySet()) {
+ if (entry.getValue().equals(remappedHost)) {
+ originalHostnames.add(entry.getKey());
+ }
+ }
+
+ return originalHostnames;
+ }
+
+ /**
+ * Applies this class's host name remappings to the specified original host, returning the remapped host name (if any), or the originalHost
+ * if there is no remapped host name.
+ *
+ * @param originalHost original host name to resolve
+ * @return a remapped host, or the original host if no mapping exists
+ */
+ public String applyRemapping(String originalHost) {
+ String remappedHost = remappedHostNames.get().get(originalHost);
+
+ if (remappedHost != null) {
+ return remappedHost;
+ } else {
+ return originalHost;
+ }
+ }
+
+ /**
+ * Resolves the specified remapped host. Subclasses should provide resolution by implementing this method, rather than overriding
+ * {@link net.lightbody.bmp.proxy.dns.HostResolver#resolve(String)}.
+ *
+ * @param remappedHost remapped hostname to resolve
+ * @return resolved InetAddresses, or an empty list if no addresses were found
+ */
+ public abstract Collection resolveRemapped(String remappedHost);
+
+ /**
+ * Retrieves the remapped hostname and resolves it using {@link #resolveRemapped(String)}.
+ *
+ * @param originalHost original hostname to resolve
+ * @return InetAddresses resolved from the remapped hostname, or an empty list if no addresses were found
+ */
+ @Override
+ public Collection resolve(String originalHost) {
+ String remappedHost = applyRemapping(originalHost);
+
+ return resolveRemapped(remappedHost);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolver.java
new file mode 100644
index 000000000..5773e36c3
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolver.java
@@ -0,0 +1,90 @@
+package net.lightbody.bmp.proxy.dns;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This interface defines the "core" DNS-manipulation functionality that BrowserMob Proxy supports, in addition to the basic name resolution
+ * capability defined in {@link net.lightbody.bmp.proxy.dns.HostResolver}. AdvancedHostResolvers should apply any remappings before attempting
+ * to resolve the hostname in the {@link HostResolver#resolve(String)} method.
+ */
+public interface AdvancedHostResolver extends HostResolver {
+ /**
+ * Replaces the host remappings in the existing list of remappings (if any) with the specified remappings. The remappings will be
+ * applied in the order specified by the Map's iterator.
+ *
+ * Note: The original hostnames must exactly match the requested hostname. It is not a domain or regular expression match.
+ *
+ * @param hostRemappings Map of {@code }
+ */
+ void remapHosts(Map hostRemappings);
+
+ /**
+ * Remaps an individual host. If there are any existing remappings, the new remapping will be applied last, after all existing
+ * remappings are applied. If there is already a remapping for the specified originalHost, it will be removed before
+ * the new remapping is added to the end of the host remapping list (and will therefore be the last remapping applied).
+ *
+ * @param originalHost Original host to remap. Must exactly match the requested hostname (not a domain or regular expression match).
+ * @param remappedHost hostname that will replace originalHost
+ */
+ void remapHost(String originalHost, String remappedHost);
+
+ /**
+ * Removes the specified host remapping. If the remapping does not exist, this method has no effect.
+ *
+ * @param originalHost currently-remapped hostname
+ */
+ void removeHostRemapping(String originalHost);
+
+ /**
+ * Removes all hostname remappings.
+ */
+ void clearHostRemappings();
+
+ /**
+ * Returns all host remappings in effect. Iterating over the returned Map is guaranteed to return remappings in the order in which the
+ * remappings are actually applied.
+ *
+ * @return Map of {@code }
+ */
+ Map getHostRemappings();
+
+ /**
+ * Returns the original address or addresses that are remapped to the specified remappedHost. Iterating over the returned Collection is
+ * guaranteed to return original mappings in the order in which the remappings are applied.
+ *
+ * @param remappedHost remapped hostname
+ * @return original hostnames that are remapped to the specified remappedHost, or an empty Collection if no remapping is defined to the remappedHost
+ */
+ Collection getOriginalHostnames(String remappedHost);
+
+ /**
+ * Clears both the positive (successful DNS lookups) and negative (failed DNS lookups) cache.
+ */
+ void clearDNSCache();
+
+ /**
+ * Sets the positive (successful DNS lookup) timeout when making DNS lookups.
+ *
+ * Note: The timeUnit parameter does not guarantee the specified precision; implementations may need to reduce precision, depending on the underlying
+ * DNS implementation. For example, the Oracle JVM's DNS cache only supports timeouts in whole seconds, so specifying a timeout of 1200ms will result
+ * in a timeout of 1 second.
+ *
+ * @param timeout maximum lookup time
+ * @param timeUnit units of the timeout value
+ */
+ void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit);
+
+ /**
+ * Sets the negative (failed DNS lookup) timeout when making DNS lookups.
+ *
+ * Note: The timeUnit parameter does not guarantee the specified precision; implementations may need to reduce precision, depending on the underlying
+ * DNS implementation. For example, the Oracle JVM's DNS cache only supports timeouts in whole seconds, so specifying a timeout of 1200ms will result
+ * in a timeout of 1 second.
+ *
+ * @param timeout maximum lookup time
+ * @param timeUnit units of the timeout value
+ */
+ void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/BasicHostResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/BasicHostResolver.java
new file mode 100644
index 000000000..b07bd6d7a
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/BasicHostResolver.java
@@ -0,0 +1,57 @@
+package net.lightbody.bmp.proxy.dns;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An {@link AdvancedHostResolver} that throws UnsupportedOperationException on all methods except {@link HostResolver#resolve(String)}.
+ * Use this class to supply a {@link HostResolver} to {@link net.lightbody.bmp.BrowserMobProxy#setHostNameResolver(AdvancedHostResolver)}
+ * if you do not need {@link AdvancedHostResolver} functionality.
+ */
+public abstract class BasicHostResolver implements AdvancedHostResolver {
+ @Override
+ public void remapHosts(Map hostRemappings) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void remapHost(String originalHost, String remappedHost) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void removeHostRemapping(String originalHost) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void clearHostRemappings() {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public Map getHostRemappings() {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public Collection getOriginalHostnames(String remappedHost) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void clearDNSCache() {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+
+ @Override
+ public void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ throw new UnsupportedOperationException(new Throwable().getStackTrace()[0].getMethodName() + " is not supported by this host resolver (" + this.getClass().getName() + ")");
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/ChainedHostResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/ChainedHostResolver.java
new file mode 100644
index 000000000..dd5ae002d
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/ChainedHostResolver.java
@@ -0,0 +1,198 @@
+package net.lightbody.bmp.proxy.dns;
+
+import com.google.common.collect.ImmutableList;
+
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * An {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} that applies the AdvancedHostResolver methods to multiple implementations. Methods
+ * are applied to the resolvers in the order specified when the ChainedHostResolver is constructed. AdvancedHostResolver methods that modify the
+ * resolver are guaranteed to complete atomically over all resolvers. For example, if one thread makes a call to
+ * {@link #resolve(String)} while another thread is remapping hosts using
+ * {@link #remapHost(String, String)}, the call to {@link #resolve(String)} is guaranteed to
+ * apply the newly-remapped hosts to all resolvers managed by this ChainedHostResolver, or to no resolvers, but the call to
+ * {@link #resolve(String)} will never result in the host name remappings applied only to "some" of the chained resolvers.
+ *
+ * For getter methods (all read-only methods except {@link #resolve(String)}), the ChainedHostResolver returns results from the first chained resolver.
+ *
+ * The atomic write methods specified by AdvancedHostResolver are:
+ *
+ */
+public class ChainedHostResolver implements AdvancedHostResolver {
+ private final List extends AdvancedHostResolver> resolvers;
+
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /**
+ * Creates a ChainedHostResolver that applies {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} methods to the specified resolvers
+ * in the order specified by the collection's iterator.
+ *
+ * @param resolvers resolvers to invoke, in the order specified by the collection's iterator
+ */
+ public ChainedHostResolver(Collection extends AdvancedHostResolver> resolvers) {
+ if (resolvers == null) {
+ this.resolvers = Collections.emptyList();
+ } else {
+ this.resolvers = ImmutableList.copyOf(resolvers);
+ }
+ }
+
+ /**
+ * Returns the resolvers used by this ChainedHostResolver. The iterator of the collection is guaranteed to return the resolvers in the order
+ * in which they are queried.
+ *
+ * @return resolvers used by this ChainedHostResolver
+ */
+ public Collection extends AdvancedHostResolver> getResolvers() {
+ return ImmutableList.copyOf(resolvers);
+ }
+
+ @Override
+ public void remapHosts(Map hostRemappings) {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.remapHosts(hostRemappings);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void remapHost(String originalHost, String remappedHost) {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.remapHost(originalHost, remappedHost);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void removeHostRemapping(String originalHost) {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.removeHostRemapping(originalHost);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void clearHostRemappings() {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.clearHostRemappings();
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Map getHostRemappings() {
+ readLock.lock();
+ try {
+ if (resolvers.isEmpty()) {
+ return Collections.emptyMap();
+ } else {
+ return resolvers.get(0).getHostRemappings();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Collection getOriginalHostnames(String remappedHost) {
+ readLock.lock();
+ try {
+ if (resolvers.isEmpty()) {
+ return Collections.emptyList();
+ } else {
+ return resolvers.get(0).getOriginalHostnames(remappedHost);
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void clearDNSCache() {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.clearDNSCache();
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.setPositiveDNSCacheTimeout(timeout, timeUnit);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ writeLock.lock();
+ try {
+ for (AdvancedHostResolver resolver : resolvers) {
+ resolver.setNegativeDNSCacheTimeout(timeout, timeUnit);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Collection resolve(String host) {
+ readLock.lock();
+ try {
+ // attempt to resolve the host using all resolvers. returns the results from the first successful resolution.
+ for (AdvancedHostResolver resolver : resolvers) {
+ Collection results = resolver.resolve(host);
+ if (!results.isEmpty()) {
+ return results;
+ }
+ }
+
+ // no resolvers returned results
+ return Collections.emptyList();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DelegatingHostResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DelegatingHostResolver.java
new file mode 100644
index 000000000..7f6bdf269
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DelegatingHostResolver.java
@@ -0,0 +1,45 @@
+package net.lightbody.bmp.proxy.dns;
+
+import com.google.common.collect.Iterables;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Collection;
+
+/**
+ * A LittleProxy HostResolver that delegates to the specified {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} instance. This class
+ * serves as a bridge between {@link AdvancedHostResolver} and {@link org.littleshoot.proxy.HostResolver}.
+*/
+public class DelegatingHostResolver implements org.littleshoot.proxy.HostResolver {
+ private volatile AdvancedHostResolver resolver;
+
+ /**
+ * Creates a new resolver that will delegate to the specified resolver.
+ *
+ * @param resolver HostResolver to delegate to
+ */
+ public DelegatingHostResolver(AdvancedHostResolver resolver) {
+ this.resolver = resolver;
+ }
+
+ public AdvancedHostResolver getResolver() {
+ return resolver;
+ }
+
+ public void setResolver(AdvancedHostResolver resolver) {
+ this.resolver = resolver;
+ }
+
+ @Override
+ public InetSocketAddress resolve(String host, int port) throws UnknownHostException {
+ Collection resolvedAddresses = resolver.resolve(host);
+ if (!resolvedAddresses.isEmpty()) {
+ InetAddress resolvedAddress = Iterables.get(resolvedAddresses, 0);
+ return new InetSocketAddress(resolvedAddress, port);
+ }
+
+ // no address found by the resolver
+ throw new UnknownHostException(host);
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DnsJavaResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DnsJavaResolver.java
new file mode 100644
index 000000000..432032ed7
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/DnsJavaResolver.java
@@ -0,0 +1,144 @@
+package net.lightbody.bmp.proxy.dns;
+
+import com.google.common.net.InetAddresses;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xbill.DNS.AAAARecord;
+import org.xbill.DNS.ARecord;
+import org.xbill.DNS.Cache;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.Lookup;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.TextParseException;
+import org.xbill.DNS.Type;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} that uses dnsjava to perform DNS lookups. This implementation provides full
+ * cache manipulation capabilities.
+ *
+ * @deprecated The dnsjava resolver has been deprecated in favor of the standard JVM resolver and will be removed in BMP >2.1.
+ */
+public class DnsJavaResolver extends AbstractHostNameRemapper implements AdvancedHostResolver {
+ private static final Logger log = LoggerFactory.getLogger(DnsJavaResolver.class);
+
+ /**
+ * DNS cache used for dnsjava lookups.
+ */
+ private final Cache cache = new Cache();
+
+ /**
+ * Maximum number of times to retry a DNS lookup due to a failure to connect to the DNS server.
+ */
+ private static final int DNS_NETWORK_FAILURE_RETRY_COUNT = 5;
+
+ @Override
+ public void clearDNSCache() {
+ cache.clearCache();
+ }
+
+ @Override
+ public void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ cache.setMaxCache((int) TimeUnit.SECONDS.convert(timeout, timeUnit));
+ }
+
+ @Override
+ public void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ cache.setMaxNCache((int) TimeUnit.SECONDS.convert(timeout, timeUnit));
+ }
+
+ @Override
+ public Collection resolveRemapped(String remappedHost) {
+ // special case for IP literals: return the InetAddress without doing a dnsjava lookup. dnsjava seems to handle ipv4 literals
+ // reasonably well, but does not handle ipv6 literals (with or without [] brackets) correctly.
+ // note this does not work properly for ipv6 literals with a scope identifier, which is a known issue for InetAddresses.isInetAddress().
+ // (dnsjava also handles the situation incorrectly)
+ if (InetAddresses.isInetAddress(remappedHost)) {
+ return Collections.singletonList(InetAddresses.forString(remappedHost));
+ }
+
+ // retrieve IPv4 addresses, then retrieve IPv6 addresses only if no IPv4 addresses are found. the current implementation always uses the
+ // first returned address, so there is no need to look for IPv6 addresses if an IPv4 address is found.
+ Collection ipv4addresses = resolveHostByType(remappedHost, Type.A);
+
+ if (!ipv4addresses.isEmpty()) {
+ return ipv4addresses;
+ } else {
+ return resolveHostByType(remappedHost, Type.AAAA);
+ }
+ }
+
+ /**
+ * Resolves the specified host using dnsjava, retrieving addresses of the specified type.
+ *
+ * @param host hostname to resolve
+ * @param type one of {@link org.xbill.DNS.Type}, typically {@link org.xbill.DNS.Type#A} (IPv4) or {@link org.xbill.DNS.Type#AAAA} (IPv6).
+ * @return resolved addresses, or an empty collection if no addresses could be resolved
+ */
+ protected Collection resolveHostByType(String host, int type) {
+ Lookup lookup;
+ try {
+ lookup = new Lookup(host, type, DClass.IN);
+ } catch (TextParseException e) {
+ return Collections.emptyList();
+ }
+
+ lookup.setCache(cache);
+
+ // we set the retry count to -1 because we want the first execution not be counted as a retry.
+ int retryCount = -1;
+ Record[] records;
+
+ // we iterate while the status is TRY_AGAIN and DNS_NETWORK_FAILURE_RETRY_COUNT is not exceeded
+ do {
+ records = lookup.run();
+ retryCount++;
+ } while (lookup.getResult() == Lookup.TRY_AGAIN && retryCount < DNS_NETWORK_FAILURE_RETRY_COUNT);
+
+ if (records == null) {
+ // no records found, so could not resolve host
+ return Collections.emptyList();
+ }
+
+ // convert the records we found into IPv4/IPv6 InetAddress objects
+ List addrList = new ArrayList(records.length);
+
+ // the InetAddresses returned by dnsjava include the trailing dot, e.g. "www.google.com." -- use the passed-in (or remapped) host value instead
+ for (Record record : records) {
+ if (record instanceof ARecord) {
+ ARecord ipv4Record = (ARecord) record;
+
+ try {
+ InetAddress resolvedAddress = InetAddress.getByAddress(host, ipv4Record.getAddress().getAddress());
+
+ addrList.add(resolvedAddress);
+ } catch (UnknownHostException e) {
+ // this should never happen, unless there is a bug in dnsjava
+ log.warn("dnsjava resolver returned an invalid InetAddress for host: " + host, e);
+ continue;
+ }
+ } else if (record instanceof AAAARecord) {
+ AAAARecord ipv6Record = (AAAARecord) record;
+
+ try {
+ InetAddress resolvedAddress = InetAddress.getByAddress(host, ipv6Record.getAddress().getAddress());
+
+ addrList.add(resolvedAddress);
+ } catch (UnknownHostException e) {
+ // this should never happen, unless there is a bug in dnsjava
+ log.warn("dnsjava resolver returned an invalid InetAddress for host: " + host, e);
+ continue;
+ }
+ }
+ }
+
+ return addrList;
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/HostResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/HostResolver.java
new file mode 100644
index 000000000..c6d380715
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/HostResolver.java
@@ -0,0 +1,20 @@
+package net.lightbody.bmp.proxy.dns;
+
+import java.net.InetAddress;
+import java.util.Collection;
+
+/**
+ * Defines the basic functionality that {@link net.lightbody.bmp.BrowserMobProxy} implementations require when resolving hostnames.
+ */
+public interface HostResolver {
+ /**
+ * Resolves a hostname to one or more IP addresses. The iterator over the returned Collection is recommended to reflect the ordering
+ * returned by the underlying name lookup service. For example, if a DNS server returns three IP addresses, 1.1.1.1, 2.2.2.2, and
+ * 3.3.3.3, corresponding to www.somehost.com, the returned Collection iterator is recommended to iterate in
+ * the order [1.1.1.1, 2.2.2.2, 3.3.3.3].
+ *
+ * @param host host to resolve
+ * @return resolved InetAddresses, or an empty collection if no addresses were found
+ */
+ Collection resolve(String host);
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeCacheManipulatingResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeCacheManipulatingResolver.java
new file mode 100644
index 000000000..d2734dba1
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeCacheManipulatingResolver.java
@@ -0,0 +1,88 @@
+package net.lightbody.bmp.proxy.dns;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.util.LinkedHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} that provides native JVM lookup using {@link net.lightbody.bmp.proxy.dns.NativeResolver}
+ * but also implements DNS cache manipulation functionality.
+ *
+ * Important note: The Oracle JVM does not provide any public facility to manipulate the JVM's DNS cache. This class uses reflection to forcibly
+ * manipulate the cache, which includes access to private class members that are not part of the published Java specification. As such, this
+ * implementation is brittle and may break in a future Java release, or may not work on non-Oracle JVMs. If this implementation cannot
+ * perform any of its operations due to a failure to find or set the relevant field using reflection, it will log a warning but will not
+ * throw an exception. You are using this class at your own risk! JVM cache manipulation does not work on Windows -- this class will behave exactly
+ * the same as {@link net.lightbody.bmp.proxy.dns.NativeResolver} on that platform.
+ */
+public class NativeCacheManipulatingResolver extends NativeResolver {
+ private static final Logger log = LoggerFactory.getLogger(NativeCacheManipulatingResolver.class);
+
+ @Override
+ public void clearDNSCache() {
+ // clear the DNS cache but replacing the LinkedHashMaps that contain the positive and negative caches on the
+ // private static InetAddress.Cache inner class with new, empty maps
+ try {
+ Field positiveCacheField = InetAddress.class.getDeclaredField("addressCache");
+ positiveCacheField.setAccessible(true);
+ Object positiveCacheInstance = positiveCacheField.get(null);
+
+ Field negativeCacheField = InetAddress.class.getDeclaredField("negativeCache");
+ negativeCacheField.setAccessible(true);
+ Object negativeCacheInstance = positiveCacheField.get(null);
+
+ Class> cacheClass = Class.forName("java.net.InetAddress$Cache");
+ Field cacheField = cacheClass.getDeclaredField("cache");
+ cacheField.setAccessible(true);
+
+ cacheField.set(positiveCacheInstance, new LinkedHashMap());
+ cacheField.set(negativeCacheInstance, new LinkedHashMap());
+ } catch (ClassNotFoundException | IllegalAccessException | NoSuchFieldException e) {
+ log.warn("Unable to clear native JVM DNS cache", e);
+ }
+ }
+
+ @Override
+ public void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ try {
+ Class> inetAddressCachePolicyClass = Class.forName("sun.net.InetAddressCachePolicy");
+
+ Field positiveCacheTimeoutSeconds = inetAddressCachePolicyClass.getDeclaredField("cachePolicy");
+ positiveCacheTimeoutSeconds.setAccessible(true);
+
+ if (timeout < 0) {
+ positiveCacheTimeoutSeconds.setInt(null, -1);
+ java.security.Security.setProperty("networkaddress.cache.ttl", "-1");
+ } else {
+ positiveCacheTimeoutSeconds.setInt(null, (int) TimeUnit.SECONDS.convert(timeout, timeUnit));
+ java.security.Security.setProperty("networkaddress.cache.ttl", Long.toString(TimeUnit.SECONDS.convert(timeout, timeUnit)));
+ }
+ } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException e) {
+ log.warn("Unable to modify native JVM DNS cache timeouts", e);
+ }
+ }
+
+ @Override
+ public void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ try {
+ Class> inetAddressCachePolicyClass = Class.forName("sun.net.InetAddressCachePolicy");
+
+ Field negativeCacheTimeoutSeconds = inetAddressCachePolicyClass.getDeclaredField("negativeCachePolicy");
+ negativeCacheTimeoutSeconds.setAccessible(true);
+
+ if (timeout < 0) {
+ negativeCacheTimeoutSeconds.setInt(null, -1);
+ java.security.Security.setProperty("networkaddress.cache.negative.ttl", "-1");
+ } else {
+ negativeCacheTimeoutSeconds.setInt(null, (int) TimeUnit.SECONDS.convert(timeout, timeUnit));
+ java.security.Security.setProperty("networkaddress.cache.negative.ttl", Long.toString(TimeUnit.SECONDS.convert(timeout, timeUnit)));
+ }
+ } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException e) {
+ log.warn("Unable to modify native JVM DNS cache timeouts", e);
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeResolver.java b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeResolver.java
new file mode 100644
index 000000000..c71c61495
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/proxy/dns/NativeResolver.java
@@ -0,0 +1,48 @@
+package net.lightbody.bmp.proxy.dns;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver} that provides native JVM lookup using {@link java.net.InetAddress}.
+ * This implementation does not provide any cache manipulation. Attempting to manipulate the DNS cache will result in a DEBUG-level
+ * log statement and will not raise an exception. The {@link net.lightbody.bmp.proxy.dns.DnsJavaResolver} provides support for cache
+ * manipulation. If you absolutely need to manipulate the native JVM DNS cache, see
+ * {@link net.lightbody.bmp.proxy.dns.NativeCacheManipulatingResolver} for details.
+ */
+public class NativeResolver extends AbstractHostNameRemapper implements AdvancedHostResolver {
+ private static final Logger log = LoggerFactory.getLogger(NativeResolver.class);
+
+ @Override
+ public void clearDNSCache() {
+ log.debug("Cannot clear native JVM DNS Cache using this Resolver");
+ }
+
+ @Override
+ public void setPositiveDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ log.debug("Cannot change native JVM DNS cache timeout using this Resolver");
+ }
+
+ @Override
+ public void setNegativeDNSCacheTimeout(int timeout, TimeUnit timeUnit) {
+ log.debug("Cannot change native JVM DNS cache timeout using this Resolver");
+ }
+
+ @Override
+ public Collection resolveRemapped(String remappedHost) {
+ try {
+ Collection addresses = Arrays.asList(InetAddress.getAllByName(remappedHost));
+
+ return addresses;
+ } catch (UnknownHostException e) {
+ return Collections.emptyList();
+ }
+ }
+}
diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java b/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java
new file mode 100644
index 000000000..98172810f
--- /dev/null
+++ b/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java
@@ -0,0 +1,300 @@
+package net.lightbody.bmp.util;
+
+import com.google.common.io.BaseEncoding;
+import com.google.common.net.HostAndPort;
+import com.google.common.net.MediaType;
+import io.netty.buffer.ByteBuf;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import net.lightbody.bmp.exception.DecompressionException;
+import net.lightbody.bmp.exception.UnsupportedCharsetException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Utility class with static methods for processing HTTP requests and responses.
+ */
+public class BrowserMobHttpUtil {
+ private static final Logger log = LoggerFactory.getLogger(BrowserMobHttpUtil.class);
+
+ /**
+ * Default MIME content type if no Content-Type header is present. According to the HTTP 1.1 spec, section 7.2.1:
+ *
+ * Any HTTP/1.1 message containing an entity-body SHOULD include a Content-Type header field defining the media
+ * type of that body. If and only if the media type is not given by a Content-Type field, the recipient MAY
+ * attempt to guess the media type via inspection of its content and/or the name extension(s) of the URI used to
+ * identify the resource. If the media type remains unknown, the recipient SHOULD treat it as
+ * type "application/octet-stream".
+ *
+ */
+ public static final String UNKNOWN_CONTENT_TYPE = "application/octet-stream";
+
+ /**
+ * The default charset when the Content-Type header does not specify a charset. According to RFC 7231 Appendix B:
+ *
+ * The default charset of ISO-8859-1 for text media types has been
+ * removed; the default is now whatever the media type definition says.
+ * Likewise, special treatment of ISO-8859-1 has been removed from the
+ * Accept-Charset header field.
+ *
+ *
+ * Technically, we would have to determine the charset on a per-content-type basis, but generally speaking, UTF-8 is a
+ * pretty safe default. (NOTE: In the previous HTTP/1.1 spec, section 3.7.1, the default charset was defined as ISO-8859-1.)
+ */
+ public static final Charset DEFAULT_HTTP_CHARSET = StandardCharsets.UTF_8;
+
+ /**
+ * Buffer size when decompressing content.
+ */
+ public static final int DECOMPRESS_BUFFER_SIZE = 16192;
+
+ /**
+ * Returns the size of the headers, including the 2 CRLFs at the end of the header block.
+ *
+ * @param headers headers to size
+ * @return length of the headers, in bytes
+ */
+ public static long getHeaderSize(HttpHeaders headers) {
+ long headersSize = 0;
+ for (Map.Entry header : headers.entries()) {
+ // +2 for ': ', +2 for new line
+ headersSize += header.getKey().length() + header.getValue().length() + 4;
+ }
+ return headersSize;
+ }
+
+ /**
+ * Decompresses the gzipped byte stream.
+ *
+ * @param fullMessage gzipped byte stream to decomress
+ * @return decompressed bytes
+ * @throws DecompressionException thrown if the fullMessage cannot be read or decompressed for any reason
+ */
+ public static byte[] decompressContents(byte[] fullMessage) throws DecompressionException {
+ InflaterInputStream gzipReader = null;
+ ByteArrayOutputStream uncompressed;
+ try {
+ gzipReader = new GZIPInputStream(new ByteArrayInputStream(fullMessage));
+
+ uncompressed = new ByteArrayOutputStream(fullMessage.length);
+
+ byte[] decompressBuffer = new byte[DECOMPRESS_BUFFER_SIZE];
+ int bytesRead;
+ while ((bytesRead = gzipReader.read(decompressBuffer)) > -1) {
+ uncompressed.write(decompressBuffer, 0, bytesRead);
+ }
+
+ fullMessage = uncompressed.toByteArray();
+ } catch (IOException e) {
+ throw new DecompressionException("Unable to decompress response", e);
+ } finally {
+ try {
+ if (gzipReader != null) {
+ gzipReader.close();
+ }
+ } catch (IOException e) {
+ log.warn("Unable to close gzip stream", e);
+ }
+ }
+ return fullMessage;
+ }
+
+ /**
+ * Returns true if the content type string indicates textual content. Currently these are any Content-Types that start with one of the
+ * following:
+ *