From 0010cc2eab3b3dac80a909112f2d9ad820f16def Mon Sep 17 00:00:00 2001 From: Valery Yatsynovich Date: Wed, 7 Jun 2017 09:41:42 +0300 Subject: [PATCH 1/4] Replace usage of deprecated c.g.c.n.HostAndPort.getHostText with HostAndPort.getHost com.google.common.net.HostAndPort.getHostText is removed in Guava 22.0, so usage of deprecated API causes runtime errors --- .../net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java | 2 +- .../net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java | 2 +- .../main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java | 2 +- .../net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java | 2 +- mitm/src/main/java/net/lightbody/bmp/util/HttpUtil.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java index 7b6610f5b..e727f2b00 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpsAwareFiltersAdapter.java @@ -102,7 +102,7 @@ public String getHost(HttpRequest modifiedRequest) { String serverHost; if (isHttps()) { HostAndPort hostAndPort = HostAndPort.fromString(getHttpsRequestHostAndPort()); - serverHost = hostAndPort.getHostText(); + serverHost = hostAndPort.getHost(); } else { serverHost = HttpUtil.getHostFromRequest(modifiedRequest); } diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java index 0e8335898..0cc4dcaa5 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/ResolvedHostnameCacheFilter.java @@ -53,7 +53,7 @@ public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocke if (resolvedAddress != null) { // place the resolved host into the hostname cache, so subsequent requests will be able to identify the IP address HostAndPort parsedHostAndPort = HostAndPort.fromString(serverHostAndPort); - String host = parsedHostAndPort.getHostText(); + String host = parsedHostAndPort.getHost(); if (host != null && !host.isEmpty()) { resolvedAddresses.put(host, resolvedAddress.getHostAddress()); diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java b/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java index 5e00891be..98172810f 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobHttpUtil.java @@ -276,7 +276,7 @@ public static String removeMatchingPort(String hostWithPort, int portNumber) { HostAndPort parsedHostAndPort = HostAndPort.fromString(hostWithPort); if (parsedHostAndPort.hasPort() && parsedHostAndPort.getPort() == portNumber) { // HostAndPort.getHostText() strips brackets from ipv6 addresses, so reparse using fromHost - return HostAndPort.fromHost(parsedHostAndPort.getHostText()).toString(); + return HostAndPort.fromHost(parsedHostAndPort.getHost()).toString(); } else { return hostWithPort; } diff --git a/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java b/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java index b2dca620d..b2c0ad061 100644 --- a/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java +++ b/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java @@ -504,7 +504,7 @@ public void setOptions(Map options) { log.warn("Chained proxy support through setOptions is deprecated. Use setUpstreamProxy() to enable chained proxy support."); HostAndPort hostAndPort = HostAndPort.fromString(httpProxy); - this.setChainedProxy(new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPortOrDefault(80))); + this.setChainedProxy(new InetSocketAddress(hostAndPort.getHost(), hostAndPort.getPortOrDefault(80))); } else { if (errorOnUnsupportedOperation) { throw new UnsupportedOperationException("The LittleProxy-based implementation of BrowserMob Proxy does not support the setOptions method. Use the methods defined in the BrowserMobProxy interface to set connection parameters."); diff --git a/mitm/src/main/java/net/lightbody/bmp/util/HttpUtil.java b/mitm/src/main/java/net/lightbody/bmp/util/HttpUtil.java index aaf45abbd..2cc2a2158 100644 --- a/mitm/src/main/java/net/lightbody/bmp/util/HttpUtil.java +++ b/mitm/src/main/java/net/lightbody/bmp/util/HttpUtil.java @@ -114,7 +114,7 @@ private static String parseHostHeader(HttpRequest httpRequest, boolean includePo return hostAndPort; } else { HostAndPort parsedHostAndPort = HostAndPort.fromString(hostAndPort); - return parsedHostAndPort.getHostText(); + return parsedHostAndPort.getHost(); } } else { return null; From 86d1e425eb5b57fa3610ae589259f34503d900fe Mon Sep 17 00:00:00 2001 From: Valery Yatsynovich Date: Wed, 7 Jun 2017 09:45:54 +0300 Subject: [PATCH 2/4] Fix flaky tests: use nanoseconds instead of milliseconds Operations may take less than 1 millisecond at powerful machines, so rounding to milliseconds results to 0 time duration. Usage of nanoseconds allows to make tests more stable --- .../dns/AdvancedHostResolverCacheTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/browsermob-core/src/test/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolverCacheTest.java b/browsermob-core/src/test/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolverCacheTest.java index e4cdb648e..385df94b1 100644 --- a/browsermob-core/src/test/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolverCacheTest.java +++ b/browsermob-core/src/test/java/net/lightbody/bmp/proxy/dns/AdvancedHostResolverCacheTest.java @@ -71,7 +71,7 @@ public void testCanClearDNSCache() { resolver.resolve("www.msn.com"); long finish = System.nanoTime(); - assertNotEquals("Expected non-zero DNS lookup time for www.msn.com after clearing DNS cache", 0, TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS)); + assertNotEquals("Expected non-zero DNS lookup time for www.msn.com after clearing DNS cache", 0, finish - start); } @Test @@ -81,17 +81,17 @@ public void testCachedPositiveLookup() { resolver.resolve("news.bing.com"); long finish = System.nanoTime(); - long uncachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long uncachedLookupNs = finish - start; - assertNotEquals("Expected non-zero DNS lookup time for news.bing.com on first lookup", 0, uncachedLookupMs); + assertNotEquals("Expected non-zero DNS lookup time for news.bing.com on first lookup", 0, uncachedLookupNs); start = System.nanoTime(); resolver.resolve("news.bing.com"); finish = System.nanoTime(); - long cachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long cachedLookupNs = finish - start; - assertTrue("Expected extremely fast DNS lookup time for news.bing.com on second (cached) lookup. Uncached: " + uncachedLookupMs + "ms; cached: " + cachedLookupMs + "ms.", cachedLookupMs <= uncachedLookupMs / 2); + assertTrue("Expected extremely fast DNS lookup time for news.bing.com on second (cached) lookup. Uncached: " + uncachedLookupNs + "ns; cached: " + cachedLookupNs + "ns.", cachedLookupNs <= uncachedLookupNs / 2); } @Test @@ -100,17 +100,17 @@ public void testCachedNegativeLookup() { resolver.resolve("fake.notarealaddress"); long finish = System.nanoTime(); - long uncachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long uncachedLookupNs = finish - start; - assertNotEquals("Expected non-zero DNS lookup time for fake.notarealaddress on first lookup", 0, uncachedLookupMs); + assertNotEquals("Expected non-zero DNS lookup time for fake.notarealaddress on first lookup", 0, uncachedLookupNs); start = System.nanoTime(); resolver.resolve("fake.notarealaddress"); finish = System.nanoTime(); - long cachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long cachedLookupNs = finish - start; - assertTrue("Expected extremely fast DNS lookup time for fake.notarealaddress on second (cached) lookup. Uncached: " + uncachedLookupMs + "ms; cached: " + cachedLookupMs + "ms.", cachedLookupMs <= uncachedLookupMs / 2); + assertTrue("Expected extremely fast DNS lookup time for fake.notarealaddress on second (cached) lookup. Uncached: " + uncachedLookupNs + "ns; cached: " + cachedLookupNs + "ns.", cachedLookupNs <= uncachedLookupNs / 2); } @Test @@ -135,7 +135,7 @@ public void testSetPositiveCacheTtl() throws InterruptedException { assertNotNull("Collection of resolved addresses should never be null", addresses); assertNotEquals("Expected to find addresses for www.msn.com", 0, addresses.size()); - assertNotEquals("Expected non-zero DNS lookup time for www.msn.com after setting positive cache TTL", 0, TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS)); + assertNotEquals("Expected non-zero DNS lookup time for www.msn.com after setting positive cache TTL", 0, finish - start); } @Test @@ -163,7 +163,7 @@ public void testSetNegativeCacheTtl() throws InterruptedException { assertNotNull("Collection of resolved addresses should never be null", addresses); assertEquals("Expected to find no addresses for " + fakeAddress, 0, addresses.size()); - assertNotEquals("Expected non-zero DNS lookup time for " + fakeAddress + " after setting negative cache TTL", 0, TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS)); + assertNotEquals("Expected non-zero DNS lookup time for " + fakeAddress + " after setting negative cache TTL", 0, finish - start); } @Test @@ -185,12 +185,12 @@ public void testSetEternalNegativeCacheTtl() { addresses = resolver.resolve(fakeAddress); long finish = System.nanoTime(); - long cachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long cachedLookupNs = finish - start; assertNotNull("Collection of resolved addresses should never be null", addresses); assertEquals("Expected to find no addresses for " + fakeAddress, 0, addresses.size()); - assertTrue("Expected extremely fast DNS lookup time for " + fakeAddress + " after setting eternal negative cache TTL. Cached lookup time: " + cachedLookupMs + "ms.", cachedLookupMs <= 10); + assertTrue("Expected extremely fast DNS lookup time for " + fakeAddress + " after setting eternal negative cache TTL. Cached lookup time: " + cachedLookupNs + "ns.", cachedLookupNs <= TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS)); } @Test @@ -204,7 +204,7 @@ public void testSetEternalPositiveCacheTtl() { long one = System.nanoTime(); Collection addresses = resolver.resolve("www.msn.com"); long two = System.nanoTime(); - log.info("Time to resolve address without cache: {}ms", TimeUnit.MILLISECONDS.convert(two - one, TimeUnit.NANOSECONDS)); + log.info("Time to resolve address without cache: {}ns", two - one); // make sure there are addresses, since this is a *positive* TTL test assertNotNull("Collection of resolved addresses should never be null", addresses); @@ -214,13 +214,13 @@ public void testSetEternalPositiveCacheTtl() { addresses = resolver.resolve("www.msn.com"); long finish = System.nanoTime(); - long cachedLookupMs = TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS); + long cachedLookupNs = finish - start; - log.info("Time to resolve address with cache: {}ms", TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS)); + log.info("Time to resolve address with cache: {}ns", cachedLookupNs); assertNotNull("Collection of resolved addresses should never be null", addresses); assertNotEquals("Expected to find addresses for www.msn.com", 0, addresses.size()); - assertTrue("Expected extremely fast DNS lookup time for www.msn.com after setting eternal negative cache TTL. Cached lookup time: " + cachedLookupMs + "ms.", cachedLookupMs <= 10); + assertTrue("Expected extremely fast DNS lookup time for www.msn.com after setting eternal negative cache TTL. Cached lookup time: " + cachedLookupNs + "ns.", cachedLookupNs <= TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS)); } } From 494c491d62bab101e0d4a2bc4be3059ac85b51e0 Mon Sep 17 00:00:00 2001 From: Valery Yatsynovich Date: Wed, 7 Jun 2017 09:49:03 +0300 Subject: [PATCH 3/4] Use login.newrelic.com because rpm.newrelic.com requires authentication now --- .../src/test/java/net/lightbody/bmp/proxy/SslTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/SslTest.java b/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/SslTest.java index fff6058c3..fa336d05a 100644 --- a/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/SslTest.java +++ b/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/SslTest.java @@ -28,11 +28,11 @@ public void testSalesforce() throws Exception { @Test public void testNewRelic() throws Exception { // see https://github.com/webmetrics/browsermob-proxy/issues/105 - proxy.remapHost("foo.newrelic.com", "rpm.newrelic.com"); - proxy.remapHost("bar.newrelic.com", "rpm.newrelic.com"); + proxy.remapHost("foo.newrelic.com", "login.newrelic.com"); + proxy.remapHost("bar.newrelic.com", "login.newrelic.com"); get("https://foo.newrelic.com/"); get("https://bar.newrelic.com/"); - get("https://rpm.newrelic.com/"); + get("https://login.newrelic.com/"); } private void get(String url) throws IOException { From 91c163dcc3a20bc34e1b53d1ec3e82d11de562b7 Mon Sep 17 00:00:00 2001 From: Valery Yatsynovich Date: Wed, 7 Jun 2017 16:02:57 +0300 Subject: [PATCH 4/4] Replace msn.com with msn.com/en-us/ to avoid redirection Redirection leads to an extra page in Har and as result to failing tests. Change of URL allows to avoid redirection and tests work as expected. --- .../src/test/java/net/lightbody/bmp/proxy/HarTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/HarTest.java b/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/HarTest.java index a98949b78..3aed0306c 100644 --- a/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/HarTest.java +++ b/browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/HarTest.java @@ -5,7 +5,6 @@ import net.lightbody.bmp.core.har.HarEntry; import net.lightbody.bmp.core.har.HarLog; import net.lightbody.bmp.core.har.HarNameValuePair; -import net.lightbody.bmp.core.har.HarNameVersion; import net.lightbody.bmp.core.har.HarPage; import net.lightbody.bmp.core.har.HarPageTimings; import net.lightbody.bmp.core.har.HarPostData; @@ -444,7 +443,7 @@ public void testEntryFieldsPopulatedForHttp() throws IOException, InterruptedExc proxy.newHar("testEntryFieldsPopulatedForHttp"); // not using localhost so we get >0ms timing - HttpGet get = new HttpGet("http://www.msn.com"); + HttpGet get = new HttpGet("https://www.msn.com/en-us/"); IOUtils.toStringAndClose(client.execute(get).getEntity().getContent()); proxy.endPage(); @@ -501,7 +500,7 @@ public void testEntryFieldsPopulatedForHttps() throws IOException, InterruptedEx proxy.newHar("testEntryFieldsPopulatedForHttps"); // not using localhost so we get >0ms timing - HttpGet get = new HttpGet("https://www.msn.com"); + HttpGet get = new HttpGet("https://www.msn.com/en-us/"); IOUtils.toStringAndClose(client.execute(get).getEntity().getContent()); proxy.endPage();