From dd3813a396e1baf6ce0a27289a48baad18e211f8 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Tue, 9 Jun 2020 13:57:50 -0500 Subject: [PATCH] Issue #4954 - Initial ByteCount API proposal Signed-off-by: Joakim Erdfelt --- .../org/eclipse/jetty/http/HttpParser.java | 5 + .../eclipse/jetty/server/ByteCountEvent.java | 50 ++++ .../jetty/server/ByteCountEventAdaptor.java | 200 ++++++++++++++++ .../server/ByteCounterChannelListener.java | 169 +++++++++++++ .../jetty/server/ByteCounterListener.java | 29 +++ .../jetty/server/ByteCountingTest.java | 222 ++++++++++++++++++ 6 files changed, 675 insertions(+) create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEvent.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEventAdaptor.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterChannelListener.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterListener.java create mode 100644 jetty-server/src/test/java/org/eclipse/jetty/server/ByteCountingTest.java diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpParser.java b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpParser.java index 4f34c27075b4..cc7fd57b626e 100644 --- a/jetty-http/src/main/java/org/eclipse/jetty/http/HttpParser.java +++ b/jetty-http/src/main/java/org/eclipse/jetty/http/HttpParser.java @@ -365,6 +365,11 @@ public long getContentRead() return _contentPosition; } + public long getHeaderLength() + { + return _headerBytes; + } + /** * Set if a HEAD response is expected * diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEvent.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEvent.java new file mode 100644 index 000000000000..faed435606e6 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEvent.java @@ -0,0 +1,50 @@ +// +// ======================================================================== +// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +public interface ByteCountEvent +{ + Request getRequest(); + + Response getResponse(); + + boolean hasFailure(); + + Throwable getRequestFailure(); + + Throwable getResponseFailure(); + + // TODO: do we care about upgraded connections? + // TODO: what about HTTP/2 ? + + interface HttpByteCount + { + long getHeaderCount(); + + long getBodyCount(); + + long getStreamAPICount(); + + long getTrailerCount(); + } + + HttpByteCount getRequestCount(); + + HttpByteCount getResponseCount(); +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEventAdaptor.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEventAdaptor.java new file mode 100644 index 000000000000..4ad594a6ef03 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCountEventAdaptor.java @@ -0,0 +1,200 @@ +// +// ======================================================================== +// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +public class ByteCountEventAdaptor implements ByteCountEvent +{ + private final Request request; + + public ByteCountEventAdaptor(Request request) + { + this.request = request; + this.requestCounts = new HttpByteCountAdaptor(); + this.responseCounts = new HttpByteCountAdaptor(); + } + + public void onComplete(long bytesIn, long bytesOut) + { + requestCounts.connectionCount = bytesIn; + responseCounts.connectionCount = bytesOut; + } + + class HttpByteCountAdaptor implements HttpByteCount + { + // The connectionCount + private long connectionCount; + // The failure condition / cause + private Throwable failure; + // The location when the failure occurred in connectionCount + private long failureLoc = -1; + // Start of headers in connectionCount; + private long headerStart; + // End of headers in connectionCount; + private long headerEnd = -1; + // Start of body in connectionCount; + private long bodyStart; + // End of body in connectionCount; + private long bodyEnd = -1; + // Start of trailer in connectionCount; + private long trailerStart; + // End of trailer in connectionCount; + private long trailerEnd = -1; + // Reported Streaming API bytes used via Servlet API + private long apiCount = -1; + + @Override + public long getHeaderCount() + { + if (headerEnd >= headerStart) + return headerEnd - headerStart; + else + return -1; + } + + @Override + public long getBodyCount() + { + if (bodyEnd >= bodyStart) + return bodyEnd - bodyStart; + else + return -1; + } + + @Override + public long getStreamAPICount() + { + return apiCount; + } + + @Override + public long getTrailerCount() + { + if (trailerEnd >= trailerStart) + return trailerEnd = trailerStart; + else + return -1; + } + + void onHeadersStart(long connectionCount) + { + this.connectionCount = connectionCount; + this.headerStart = connectionCount; + } + + void onHeadersEnd(long connectionCount) + { + this.connectionCount = connectionCount; + this.headerEnd = connectionCount; + } + + void onBodyStart(long connectionCount) + { + this.connectionCount = connectionCount; + this.bodyStart = connectionCount; + } + + void onBodyEnd(long connectionCount, long byteCountAPI) + { + this.connectionCount = connectionCount; + this.bodyEnd = connectionCount; + this.apiCount = byteCountAPI; + this.trailerStart = connectionCount; + } + + void onTrailerEnd(long connectionCount) + { + this.trailerEnd = connectionCount; + } + + public void onFailure(long connectionCount, Throwable failure) + { + this.connectionCount = connectionCount; + this.failure = failure; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder("HttpByteCount["); + sb.append("connectionCount=").append(connectionCount); + sb.append(", failure=").append(failure); + sb.append(", failureLoc=").append(failureLoc); + sb.append(", headerStart=").append(headerStart); + sb.append(", headerEnd=").append(headerEnd); + sb.append(", bodyStart=").append(bodyStart); + sb.append(", bodyEnd=").append(bodyEnd); + sb.append(", trailerStart=").append(trailerStart); + sb.append(", trailerEnd=").append(trailerEnd); + sb.append(", apiCount=").append(apiCount); + sb.append(']'); + return sb.toString(); + } + } + + /** + * Request Counts. (including HttpConnection.bytesIn) + */ + private HttpByteCountAdaptor requestCounts; + /** + * Response Counts. (including HttpConnection.bytesOut) + */ + private HttpByteCountAdaptor responseCounts; + + @Override + public Request getRequest() + { + return request; + } + + @Override + public HttpByteCountAdaptor getRequestCount() + { + return requestCounts; + } + + @Override + public HttpByteCountAdaptor getResponseCount() + { + return responseCounts; + } + + @Override + public Response getResponse() + { + return request.getResponse(); + } + + @Override + public boolean hasFailure() + { + return (requestCounts.failure != null) || (responseCounts.failure != null); + } + + @Override + public Throwable getRequestFailure() + { + return requestCounts.failure; + } + + @Override + public Throwable getResponseFailure() + { + return responseCounts.failure; + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterChannelListener.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterChannelListener.java new file mode 100644 index 000000000000..9a6591c39b84 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterChannelListener.java @@ -0,0 +1,169 @@ +// +// ======================================================================== +// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jetty.http.HttpParser; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + +public class ByteCounterChannelListener implements HttpChannel.Listener +{ + private static final Logger LOG = Log.getLogger(ByteCounterListener.class); + private static final String ATTR_KEY = ByteCountEvent.class.getName(); + private final List listeners = new ArrayList<>(); + + public void addListener(ByteCounterListener listener) + { + this.listeners.add(listener); + } + + @Override + public void onRequestBegin(Request request) + { + HttpConnection connection = getHttpConnection(request); + if (connection != null) + { + HttpParser httpParser = connection.getParser(); + ByteCountEventAdaptor byteCountEventAdaptor = new ByteCountEventAdaptor(request); + byteCountEventAdaptor.getRequestCount().onHeadersEnd(httpParser.getHeaderLength()); + byteCountEventAdaptor.getRequestCount().onBodyStart(connection.getBytesIn()); + request.setAttribute(ATTR_KEY, byteCountEventAdaptor); + } + } + + @Override + public void onRequestEnd(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + HttpInput httpInput = request.getHttpInput(); + long byteCountRequestAPI = httpInput.getContentConsumed(); + byteCountEventAdaptor.getRequestCount().onBodyEnd(connection.getBytesIn(), byteCountRequestAPI); + } + } + + @Override + public void onRequestTrailers(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.getRequestCount().onTrailerEnd(connection.getBytesIn()); + } + } + + @Override + public void onRequestFailure(Request request, Throwable failure) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.getRequestCount().onFailure(connection.getBytesIn(), failure); + } + } + + @Override + public void onResponseBegin(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.getResponseCount().onHeadersStart(connection.getBytesOut()); + } + } + + @Override + public void onResponseCommit(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.getResponseCount().onHeadersEnd(connection.getBytesOut()); + byteCountEventAdaptor.getResponseCount().onBodyStart(connection.getBytesOut()); + } + } + + @Override + public void onResponseEnd(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + Response response = request.getResponse(); + HttpOutput httpOutput = response.getHttpOutput(); + long byteCountResponseAPI = httpOutput.getWritten(); + byteCountEventAdaptor.getResponseCount().onBodyEnd(connection.getBytesOut(), byteCountResponseAPI); + } + } + + // TODO: need Response Trailers? + + @Override + public void onResponseFailure(Request request, Throwable failure) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.getResponseCount().onFailure(connection.getBytesOut(), failure); + } + } + + @Override + public void onComplete(Request request) + { + HttpConnection connection = getHttpConnection(request); + ByteCountEventAdaptor byteCountEventAdaptor = (ByteCountEventAdaptor)request.getAttribute(ATTR_KEY); + if ((connection != null) && (byteCountEventAdaptor != null)) + { + byteCountEventAdaptor.onComplete(connection.getBytesIn(), connection.getBytesOut()); + notifyByteCount(byteCountEventAdaptor); + } + } + + private void notifyByteCount(ByteCountEvent byteCountEvent) + { + for (ByteCounterListener listener : listeners) + { + try + { + listener.onByteCount(byteCountEvent); + } + catch (Throwable cause) + { + LOG.warn("Unable to notify onByteCount", cause); + } + } + } + + private HttpConnection getHttpConnection(Request request) + { + return (HttpConnection)request.getAttribute("org.eclipse.jetty.server.HttpConnection"); + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterListener.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterListener.java new file mode 100644 index 000000000000..d4af8ddae121 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ByteCounterListener.java @@ -0,0 +1,29 @@ +// +// ======================================================================== +// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +public interface ByteCounterListener +{ + /** + * Event for when the byte counts are known for a particular request. + * + * @param byteCountEvent the byte count details + */ + void onByteCount(ByteCountEvent byteCountEvent); +} diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/ByteCountingTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/ByteCountingTest.java new file mode 100644 index 000000000000..1a39079f391a --- /dev/null +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/ByteCountingTest.java @@ -0,0 +1,222 @@ +// +// ======================================================================== +// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.net.URI; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.http.HttpTester; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.component.LifeCycle; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class ByteCountingTest +{ + private Server server; + + @BeforeEach + public void setup() throws Exception + { + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(0); + + ByteCounterChannelListener byteCounterChannelListener = new ByteCounterChannelListener(); + byteCounterChannelListener.addListener(new ByteCounterLogger()); + connector.addBean(byteCounterChannelListener); + + server.addConnector(connector); + + server.setHandler(new AbstractHandler() + { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + response.setCharacterEncoding("utf-8"); + response.getOutputStream().println("Hello ByteCountingTest"); + baseRequest.setHandled(true); + } + }); + + server.start(); + } + + @AfterEach + public void teardown() + { + LifeCycle.stop(server); + } + + private void dump(HttpTester.Response response) + { + System.err.printf("%s %d %s%n", response.getVersion(), response.getStatus(), response.getReason()); + System.err.println(response); + System.err.println(response.getContent()); + } + + private String makeHttpRequests(CharSequence rawRequest) throws IOException + { + URI baseURI = server.getURI().resolve("/"); + String host = baseURI.getHost(); + int port = baseURI.getPort(); + try (Socket socket = new Socket(host, port); + OutputStream out = socket.getOutputStream(); + InputStream in = socket.getInputStream()) + { + out.write(rawRequest.toString().getBytes(UTF_8)); + out.flush(); + + return IO.toString(in, UTF_8); + } + } + + @Test + public void testSimpleGET() throws IOException + { + StringBuilder rawRequest = new StringBuilder(); + rawRequest.append("GET / HTTP/1.1\r\n"); + rawRequest.append("Host: localhost:").append(server.getURI().getPort()).append("\r\n"); + rawRequest.append("Connection: close\r\n"); + rawRequest.append("\r\n"); + + String rawResponse = makeHttpRequests(rawRequest); + HttpTester.Response response = HttpTester.parseResponse(rawResponse); + dump(response); + assertThat(response.getStatus(), is(HttpStatus.OK_200)); + } + + @Test + public void testSimpleGETError400() throws IOException + { + StringBuilder rawRequest = new StringBuilder(); + rawRequest.append("GET / HTTP/1.1\r\n"); + rawRequest.append("Host: not a vali=d header\r\n"); + rawRequest.append("X Foo: Messy\r\n"); + rawRequest.append("Connection: close\r\n"); + rawRequest.append("\r\n"); + + String rawResponse = makeHttpRequests(rawRequest); + HttpTester.Response response = HttpTester.parseResponse(rawResponse); + dump(response); + assertThat(response.getStatus(), is(HttpStatus.BAD_REQUEST_400)); + } + + @Test + public void testConnectionReuse() throws IOException + { + + URI baseURI = server.getURI().resolve("/"); + String host = baseURI.getHost(); + int port = baseURI.getPort(); + try (Socket socket = new Socket(host, port); + OutputStream out = socket.getOutputStream(); + InputStream in = socket.getInputStream()) + { + StringBuilder rawRequest1 = new StringBuilder(); + rawRequest1.append("GET / HTTP/1.1\r\n"); + rawRequest1.append("Host: localhost:").append(server.getURI().getPort()).append("\r\n"); + rawRequest1.append("\r\n"); + + out.write(rawRequest1.toString().getBytes(UTF_8)); + + StringBuilder rawRequest2 = new StringBuilder(); + rawRequest2.append("GET /Eclipse HTTP/1.1\r\n"); + rawRequest2.append("Host: localhost:").append(server.getURI().getPort()).append("\r\n"); + rawRequest2.append("\r\n"); + + out.write(rawRequest2.toString().getBytes(UTF_8)); + + StringBuilder rawRequest3 = new StringBuilder(); + rawRequest3.append("GET /Jetty HTTP/1.1\r\n"); + rawRequest3.append("Host: localhost:").append(server.getURI().getPort()).append("\r\n"); + rawRequest3.append("Connection: close\r\n"); + rawRequest3.append("\r\n"); + + out.write(rawRequest3.toString().getBytes(UTF_8)); + out.flush(); + + HttpTester.Response response; + + response = HttpTester.parseResponse(in); + dump(response); + response = HttpTester.parseResponse(in); + dump(response); + response = HttpTester.parseResponse(in); + dump(response); + } + } + + public static class DumpServlet extends HttpServlet + { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException + { + resp.setContentType("text/plain"); + resp.setCharacterEncoding("utf-8"); + String audience = "World"; + if (req.getPathInfo() != null) + { + audience = req.getPathInfo(); + while (audience.startsWith("/")) + { + audience = audience.substring(1); + } + } + resp.getOutputStream().print("Hello " + audience); + } + } + + public static class ByteCounterLogger implements ByteCounterListener + { + private static final Logger LOG = Log.getLogger(ByteCounterLogger.class); + + @Override + public void onByteCount(ByteCountEvent event) + { + LOG.info("ByteCount [{}] Request=h:{}/b:{}/t:{}/a:{} Response=h:{}/b:{}/t:{}/a:{}", + event.getRequest().getHttpURI(), + event.getRequestCount().getHeaderCount(), + event.getRequestCount().getBodyCount(), + event.getRequestCount().getTrailerCount(), + event.getRequestCount().getStreamAPICount(), + event.getResponseCount().getHeaderCount(), + event.getResponseCount().getBodyCount(), + event.getResponseCount().getTrailerCount(), + event.getResponseCount().getStreamAPICount() + ); + } + } +}