Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 35 additions & 0 deletions docs/content/exporters/httpserver.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,41 @@ or [inetAddress()](</client_java/api/io/prometheus/metrics/exporter/httpserver/H
The default handler can be changed
with [defaultHandler()](</client_java/api/io/prometheus/metrics/exporter/httpserver/HTTPServer.Builder.html#defaultHandler(com.sun.net.httpserver.HttpHandler)>).

## Scrape error handling

By default, scrape failures return a generic HTTP 500 response. Exception details are not
included in the response or logged, because the server may run inside an application or a
Java agent with its own diagnostic pipeline.

Configure a reporter to send exception details to an appropriate logging or telemetry sink:

```java
HTTPServer server = HTTPServer.builder()
.port(9400)
.errorHandlingPolicy(
HttpErrorHandlingPolicy.builder()
.errorReporter(error -> logger.log(Level.SEVERE, "Prometheus scrape failed", error))
.build())
.buildAndStart();
```

The reporter runs synchronously on the request thread and may be called concurrently. Reporter
runtime exceptions do not prevent the generic HTTP 500 response from being sent. Rate limiting
or deduplication can be implemented in the reporter when needed.

For local debugging, an unsafe response containing the full exception stack trace can be enabled
explicitly:

```java
HttpErrorHandlingPolicy.builder()
.unsafeDebugResponse(true)
.build()
```

This setting is independent of the error reporter, so both can be configured when needed. The
unsafe debug response can disclose application internals and must not be enabled for an endpoint
reachable by untrusted clients.

## Authentication and HTTPS

- [authenticator()](</client_java/api/io/prometheus/metrics/exporter/httpserver/HTTPServer.Builder.html#authenticator(com.sun.net.httpserver.Authenticator)>)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,11 @@ void testErrorHandling() throws IOException {
start("error");
Response response = scrape("GET", "");
assertThat(response.status).isEqualTo(500);
assertThat(response.stringBody()).contains("Simulating an error.");
assertErrorResponseBody(response.stringBody());
}

protected void assertErrorResponseBody(String body) {
assertThat(body).contains("Simulating an error.");
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,21 @@
package io.prometheus.metrics.it.exporter.test;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.IOException;
import java.net.URISyntaxException;

class HttpServerIT extends ExporterIT {
public HttpServerIT() throws IOException, URISyntaxException {
super("exporter-httpserver-sample");
}

@Override
protected void assertErrorResponseBody(String body) {
assertThat(body)
.isEqualTo(
"An internal error occurred while scraping metrics. "
+ "Configure an HTTP error reporter for details.\n")
.doesNotContain("Simulating an error.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ private HTTPServer(
@Nullable String authenticatedSubjectAttributeName,
@Nullable HttpHandler defaultHandler,
@Nullable String metricsHandlerPath,
@Nullable Boolean registerHealthHandler) {
@Nullable Boolean registerHealthHandler,
HttpErrorHandlingPolicy errorHandlingPolicy) {
if (httpServer.getAddress() == null) {
throw new IllegalArgumentException("HttpServer hasn't been bound to an address");
}
Expand All @@ -85,7 +86,7 @@ private HTTPServer(
}
registerHandler(
metricsPath,
new MetricsHandler(config, registry),
new MetricsHandler(config, registry, errorHandlingPolicy),
authenticator,
authenticatedSubjectAttributeName);
if (registerHealthHandler == null || registerHealthHandler) {
Expand Down Expand Up @@ -211,6 +212,7 @@ public static class Builder {
@Nullable private HttpHandler defaultHandler = null;
@Nullable private String metricsHandlerPath = null;
@Nullable private Boolean registerHealthHandler = null;
private HttpErrorHandlingPolicy errorHandlingPolicy = HttpErrorHandlingPolicy.builder().build();

private Builder(PrometheusProperties config) {
this.config = config;
Expand Down Expand Up @@ -295,6 +297,20 @@ public Builder registerHealthHandler(boolean registerHealthHandler) {
return this;
}

/**
* Configure how exceptions raised while scraping metrics are reported to the client and
* optionally to a caller-supplied diagnostic sink.
*
* <p>Default is {@code HttpErrorHandlingPolicy.builder().build()}.
*/
public Builder errorHandlingPolicy(HttpErrorHandlingPolicy errorHandlingPolicy) {
if (errorHandlingPolicy == null) {
throw new NullPointerException("errorHandlingPolicy");
}
this.errorHandlingPolicy = errorHandlingPolicy;
return this;
}

/** Build and start the HTTPServer. */
public HTTPServer buildAndStart() throws IOException {
if (registry == null) {
Expand All @@ -318,7 +334,8 @@ public HTTPServer buildAndStart() throws IOException {
authenticatedSubjectAttributeName,
defaultHandler,
metricsHandlerPath,
registerHealthHandler);
registerHealthHandler,
errorHandlingPolicy);
}

private InetSocketAddress makeInetSocketAddress() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package io.prometheus.metrics.exporter.httpserver;

import io.prometheus.metrics.annotations.StableApi;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.function.Consumer;
import javax.annotation.Nullable;

/**
* Controls how the {@link HTTPServer} handles exceptions raised while scraping metrics.
*
* <p>The default policy built by {@link #builder()} does not expose exception details and does not
* report the exception. Configure the builder to route diagnostic details to an
* application-appropriate sink.
*/
@StableApi
public final class HttpErrorHandlingPolicy {

private static final byte[] GENERIC_RESPONSE =
("An internal error occurred while scraping metrics. "
+ "Configure an HTTP error reporter for details.\n")
.getBytes(StandardCharsets.UTF_8);

private final boolean unsafeDebugResponse;
@Nullable private final Consumer<Throwable> errorReporter;

private HttpErrorHandlingPolicy(
boolean unsafeDebugResponse, @Nullable Consumer<Throwable> errorReporter) {
this.unsafeDebugResponse = unsafeDebugResponse;
this.errorReporter = errorReporter;
}

/**
* Returns a builder for configuring scrape error handling.
*
* <p>The builder defaults to a generic HTTP 500 response with no error reporter. This avoids
* exposing exception details to scrape clients or adding an implicit dependency on an
* application's logging configuration.
*/
public static Builder builder() {
return new Builder();
}

byte[] getErrorResponse(Exception exception) {
if (!unsafeDebugResponse) {
return GENERIC_RESPONSE;
}
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
printWriter.write("An Exception occurred while scraping metrics: ");
exception.printStackTrace(printWriter);
return stringWriter.toString().getBytes(StandardCharsets.UTF_8);
}

void report(Throwable error) {
if (errorReporter != null) {
errorReporter.accept(error);
}
}

/** Builder for {@link HttpErrorHandlingPolicy}. */
public static final class Builder {

private boolean unsafeDebugResponse = false;
@Nullable private Consumer<Throwable> errorReporter;

private Builder() {}

/**
* Pass scrape exceptions to {@code errorReporter}.
*
* <p>The reporter runs synchronously on the HTTP request thread. It should return promptly and
* must be safe to call concurrently. Runtime exceptions thrown by the reporter are isolated
* from HTTP response handling.
*/
public Builder errorReporter(Consumer<Throwable> errorReporter) {
if (errorReporter == null) {
throw new NullPointerException("errorReporter");
}
this.errorReporter = errorReporter;
return this;
}

/**
* Configure whether the HTTP 500 response includes the full exception stack trace.
*
* <p><strong>Security warning:</strong> Setting this to {@code true} exposes internal exception
* information to scrape clients. Do not enable it for endpoints reachable by untrusted clients.
*
* <p>This setting is independent of {@link #errorReporter(Consumer)}.
*/
public Builder unsafeDebugResponse(boolean unsafeDebugResponse) {
this.unsafeDebugResponse = unsafeDebugResponse;
return this;
}

/** Build the policy. */
public HttpErrorHandlingPolicy build() {
return new HttpErrorHandlingPolicy(unsafeDebugResponse, errorReporter);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@
import io.prometheus.metrics.exporter.common.PrometheusHttpResponse;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
Expand All @@ -18,13 +15,21 @@

public class HttpExchangeAdapter implements PrometheusHttpExchange {

private static final Logger logger = Logger.getLogger(HttpExchangeAdapter.class.getName());

private final HttpExchange httpExchange;
private final HttpErrorHandlingPolicy errorHandlingPolicy;
private final HttpRequest request = new HttpRequest();
private final HttpResponse response = new HttpResponse();
private volatile boolean responseSent = false;

public HttpExchangeAdapter(HttpExchange httpExchange) {
this(httpExchange, HttpErrorHandlingPolicy.builder().build());
}

HttpExchangeAdapter(HttpExchange httpExchange, HttpErrorHandlingPolicy errorHandlingPolicy) {
this.httpExchange = httpExchange;
this.errorHandlingPolicy = errorHandlingPolicy;
}

public class HttpRequest implements PrometheusHttpRequest {
Expand Down Expand Up @@ -92,52 +97,53 @@ public HttpResponse getResponse() {

@Override
public void handleException(IOException e) throws IOException {
sendErrorResponseWithStackTrace(e);
sendErrorResponse(e);
}

@Override
public void handleException(RuntimeException e) {
sendErrorResponseWithStackTrace(e);
sendErrorResponse(e);
}

private void sendErrorResponseWithStackTrace(Exception requestHandlerException) {
private void sendErrorResponse(Exception requestHandlerException) {
if (!responseSent) {
responseSent = true;
reportException(requestHandlerException);
byte[] errorResponse = errorHandlingPolicy.getErrorResponse(requestHandlerException);
try {
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
printWriter.write("An Exception occurred while scraping metrics: ");
requestHandlerException.printStackTrace(new PrintWriter(printWriter));
byte[] stackTrace = stringWriter.toString().getBytes(StandardCharsets.UTF_8);
httpExchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8");
httpExchange.sendResponseHeaders(500, stackTrace.length);
httpExchange.getResponseBody().write(stackTrace);
httpExchange.sendResponseHeaders(500, errorResponse.length);
httpExchange.getResponseBody().write(errorResponse);
} catch (IOException errorWriterException) {
// We want to avoid logging so that we don't mess with application logs when the HTTPServer
Comment thread
zeitlinger marked this conversation as resolved.
// is used in a Java agent.
// However, if we can't even send an error response to the client there's nothing we can do
// but logging a message.
Logger.getLogger(this.getClass().getName())
.log(
Level.SEVERE,
"The Prometheus metrics HTTPServer caught an Exception during scrape and "
+ "failed to send an error response to the client.",
errorWriterException);
Logger.getLogger(this.getClass().getName())
.log(
Level.SEVERE,
"Original Exception that caused the Prometheus scrape error:",
requestHandlerException);
// If we can't even send an error response to the client, logging is the only remaining
// signal.
logger.log(
Level.SEVERE,
"The Prometheus metrics HTTPServer caught an Exception during scrape and "
+ "failed to send an error response to the client.",
errorWriterException);
logger.log(
Level.SEVERE,
"Original Exception that caused the Prometheus scrape error:",
requestHandlerException);
}
} else {
// If the exception occurs after response headers have been sent, it's too late to respond
// with HTTP 500.
Logger.getLogger(this.getClass().getName())
.log(
Level.SEVERE,
"The Prometheus metrics HTTPServer caught an Exception while trying to send "
+ "the metrics response.",
requestHandlerException);
logger.log(
Level.SEVERE,
"The Prometheus metrics HTTPServer caught an Exception while trying to send "
+ "the metrics response.",
requestHandlerException);
}
}

private void reportException(Exception requestHandlerException) {
try {
errorHandlingPolicy.report(requestHandlerException);
} catch (RuntimeException ignored) {
// A caller-supplied reporter must not prevent the safe error response from being sent or
// implicitly fall back to application logging.
}
}

Expand Down
Loading