diff --git a/spring-web/src/main/java/org/springframework/web/server/MethodNotAllowedException.java b/spring-web/src/main/java/org/springframework/web/server/MethodNotAllowedException.java
index 14dcd2a62b8f..40cf7d46607a 100644
--- a/spring-web/src/main/java/org/springframework/web/server/MethodNotAllowedException.java
+++ b/spring-web/src/main/java/org/springframework/web/server/MethodNotAllowedException.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,12 +22,12 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
-import org.springframework.util.StringUtils;
 
 /**
  * Exception for errors that fit response status 405 (method not allowed).
@@ -62,11 +62,24 @@ public MethodNotAllowedException(String method, @Nullable Collection<HttpMethod>
 	 * Return a Map with an "Allow" header.
 	 * @since 5.1.11
 	 */
+	@SuppressWarnings("deprecation")
 	@Override
 	public Map<String, String> getHeaders() {
-		return !CollectionUtils.isEmpty(this.httpMethods) ?
-				Collections.singletonMap("Allow", StringUtils.collectionToDelimitedString(this.httpMethods, ", ")) :
-				Collections.emptyMap();
+		return getResponseHeaders().toSingleValueMap();
+	}
+
+	/**
+	 * Return HttpHeaders with an "Allow" header.
+	 * @since 5.1.13
+	 */
+	@Override
+	public HttpHeaders getResponseHeaders() {
+		if (CollectionUtils.isEmpty(this.httpMethods)) {
+			return HttpHeaders.EMPTY;
+		}
+		HttpHeaders headers = new HttpHeaders();
+		headers.setAllow(this.httpMethods);
+		return headers;
 	}
 
 	/**
diff --git a/spring-web/src/main/java/org/springframework/web/server/NotAcceptableStatusException.java b/spring-web/src/main/java/org/springframework/web/server/NotAcceptableStatusException.java
index aa8a01d46a68..6045dbd51693 100644
--- a/spring-web/src/main/java/org/springframework/web/server/NotAcceptableStatusException.java
+++ b/spring-web/src/main/java/org/springframework/web/server/NotAcceptableStatusException.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.util.CollectionUtils;
@@ -57,11 +58,24 @@ public NotAcceptableStatusException(List<MediaType> supportedMediaTypes) {
 	 * Return a Map with an "Accept" header.
 	 * @since 5.1.11
 	 */
+	@SuppressWarnings("deprecation")
 	@Override
 	public Map<String, String> getHeaders() {
-		return !CollectionUtils.isEmpty(this.supportedMediaTypes) ?
-				Collections.singletonMap("Accept", MediaType.toString(this.supportedMediaTypes)) :
-				Collections.emptyMap();
+		return getResponseHeaders().toSingleValueMap();
+	}
+
+	/**
+	 * Return HttpHeaders with an "Accept" header, or an empty instance.
+	 * @since 5.1.13
+	 */
+	@Override
+	public HttpHeaders getResponseHeaders() {
+		if (CollectionUtils.isEmpty(this.supportedMediaTypes)) {
+			return HttpHeaders.EMPTY;
+		}
+		HttpHeaders headers = new HttpHeaders();
+		headers.setAccept(this.supportedMediaTypes);
+		return headers;
 	}
 
 	/**
diff --git a/spring-web/src/main/java/org/springframework/web/server/ResponseStatusException.java b/spring-web/src/main/java/org/springframework/web/server/ResponseStatusException.java
index a0245686a081..67c8d78391ea 100644
--- a/spring-web/src/main/java/org/springframework/web/server/ResponseStatusException.java
+++ b/spring-web/src/main/java/org/springframework/web/server/ResponseStatusException.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 
 import org.springframework.core.NestedExceptionUtils;
 import org.springframework.core.NestedRuntimeException;
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
@@ -82,14 +83,33 @@ public HttpStatus getStatus() {
 	}
 
 	/**
-	 * Return response headers associated with the exception, possibly required
-	 * for the given status code (e.g. "Allow", "Accept").
+	 * Return headers associated with the exception that should be added to the
+	 * error response, e.g. "Allow", "Accept", etc.
+	 * <p>The default implementation in this class returns an empty map.
 	 * @since 5.1.11
+	 * @deprecated as of 5.1.13 in favor of {@link #getResponseHeaders()}
 	 */
+	@Deprecated
 	public Map<String, String> getHeaders() {
 		return Collections.emptyMap();
 	}
 
+	/**
+	 * Return headers associated with the exception that should be added to the
+	 * error response, e.g. "Allow", "Accept", etc.
+	 * <p>The default implementation in this class returns empty headers.
+	 * @since 5.1.13
+	 */
+	public HttpHeaders getResponseHeaders() {
+		Map<String, String> headers = getHeaders();
+		if (headers.isEmpty()) {
+			return HttpHeaders.EMPTY;
+		}
+		HttpHeaders result = new HttpHeaders();
+		getHeaders().forEach(result::add);
+		return result;
+	}
+
 	/**
 	 * The reason explaining the exception (potentially {@code null} or empty).
 	 */
diff --git a/spring-web/src/main/java/org/springframework/web/server/handler/ResponseStatusExceptionHandler.java b/spring-web/src/main/java/org/springframework/web/server/handler/ResponseStatusExceptionHandler.java
index 263e9cf1afe0..cda26cba2136 100644
--- a/spring-web/src/main/java/org/springframework/web/server/handler/ResponseStatusExceptionHandler.java
+++ b/spring-web/src/main/java/org/springframework/web/server/handler/ResponseStatusExceptionHandler.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -92,8 +92,9 @@ private boolean updateResponse(ServerHttpResponse response, Throwable ex) {
 		if (status != null) {
 			if (response.setStatusCode(status)) {
 				if (ex instanceof ResponseStatusException) {
-					((ResponseStatusException) ex).getHeaders()
-							.forEach((name, value) -> response.getHeaders().add(name, value));
+					((ResponseStatusException) ex).getResponseHeaders()
+							.forEach((name, values) ->
+									values.forEach(value -> response.getHeaders().add(name, value)));
 				}
 				result = true;
 			}