diff --git a/README.md b/README.md index 15bc97bb78..a1d08ca4e1 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Protocol conformance - [RFC 2817](https://datatracker.ietf.org/doc/html/rfc2817) - Upgrading to TLS Within HTTP/1.1 - [RFC 9218](https://datatracker.ietf.org/doc/html/rfc9218) - Extensible Prioritization Scheme for HTTP - [RFC 7804](https://datatracker.ietf.org/doc/html/rfc7804) - Salted Challenge Response HTTP Authentication Mechanism +- [RFC 10008](https://datatracker.ietf.org/doc/html/rfc10008) - The HTTP QUERY Method Licensing --------- diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/AsyncCachingExec.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/AsyncCachingExec.java index 01141b32c6..aea9121533 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/AsyncCachingExec.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/AsyncCachingExec.java @@ -72,10 +72,12 @@ import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.HttpResponse; import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.Method; import org.apache.hc.core5.http.impl.BasicEntityDetails; import org.apache.hc.core5.http.nio.AsyncDataConsumer; import org.apache.hc.core5.http.nio.AsyncEntityProducer; import org.apache.hc.core5.http.nio.CapacityChannel; +import org.apache.hc.core5.http.nio.DataStreamChannel; import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer; import org.apache.hc.core5.http.nio.entity.StringAsyncEntityProducer; import org.apache.hc.core5.net.URIAuthority; @@ -412,12 +414,87 @@ public void cancelled() { } SimpleHttpRequest prepareRequest(final HttpRequest request, - final AsyncEntityProducer entityProducer) { - // To be revised when implementing QUERY support - if (entityProducer != null) { - return null; + final AsyncEntityProducer entityProducer) throws IOException { + if (entityProducer == null) { + return SimpleRequestBuilder.copy(request).build(); + } + // QUERY content must be read in full in order to determine the cache key + if (Method.QUERY.isSame(request.getMethod()) + && entityProducer.isRepeatable() + && entityProducer.getContentEncoding() == null + && entityProducer.getContentLength() <= MAX_BUFFERED_CONTENT_LENGTH) { + final byte[] content = drainEntityProducer(entityProducer); + if (content != null) { + return SimpleRequestBuilder.copy(request) + .setBody(content, ContentType.parseLenient(entityProducer.getContentType())) + .build(); + } + } + return null; + } + + /** + * Drains the given repeatable entity producer into a byte array without an I/O reactor. + * Returns {@code null} if the producer fails to make progress or produces more than + * {@link CachingExecBase#MAX_BUFFERED_CONTENT_LENGTH} bytes of content, in which case + * it is reset by {@link AsyncEntityProducer#releaseResources()} and can still be used + * to execute the request directly. + */ + private static byte[] drainEntityProducer(final AsyncEntityProducer entityProducer) throws IOException { + final ByteArrayBuffer buf = new ByteArrayBuffer(1024); + final AtomicBoolean endStream = new AtomicBoolean(); + final DataStreamChannel channel = new DataStreamChannel() { + + @Override + public void requestOutput() { + } + + @Override + public int write(final ByteBuffer src) { + final int len = src.remaining(); + // Buffer at most one byte over the limit; the drain loop treats that as oversize + final int chunk = Math.min(len, MAX_BUFFERED_CONTENT_LENGTH + 1 - buf.length()); + if (chunk > 0) { + if (src.hasArray()) { + buf.append(src.array(), src.arrayOffset() + src.position(), chunk); + } else { + final byte[] tmp = new byte[chunk]; + src.get(tmp); + buf.append(tmp, 0, chunk); + } + } + src.position(src.limit()); + return len; + } + + @Override + public void endStream() { + endStream.set(true); + } + + @Override + public void endStream(final List trailers) { + endStream.set(true); + } + + }; + try { + while (!endStream.get()) { + final int before = buf.length(); + entityProducer.produce(channel); + if (buf.length() > MAX_BUFFERED_CONTENT_LENGTH) { + return null; + } + if (!endStream.get() && buf.length() == before) { + // The producer is not self-contained and cannot be drained + // without an I/O reactor + return null; + } + } + } finally { + entityProducer.releaseResources(); } - return SimpleRequestBuilder.copy(request).build(); + return buf.toByteArray(); } void callChain( diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheKeyGenerator.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheKeyGenerator.java index 51dcf48961..4aeb2d389f 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheKeyGenerator.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheKeyGenerator.java @@ -29,6 +29,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -40,17 +42,22 @@ import java.util.function.Consumer; import java.util.stream.StreamSupport; +import org.apache.hc.client5.http.async.methods.SimpleBody; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.cache.HttpCacheEntry; +import org.apache.hc.client5.http.utils.Hex; import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.Internal; import org.apache.hc.core5.annotation.ThreadingBehavior; import org.apache.hc.core5.function.Resolver; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HeaderElement; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.MessageHeaders; +import org.apache.hc.core5.http.Method; import org.apache.hc.core5.http.NameValuePair; import org.apache.hc.core5.http.message.BasicHeaderElementIterator; import org.apache.hc.core5.http.message.BasicHeaderValueFormatter; @@ -131,6 +138,10 @@ public String generateKey(final URI requestUri) { /** * Computes a root key for the given {@link HttpHost} and {@link HttpRequest} * that can be used as a unique identifier for cached resources. + *

+ * For {@literal QUERY} requests with an enclosed body the key also incorporates + * a digest of the request content and its content type, as required by the HTTP + * QUERY method specification (RFC 10008). * * @param host The host for this request * @param request the {@link HttpRequest} @@ -138,11 +149,39 @@ public String generateKey(final URI requestUri) { */ public String generateKey(final HttpHost host, final HttpRequest request) { final String s = CacheSupport.requestUriRaw(host, request); + String rootKey; try { - return generateKey(new URI(s)); + rootKey = generateKey(new URI(s)); } catch (final URISyntaxException ex) { - return s; + rootKey = s; } + if (request instanceof SimpleHttpRequest && Method.QUERY.isSame(request.getMethod())) { + final SimpleBody body = ((SimpleHttpRequest) request).getBody(); + if (body != null) { + return "{QUERY:" + generateContentDigest(body) + "}" + rootKey; + } + } + return rootKey; + } + + /** + * Computes a digest of request content and its content type suitable + * for cache key generation. + */ + static String generateContentDigest(final SimpleBody body) { + final MessageDigest digester; + try { + digester = MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException ex) { + throw new IllegalStateException("SHA-256 message digest is not supported", ex); + } + final ContentType contentType = body.getContentType(); + if (contentType != null) { + digester.update(contentType.toString().getBytes(StandardCharsets.UTF_8)); + } + digester.update((byte) 0); + digester.update(body.getBodyBytes()); + return Hex.encodeHexString(digester.digest()); } /** diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheableRequestPolicy.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheableRequestPolicy.java index f1fbdb6a7f..3006224763 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheableRequestPolicy.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheableRequestPolicy.java @@ -57,7 +57,7 @@ public boolean canBeServedFromCache(final RequestCacheControl cacheControl, fina return false; } - if (!Method.GET.isSame(method) && !Method.HEAD.isSame(method)) { + if (!Method.GET.isSame(method) && !Method.HEAD.isSame(method) && !Method.QUERY.isSame(method)) { if (LOG.isDebugEnabled()) { LOG.debug("{} request cannot be served from cache", method); } diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExec.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExec.java index 81f1abc936..a32daec0ae 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExec.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExec.java @@ -61,6 +61,7 @@ import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.HttpVersion; +import org.apache.hc.core5.http.Method; import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.StringEntity; @@ -221,12 +222,28 @@ private static ClassicHttpResponse convert(final SimpleHttpResponse cacheRespons return response; } - SimpleHttpRequest prepareRequest(final ClassicHttpRequest request) { - // To be revised when implementing QUERY support - if (request.getEntity() != null) { - return null; + SimpleHttpRequest prepareRequest(final ClassicHttpRequest request) throws IOException { + final HttpEntity entity = request.getEntity(); + if (entity == null) { + return SimpleRequestBuilder.copy(request).build(); + } + // QUERY content must be read in full in order to determine the cache key + if (Method.QUERY.isSame(request.getMethod()) + && entity.isRepeatable() + && entity.getContentEncoding() == null + && entity.getContentLength() <= MAX_BUFFERED_CONTENT_LENGTH) { + // Read one byte past the limit in order to distinguish an at-limit body from an oversized one + final byte[] content = EntityUtils.toByteArray(entity, MAX_BUFFERED_CONTENT_LENGTH + 1); + if (content != null && content.length > MAX_BUFFERED_CONTENT_LENGTH) { + return null; + } + final SimpleRequestBuilder builder = SimpleRequestBuilder.copy(request); + if (content != null) { + builder.setBody(content, ContentType.parseLenient(entity.getContentType())); + } + return builder.build(); } - return SimpleRequestBuilder.copy(request).build(); + return null; } ClassicHttpResponse callBackend( diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExecBase.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExecBase.java index 5cef2ec22a..53b6a92b6c 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExecBase.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CachingExecBase.java @@ -42,6 +42,13 @@ public class CachingExecBase { + /** + * Maximum size of a request content body that can be buffered in memory + * in order to compute a cache key. Requests with a larger content body + * are considered non-cacheable. + */ + static final int MAX_BUFFERED_CONTENT_LENGTH = 25 * 1024; + final AtomicLong cacheHits = new AtomicLong(); final AtomicLong cacheMisses = new AtomicLong(); final AtomicLong cacheUpdates = new AtomicLong(); diff --git a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCachingPolicy.java b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCachingPolicy.java index f5d25c35c6..5c5c3af929 100644 --- a/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCachingPolicy.java +++ b/httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCachingPolicy.java @@ -98,9 +98,9 @@ public boolean isResponseCacheable(final ResponseCacheControl cacheControl, fina return false; } - // Presently only GET and HEAD methods are supported + // Presently only GET, HEAD and QUERY methods are supported final String httpMethod = request.getMethod(); - if (!Method.GET.isSame(httpMethod) && !Method.HEAD.isSame(httpMethod)) { + if (!Method.GET.isSame(httpMethod) && !Method.HEAD.isSame(httpMethod) && !Method.QUERY.isSame(httpMethod)) { if (LOG.isDebugEnabled()) { LOG.debug("{} method response is not cacheable", httpMethod); } diff --git a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestAsyncCachingExecQuery.java b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestAsyncCachingExecQuery.java new file mode 100644 index 0000000000..d50dc5ce91 --- /dev/null +++ b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestAsyncCachingExecQuery.java @@ -0,0 +1,213 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.cache; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Method; +import org.apache.hc.core5.http.message.BasicHttpRequest; +import org.apache.hc.core5.http.nio.AsyncEntityProducer; +import org.apache.hc.core5.http.nio.DataStreamChannel; +import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestAsyncCachingExecQuery { + + private AsyncCachingExec impl; + + @BeforeEach + void setUp() { + final HttpAsyncCache cache = new BasicHttpAsyncCache( + HeapResourceFactory.INSTANCE, new SimpleHttpAsyncCacheStorage()); + impl = new AsyncCachingExec(cache, null, CacheConfig.DEFAULT); + } + + @Test + void testPrepareRequestBuffersQueryContentFromRepeatableProducer() throws Exception { + final AsyncEntityProducer producer = new BasicAsyncEntityProducer( + "select a".getBytes(StandardCharsets.UTF_8), ContentType.TEXT_PLAIN); + + final SimpleHttpRequest converted = impl.prepareRequest( + new BasicHttpRequest(Method.QUERY, "/stuff"), producer); + + Assertions.assertNotNull(converted); + Assertions.assertNotNull(converted.getBody()); + Assertions.assertEquals("select a", + new String(converted.getBody().getBodyBytes(), StandardCharsets.UTF_8)); + Assertions.assertEquals(ContentType.TEXT_PLAIN.getMimeType(), + converted.getBody().getContentType().getMimeType()); + } + + @Test + void testPrepareRequestBypassesNonQueryRequestsWithContent() throws Exception { + final AsyncEntityProducer producer = new BasicAsyncEntityProducer( + "stuff".getBytes(StandardCharsets.UTF_8), ContentType.TEXT_PLAIN); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.POST, "/stuff"), producer)); + } + + @Test + void testPrepareRequestBypassesQueryWithNonRepeatableProducer() throws Exception { + final StallingEntityProducer producer = new StallingEntityProducer(false); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.QUERY, "/stuff"), producer)); + } + + @Test + void testPrepareRequestBypassesAndResetsStallingRepeatableProducer() throws Exception { + final StallingEntityProducer producer = new StallingEntityProducer(true); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.QUERY, "/stuff"), producer)); + Assertions.assertTrue(producer.released, + "a partially drained producer must be reset so that it can still be used"); + } + + @Test + void testPrepareRequestBypassesOversizedQueryContent() throws Exception { + final AsyncEntityProducer producer = new BasicAsyncEntityProducer( + new byte[CachingExecBase.MAX_BUFFERED_CONTENT_LENGTH + 1], ContentType.TEXT_PLAIN); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.QUERY, "/stuff"), producer)); + } + + @Test + void testPrepareRequestBypassesAndResetsOversizedQueryContentOfUnknownLength() throws Exception { + final UnboundedEntityProducer producer = new UnboundedEntityProducer(); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.QUERY, "/stuff"), producer)); + Assertions.assertTrue(producer.released, + "a partially drained producer must be reset so that it can still be used"); + } + + @Test + void testPrepareRequestBypassesOversizedQueryContentWrittenInSingleProduceCall() throws Exception { + final SingleWriteEntityProducer producer = new SingleWriteEntityProducer( + CachingExecBase.MAX_BUFFERED_CONTENT_LENGTH * 4); + + Assertions.assertNull(impl.prepareRequest(new BasicHttpRequest(Method.QUERY, "/stuff"), producer)); + Assertions.assertTrue(producer.released, + "a partially drained producer must be reset so that it can still be used"); + } + + static class StallingEntityProducer implements AsyncEntityProducer { + + private final boolean repeatable; + + boolean released; + + StallingEntityProducer(final boolean repeatable) { + this.repeatable = repeatable; + } + + @Override + public boolean isRepeatable() { + return repeatable; + } + + @Override + public void failed(final Exception cause) { + } + + @Override + public long getContentLength() { + return -1; + } + + @Override + public String getContentType() { + return ContentType.TEXT_PLAIN.toString(); + } + + @Override + public String getContentEncoding() { + return null; + } + + @Override + public boolean isChunked() { + return true; + } + + @Override + public Set getTrailerNames() { + return null; + } + + @Override + public int available() { + return 0; + } + + @Override + public void produce(final DataStreamChannel channel) throws IOException { + } + + @Override + public void releaseResources() { + released = true; + } + + } + + static class UnboundedEntityProducer extends StallingEntityProducer { + + UnboundedEntityProducer() { + super(true); + } + + @Override + public void produce(final DataStreamChannel channel) throws IOException { + channel.write(ByteBuffer.wrap(new byte[8 * 1024])); + } + + } + + static class SingleWriteEntityProducer extends StallingEntityProducer { + + private final int contentLength; + + SingleWriteEntityProducer(final int contentLength) { + super(true); + this.contentLength = contentLength; + } + + @Override + public void produce(final DataStreamChannel channel) throws IOException { + channel.write(ByteBuffer.wrap(new byte[contentLength])); + channel.endStream(); + } + + } + +} diff --git a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheKeyGenerator.java b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheKeyGenerator.java index b1784e83d7..62b3f2577a 100644 --- a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheKeyGenerator.java +++ b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheKeyGenerator.java @@ -32,8 +32,11 @@ import java.util.Iterator; import java.util.List; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; import org.apache.hc.client5.http.cache.HttpCacheEntry; import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; @@ -354,4 +357,54 @@ void testGetVariantKeyFromCachedResponse() { Assertions.assertEquals("{accept-encoding=text%2Fplain&user-agent=agent1}", extractor.generateVariantKey(request, entry2)); } + @Test + void testQueryKeyIncorporatesRequestContent() { + final HttpHost host = new HttpHost("foo.example.com"); + final SimpleHttpRequest query1 = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select a", ContentType.TEXT_PLAIN) + .build(); + final SimpleHttpRequest query2 = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select b", ContentType.TEXT_PLAIN) + .build(); + final SimpleHttpRequest query3 = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select a", ContentType.TEXT_PLAIN) + .build(); + + Assertions.assertNotEquals(extractor.generateKey(host, query1), extractor.generateKey(host, query2)); + Assertions.assertEquals(extractor.generateKey(host, query1), extractor.generateKey(host, query3)); + } + + @Test + void testQueryKeyIncorporatesContentType() { + final HttpHost host = new HttpHost("foo.example.com"); + final SimpleHttpRequest query1 = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select a", ContentType.TEXT_PLAIN) + .build(); + final SimpleHttpRequest query2 = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select a", ContentType.APPLICATION_JSON) + .build(); + + Assertions.assertNotEquals(extractor.generateKey(host, query1), extractor.generateKey(host, query2)); + } + + @Test + void testQueryKeyDiffersFromUriKey() { + final HttpHost host = new HttpHost("foo.example.com"); + final SimpleHttpRequest get = SimpleRequestBuilder.get("http://foo.example.com/stuff").build(); + final SimpleHttpRequest query = SimpleRequestBuilder.query("http://foo.example.com/stuff") + .setBody("select a", ContentType.TEXT_PLAIN) + .build(); + + Assertions.assertNotEquals(extractor.generateKey(host, get), extractor.generateKey(host, query)); + } + + @Test + void testQueryKeyWithoutContentMatchesUriKey() { + final HttpHost host = new HttpHost("foo.example.com"); + final SimpleHttpRequest get = SimpleRequestBuilder.get("http://foo.example.com/stuff").build(); + final SimpleHttpRequest query = SimpleRequestBuilder.query("http://foo.example.com/stuff").build(); + + Assertions.assertEquals(extractor.generateKey(host, get), extractor.generateKey(host, query)); + } + } diff --git a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheableRequestPolicy.java b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheableRequestPolicy.java index 99b423e9a5..655a742d65 100644 --- a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheableRequestPolicy.java +++ b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCacheableRequestPolicy.java @@ -113,4 +113,22 @@ void testIsArbitraryMethodServableFromCache() { } + @Test + void testIsQueryServableFromCache() { + final BasicHttpRequest request = new BasicHttpRequest("QUERY", "someUri"); + final RequestCacheControl cacheControl = RequestCacheControl.builder().build(); + + Assertions.assertTrue(policy.canBeServedFromCache(cacheControl, request)); + } + + @Test + void testIsQueryWithCacheControlServableFromCache() { + final BasicHttpRequest request = new BasicHttpRequest("QUERY", "someUri"); + final RequestCacheControl cacheControl = RequestCacheControl.builder() + .setNoCache(true) + .build(); + + Assertions.assertFalse(policy.canBeServedFromCache(cacheControl, request)); + } + } diff --git a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCachingExecChain.java b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCachingExecChain.java index 4af20bb94d..608b8f4fc6 100644 --- a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCachingExecChain.java +++ b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestCachingExecChain.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.mock; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.SocketException; @@ -55,12 +56,18 @@ import org.apache.hc.client5.http.validator.ETag; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.Method; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.HttpEntityWrapper; import org.apache.hc.core5.http.io.entity.InputStreamEntity; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; import org.apache.hc.core5.http.message.BasicClassicHttpRequest; import org.apache.hc.core5.http.message.BasicClassicHttpResponse; @@ -1291,4 +1298,115 @@ void testNoCacheFieldsRevalidation() throws Exception { Mockito.verify(mockExecChain, Mockito.times(5)).proceed(Mockito.any(), Mockito.any()); } + @Test + void testPrepareRequestBuffersQueryContent() throws Exception { + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(new StringEntity("select a", ContentType.TEXT_PLAIN)); + + final SimpleHttpRequest converted = impl.prepareRequest(query); + + Assertions.assertNotNull(converted); + Assertions.assertNotNull(converted.getBody()); + Assertions.assertEquals("select a", converted.getBody().getBodyText()); + Assertions.assertEquals(ContentType.TEXT_PLAIN.getMimeType(), + converted.getBody().getContentType().getMimeType()); + } + + @Test + void testPrepareRequestBypassesNonQueryRequestsWithContent() throws Exception { + final ClassicHttpRequest post = new BasicClassicHttpRequest("POST", "/stuff"); + post.setEntity(new StringEntity("stuff", ContentType.TEXT_PLAIN)); + + Assertions.assertNull(impl.prepareRequest(post)); + } + + @Test + void testPrepareRequestHandlesQueryEntityWithoutContent() throws Exception { + final HttpEntity entity = mock(HttpEntity.class); + Mockito.when(entity.isRepeatable()).thenReturn(true); + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(entity); + + final SimpleHttpRequest converted = impl.prepareRequest(query); + + Assertions.assertNotNull(converted); + Assertions.assertNull(converted.getBody()); + } + + @Test + void testPrepareRequestBypassesNonRepeatableQueryContent() throws Exception { + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(new InputStreamEntity( + new ByteArrayInputStream(new byte[16]), ContentType.TEXT_PLAIN)); + + Assertions.assertNull(impl.prepareRequest(query)); + } + + @Test + void testPrepareRequestBypassesOversizedQueryContent() throws Exception { + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(new ByteArrayEntity( + new byte[CachingExecBase.MAX_BUFFERED_CONTENT_LENGTH + 1], ContentType.TEXT_PLAIN)); + + Assertions.assertNull(impl.prepareRequest(query)); + } + + @Test + void testPrepareRequestBypassesOversizedQueryContentOfUnknownLength() throws Exception { + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(new HttpEntityWrapper(new ByteArrayEntity( + new byte[CachingExecBase.MAX_BUFFERED_CONTENT_LENGTH + 1], ContentType.TEXT_PLAIN)) { + + @Override + public long getContentLength() { + return -1; + } + + }); + + Assertions.assertNull(impl.prepareRequest(query)); + } + + private static ClassicHttpRequest makeQueryRequest(final String queryContent) { + final ClassicHttpRequest query = new BasicClassicHttpRequest("QUERY", "/stuff"); + query.setEntity(new StringEntity(queryContent, ContentType.TEXT_PLAIN)); + return query; + } + + @Test + void testQueryResponseServedFromCache() throws Exception { + // The cache relies on QUERY being safe for entry invalidation purposes + Assertions.assertTrue(Method.QUERY.isSafe()); + + final ClassicHttpResponse resp1 = HttpTestUtils.make200Response(); + resp1.setHeader("Cache-Control", "max-age=3600"); + + Mockito.when(mockExecChain.proceed(Mockito.any(), Mockito.any())).thenReturn(resp1); + + final ClassicHttpResponse result1 = execute(makeQueryRequest("select a")); + EntityUtils.consume(result1.getEntity()); + final ClassicHttpResponse result2 = execute(makeQueryRequest("select a")); + + Assertions.assertEquals(HttpStatus.SC_OK, result2.getCode()); + Mockito.verify(mockExecChain, Mockito.times(1)).proceed(Mockito.any(), Mockito.any()); + Assertions.assertEquals(CacheResponseStatus.CACHE_HIT, context.getCacheResponseStatus()); + } + + @Test + void testQueryWithDifferentContentNotServedFromCache() throws Exception { + final ClassicHttpResponse resp1 = HttpTestUtils.make200Response(); + resp1.setHeader("Cache-Control", "max-age=3600"); + final ClassicHttpResponse resp2 = HttpTestUtils.make200Response(); + resp2.setHeader("Cache-Control", "max-age=3600"); + + Mockito.when(mockExecChain.proceed(Mockito.any(), Mockito.any())).thenReturn(resp1, resp2); + + final ClassicHttpResponse result1 = execute(makeQueryRequest("select a")); + EntityUtils.consume(result1.getEntity()); + final ClassicHttpResponse result2 = execute(makeQueryRequest("select b")); + EntityUtils.consume(result2.getEntity()); + + Mockito.verify(mockExecChain, Mockito.times(2)).proceed(Mockito.any(), Mockito.any()); + } + } diff --git a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestResponseCachingPolicy.java b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestResponseCachingPolicy.java index fe263e9065..ec90345c8a 100644 --- a/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestResponseCachingPolicy.java +++ b/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/TestResponseCachingPolicy.java @@ -90,6 +90,13 @@ void testHeadCacheable() { Assertions.assertTrue(policy.isResponseCacheable(responseCacheControl, request, response)); } + @Test + void testQueryCacheable() { + policy = new ResponseCachingPolicy(true, false, false); + request = new BasicHttpRequest(Method.QUERY, "/"); + Assertions.assertTrue(policy.isResponseCacheable(responseCacheControl, request, response)); + } + @Test void testArbitraryMethodNotCacheable() { request = new BasicHttpRequest("PUT", "/"); diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/ClientQueryMethod.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/ClientQueryMethod.java new file mode 100644 index 0000000000..7cbda480ad --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/ClientQueryMethod.java @@ -0,0 +1,58 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.examples; + +import org.apache.hc.client5.http.classic.methods.HttpQuery; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.StatusLine; + +/** + * Example of a safe and idempotent {@code QUERY} request (RFC 10008) with + * an enclosed request body. + */ +public class ClientQueryMethod { + + public static void main(final String[] args) throws Exception { + try (final CloseableHttpClient httpclient = HttpClients.createDefault()) { + final HttpQuery httpQuery = new HttpQuery("http://httpbin.org/anything"); + httpQuery.setEntity(new StringEntity("select a, b from things", ContentType.TEXT_PLAIN)); + + System.out.println("Executing request " + httpQuery.getMethod() + " " + httpQuery.getUri()); + httpclient.execute(httpQuery, response -> { + System.out.println("----------------------------------------"); + System.out.println(httpQuery + "->" + new StatusLine(response)); + System.out.println(EntityUtils.toString(response.getEntity())); + return null; + }); + } + } + +}