Add support for request compression #968
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.java.client.http.compression; | ||
|
|
||
| import java.util.List; | ||
| import software.amazon.smithy.java.io.datastream.DataStream; | ||
|
|
||
| /** | ||
| * Represents a compression algorithm that can be used to compress request | ||
| * bodies. | ||
| */ | ||
| public interface CompressionAlgorithm { | ||
joewyz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| /** | ||
| * The ID of the compression algorithm. This is matched against the algorithm | ||
| * names used in the trait e.g. "gzip" | ||
| */ | ||
| String algorithmId(); | ||
|
|
||
| /** | ||
| * Compresses content of fixed length | ||
| */ | ||
| DataStream compress(DataStream data); | ||
|
|
||
| List<CompressionAlgorithm> SUPPORTED_ALGORITHMS = List.of(new Gzip()); | ||
|
|
||
| static List<CompressionAlgorithm> supportedAlgorithms() { | ||
| return SUPPORTED_ALGORITHMS; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.java.client.http.compression; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.UncheckedIOException; | ||
| import java.util.zip.GZIPOutputStream; | ||
| import software.amazon.smithy.java.io.ByteBufferOutputStream; | ||
| import software.amazon.smithy.java.io.datastream.DataStream; | ||
|
|
||
| public final class Gzip implements CompressionAlgorithm { | ||
|
|
||
| @Override | ||
| public String algorithmId() { | ||
| return "gzip"; | ||
| } | ||
|
|
||
| @Override | ||
| public DataStream compress(DataStream data) { | ||
| if (!data.hasKnownLength()) { // Using streaming | ||
joewyz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return DataStream.ofInputStream( | ||
| new GzipCompressingInputStream(data.asInputStream()), | ||
| data.contentType(), | ||
| -1); | ||
| } | ||
|
|
||
| try (var bos = new ByteBufferOutputStream(); | ||
| var in = data.asInputStream()) { | ||
| var gzip = new GZIPOutputStream(bos); | ||
| in.transferTo(gzip); | ||
| gzip.close(); | ||
joewyz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return DataStream.ofBytes(bos.toByteBuffer().array()); | ||
| } catch (IOException e) { | ||
| throw new UncheckedIOException(e); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.java.client.http.compression; | ||
|
|
||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import java.io.UncheckedIOException; | ||
| import java.util.zip.GZIPOutputStream; | ||
|
|
||
| /** | ||
| * An InputStream that compresses data from a source InputStream using GZIP compression. | ||
| * This implementation lazily compress from the source data on-demand as it's read. | ||
| */ | ||
| final class GzipCompressingInputStream extends InputStream { | ||
|
Contributor
There was a problem hiding this comment. This implementation underneath still reads the whole input stream into memory to do the compression. |
||
| private static final int CHUNK_SIZE = 8192; | ||
| private final InputStream source; | ||
| private final ByteArrayOutputStream bufferStream; | ||
| private final GZIPOutputStream gzipStream; | ||
| private final byte[] chunk = new byte[CHUNK_SIZE]; | ||
| private byte[] buffer; | ||
| private int bufferPos; | ||
| private int bufferLimit; | ||
| private boolean sourceExhausted; | ||
| private boolean closed; | ||
|
|
||
| public GzipCompressingInputStream(InputStream source) { | ||
| this.source = source; | ||
| this.bufferStream = new ByteArrayOutputStream(); | ||
| this.gzipStream = createGzipOutputStream(bufferStream); | ||
| this.buffer = new byte[0]; | ||
| this.bufferPos = 0; | ||
| this.bufferLimit = 0; | ||
| this.sourceExhausted = false; | ||
| this.closed = false; | ||
| } | ||
|
|
||
| @Override | ||
| public int read() throws IOException { | ||
| byte[] b = new byte[1]; | ||
| int result = read(b, 0, 1); | ||
| return result == -1 ? -1 : (b[0] & 0xFF); | ||
| } | ||
|
|
||
| @Override | ||
| public int read(byte[] b, int off, int len) throws IOException { | ||
| if (closed) { | ||
| throw new IOException("Stream closed"); | ||
| } | ||
|
|
||
| if (b == null) { | ||
| throw new NullPointerException("b"); | ||
| } else if (off < 0 || len < 0 || len > b.length - off) { | ||
| throw new IndexOutOfBoundsException(); | ||
| } else if (len == 0) { | ||
| return 0; | ||
| } | ||
|
|
||
| // Try to fill the output buffer if it's empty | ||
| while (bufferPos >= bufferLimit) { | ||
| if (!fillBuffer()) { | ||
| return -1; // End of stream | ||
| } | ||
| } | ||
|
|
||
| // Copy available data from buffer | ||
| int available = bufferLimit - bufferPos; | ||
| int toRead = Math.min(available, len); | ||
| System.arraycopy(buffer, bufferPos, b, off, toRead); | ||
| bufferPos += toRead; | ||
|
|
||
| return toRead; | ||
| } | ||
|
|
||
| /** | ||
| * Reads a chunk from the source, compresses it, and fills the internal buffer. | ||
| * | ||
| * @return true if data was added to buffer, false if end of stream reached | ||
| */ | ||
| private boolean fillBuffer() throws IOException { | ||
| if (sourceExhausted) { | ||
| return false; | ||
| } | ||
|
|
||
| // Read a chunk from source | ||
| int bytesRead = source.read(chunk); | ||
|
|
||
| if (bytesRead == -1) { | ||
| // Source is exhausted, finish compression | ||
| gzipStream.finish(); | ||
| sourceExhausted = true; | ||
| } else { | ||
| // Compress the chunk | ||
| gzipStream.write(chunk, 0, bytesRead); | ||
| gzipStream.flush(); | ||
| } | ||
|
|
||
| // Get compressed data from buffer stream | ||
| byte[] compressed = bufferStream.toByteArray(); | ||
| if (compressed.length > 0) { | ||
| buffer = compressed; | ||
| bufferPos = 0; | ||
| bufferLimit = compressed.length; | ||
| bufferStream.reset(); | ||
| return true; | ||
| } | ||
|
|
||
| if (sourceExhausted) { | ||
| return bufferPos >= bufferLimit; | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
| if (!closed) { | ||
| closed = true; | ||
| try { | ||
| gzipStream.close(); | ||
| } finally { | ||
| source.close(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public int available() throws IOException { | ||
| if (closed) { | ||
| throw new IOException("Stream closed"); | ||
| } | ||
| return bufferLimit - bufferPos; | ||
| } | ||
|
|
||
| /** | ||
| * Utility method to avoid having to throw the checked IOException exception. | ||
| */ | ||
| private GZIPOutputStream createGzipOutputStream(OutputStream bufferStream) { | ||
| try { | ||
| return new GZIPOutputStream(bufferStream); | ||
| } catch (IOException e) { | ||
| throw new UncheckedIOException(e); | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.