Skip to content

Commit

Permalink
Accept output buffer in Response::Body#readpartial
Browse files Browse the repository at this point in the history
This makes it possible to use a `HTTP::Response::Body` object as a
source argument in `IO.copy_stream`, allowing easy streaming.

  response = HTTP.get("http://example.com/download")
  IO.copy_stream(response.body, "destination.txt")

Since `IO.copy_stream` uses an output buffer, it makes retrieving the
response body much more memory efficient, because we can now deallocate
retrieved chunks.
  • Loading branch information
janko committed May 25, 2018
1 parent 2bd7f84 commit a06be0b
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 6 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ env: JRUBY_OPTS="$JRUBY_OPTS --debug"

rvm:
# Include JRuby first because it takes the longest
- jruby-9.1.16.0
- jruby-9.2.0.0
- 2.3
- 2.4
- 2.5
Expand Down
3 changes: 2 additions & 1 deletion lib/http/connection.rb
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,15 @@ def send_request(req)
def readpartial(size = BUFFER_SIZE)
return unless @pending_response

# return chunk of body that was retrieved when reading response headers
chunk = @parser.read(size)
return chunk if chunk

finished = (read_more(size) == :eof) || @parser.finished?
chunk = @parser.read(size)
finish_response if finished

chunk.to_s
chunk
end

# Reads data from socket up until headers are loaded
Expand Down
16 changes: 13 additions & 3 deletions lib/http/response/body.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,21 @@ def initialize(stream, encoding: Encoding::BINARY)
@encoding = find_encoding(encoding)
end

# (see HTTP::Client#readpartial)
def readpartial(*args)
# (see HTTP::Connection#readpartial)
def readpartial(length = nil, outbuf = nil)
stream!
chunk = @stream.readpartial(*args)
chunk = @stream.readpartial(*length)
chunk.force_encoding(@encoding) if chunk

if outbuf
outbuf.clear.force_encoding(@encoding)
raise EOFError if chunk.nil? # IO.copy_stream expects this to be raised
outbuf << chunk
chunk.clear
outbuf
else
chunk
end
end

# Iterate over the body, allowing it to be enumerable
Expand Down
24 changes: 23 additions & 1 deletion spec/lib/http/response/body_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

before do
allow(connection).to receive(:readpartial) { chunks.shift }
allow(connection).to receive(:body_completed?) { chunks.empty? }
end

subject(:body) { described_class.new(connection, :encoding => Encoding::UTF_8) }
Expand Down Expand Up @@ -42,6 +41,29 @@
end
end

context "with size and output buffer given" do
it "fills the output buffer" do
expect(connection).to receive(:readpartial).with(3).and_return(String.new("foo"))
outbuf = String.new
chunk = body.readpartial 3, outbuf
expect(outbuf).to eq "foo"
expect(chunk).to equal outbuf
end

it "returns nil when there is no more content" do
outbuf = String.new
expect(body.readpartial(7, outbuf)).to eq "Hello, "
expect(body.readpartial(6, outbuf)).to eq "World!"
expect { body.readpartial(1, outbuf) }.to raise_error(EOFError)
end

it "supports IO.copy_stream" do
stringio = StringIO.new
IO.copy_stream(body, stringio)
expect(stringio.string).to eq "Hello, World!"
end
end

it "returns content in specified encoding" do
body = described_class.new(connection)
expect(connection).to receive(:readpartial).
Expand Down

0 comments on commit a06be0b

Please sign in to comment.