Streaming JSON results.

Gemfile

@@ -1,3 +1,6 @@
 source "http://rubygems.org"
 
 gemspec
+
+gem "thread_queues", path: "/home/kaspernj/Dev/Ruby/thread_queues"
+gem "json_streamer", path: "/home/kaspernj/Dev/Ruby/json_streamer"

Gemfile.lock

@@ -3,6 +3,13 @@ PATH
   specs:
     http2 (0.0.29)
       string-cases
+      thread_queues (>= 0.0.2)
+
+PATH
+  remote: /home/kaspernj/Dev/Ruby/thread_queues
+  specs:
+    thread_queues (0.0.3)
+      string-cases
 
 GEM
   remote: http://rubygems.org/
@@ -62,6 +69,7 @@ GEM
     tpool (0.0.4)
     tsafe (0.0.11)
     wref (0.0.6)
+    yajl-ruby (1.2.1)
 
 PLATFORMS
   ruby
@@ -75,3 +83,5 @@ DEPENDENCIES
   rdoc (~> 3.12)
   rspec (~> 2.8.0)
   sqlite3
+  thread_queues!
+  yajl-ruby

http2.gemspec

@@ -24,12 +24,13 @@ Gem::Specification.new do |s|
   s.summary = "A lightweight framework for doing http-connections in Ruby. Supports cookies, keep-alive, compressing and much more."
 
   s.add_runtime_dependency("string-cases", ">= 0")
+  s.add_runtime_dependency("thread_queues", ">= 0.0.2")
   s.add_development_dependency("rake")
   s.add_development_dependency("rspec", "~> 2.8.0")
   s.add_development_dependency("rdoc", "~> 3.12")
   s.add_development_dependency("bundler", ">= 1.0.0")
   s.add_development_dependency("hayabusa", ">= 0.0.25")
   s.add_development_dependency("sqlite3")
   s.add_development_dependency("codeclimate-test-reporter")
+  s.add_development_dependency("json_streamer")
 end
-

include/connection.rb

@@ -1,6 +1,6 @@
 class Http2::Connection
   def initialize(http2)
-    @http2, @debug, @args = http2, http2.debug, http2.args
+    @http2, @debug, @args = http2, http2.debug?, http2.args
     reconnect
   end
 
@@ -59,15 +59,15 @@ def write(str)
 
   # Reconnects to the host.
   def reconnect
-    puts "Http2: Reconnect." if @debug
+    @http2.debug "Reconnect." if @debug
 
     #Open connection.
     if @args[:proxy] && @args[:ssl]
       connect_proxy_ssl
     elsif @args[:proxy]
       connect_proxy
     else
-      puts "Http2: Opening socket connection to '#{@http2.host}:#{@http2.port}'." if @debug
+      @http2.debug "Opening socket connection to '#{@http2.host}:#{@http2.port}'." if @debug
       @sock_plain = TCPSocket.new(@http2.host, @http2.port)
     end
 
@@ -87,7 +87,7 @@ def socket_working?
     if @keepalive_timeout && @request_last
       between = Time.now.to_i - @request_last.to_i
       if between >= @keepalive_timeout
-        puts "Http2: We are over the keepalive-wait - returning false for socket_working?." if @debug
+        @http2.debug "We are over the keepalive-wait - returning false for socket_working?." if @debug
         return false
       end
     end
@@ -103,7 +103,7 @@ def close
   end
 
   def connect_proxy_ssl
-    puts "Http2: Initializing proxy stuff." if @debug
+    @http2.debug "Initializing proxy stuff." if @debug
     @sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port])
 
     @sock_plain.write("CONNECT #{@args[:host]}:#{@args[:port]} HTTP/1.0#{@nl}")
@@ -122,12 +122,12 @@ def connect_proxy_ssl
   end
 
   def connect_proxy
-    puts "Http2: Opening socket connection to '#{@args[:host]}:#{@args[:port]}' through proxy '#{@args[:proxy][:host]}:#{@args[:proxy][:port]}'." if @debug
+    @http2.debug "Opening socket connection to '#{@args[:host]}:#{@args[:port]}' through proxy '#{@args[:proxy][:host]}:#{@args[:proxy][:port]}'." if @debug
     @sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port].to_i)
   end
 
   def apply_ssl
-    puts "Http2: Initializing SSL." if @debug
+    @http2.debug "Initializing SSL." if @debug
     require "openssl" unless ::Kernel.const_defined?(:OpenSSL)
 
     ssl_context = OpenSSL::SSL::SSLContext.new

include/get_request.rb

@@ -1,17 +1,17 @@
 class Http2::GetRequest
   def initialize(http2, args)
-    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug, http2.nl
+    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug?, http2.nl
   end
 
   def execute
     @http2.mutex.synchronize do
-      puts "Http2: Writing headers: #{header_string}" if @debug
+      @http2.debug "Writing headers: #{header_string}" if @debug
       @http2.connection.write(header_string)
 
-      puts "Http2: Reading response." if @debug
+      @http2.debug "Reading response." if @debug
       resp = @http2.read_response(@args)
 
-      puts "Http2: Done with get request." if @debug
+      @http2.debug "Done with get request." if @debug
       return resp
     end
   end

include/post_multipart_request.rb

@@ -11,7 +11,7 @@ def initialize(http2, *args)
 
   def execute
     generate_raw(@phash) do |helper, praw|
-      puts "Http2: Header string: #{header_string}" if @debug
+      @http2.debug "Header string: #{header_string}" if @debug
 
       @http2.mutex.synchronize do
         @conn.write(header_string(praw))

include/post_request.rb

@@ -1,21 +1,21 @@
 class Http2::PostRequest
-  VALID_ARGUMENTS_POST = [:post, :url, :default_headers, :headers, :json, :method, :cookies, :on_content, :content_type]
+  VALID_ARGUMENTS_POST = [:body_as, :post, :url, :default_headers, :headers, :json, :method, :cookies, :on_content, :content_type]
 
   def initialize(http2, args)
     args.each do |key, val|
       raise "Invalid key: '#{key}'." unless VALID_ARGUMENTS_POST.include?(key)
     end
 
-    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug, http2.nl
+    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug?, http2.nl
     @conn = @http2.connection
   end
 
   def execute
     @data = raw_data
 
     @http2.mutex.synchronize do
-      puts "Http2: Doing post." if @debug
-      puts "Http2: Header str: #{header_str}" if @debug
+      @http2.debug "Doing post." if @debug
+      @http2.debug "Header str: #{header_str}" if @debug
 
       @conn.write(header_string)
       return @http2.read_response(@args)

include/response.rb

@@ -1,15 +1,64 @@
 #This object will be returned as the response for each request.
 class Http2::Response
   #All the data the response contains. Headers, body, cookies, requested URL and more.
-  attr_reader :args
+  attr_reader :args, :tempfile, :buffer
   attr_accessor :body, :charset, :code, :content_type, :http_version
 
   #This method should not be called manually.
   def initialize(args = {})
     @args = args
     @args[:headers] = {} unless @args.key?(:headers)
-    @body = args[:body] || ""
     @debug = @args[:debug]
+
+    if args[:request_args][:args][:body_as] == :tempfile
+      @tempfile = Tempfile.new
+    elsif args[:request_args][:args][:body_as] == :buffer
+      require "thread_queues"
+      @queue = ThreadQueues::BufferedQueue.new(50)
+      @buffer = ThreadQueues::StringBuffer.new(@queue)
+    else
+      @body = ""
+    end
+  end
+
+  def buffer
+    if @buffer
+      if block_given?
+        begin
+          yield @buffer
+        ensure
+          @queue.close
+        end
+      else
+        return @buffer
+      end
+    else
+      raise "Not in buffer-mode"
+    end
+  end
+
+  def tempfile
+    if @tempfile
+      return @tempfile
+    else
+      raise "Not in tempfile-mode"
+    end
+  end
+
+  def body
+    if body?
+      return @body
+    else
+      raise "Not in body-mode"
+    end
+  end
+
+  def body?
+    return true if @body
+  end
+
+  def finish
+    @queue.close if @queue
   end
 
   #Returns headers given from the host for the result.
@@ -45,6 +94,16 @@ def content_length
     end
   end
 
+  def content_length?
+    if header?("content-length")
+      return true
+    elsif @body
+      return true
+    end
+
+    return false
+  end
+
   #Returns the requested URL as a string.
   #===Examples
   # res.requested_url #=> "?show=status&action=getstatus"
@@ -55,25 +114,46 @@ def requested_url
 
   # Checks the data that has been sat on the object and raises various exceptions, if it does not validate somehow.
   def validate!
-    puts "Http2: Validating response length." if @debug
+    @http2.debug "Validating response length." if @debug
     validate_body_versus_content_length!
   end
 
+  def add_to_body(obj)
+    if @tempfile
+      @tempfile.write(obj)
+    elsif @queue
+      @queue.push(obj)
+    elsif @body
+      @body << obj
+    else
+      raise "Don't know how to add to body?"
+    end
+  end
+
 private
 
   # Checks that the length of the body is the same as the given content-length if given.
   def validate_body_versus_content_length!
     unless self.header?("content-length")
-      puts "Http2: No content length given - skipping length validation." if @debug
+      @http2.debug "No content length given - skipping length validation." if @debug
       return nil
     end
 
-    content_length = header("content-length").to_i
-    body_length = @body.bytesize
+    if @body
+      body_length = @body.bytesize
+    elsif @tempfile
+      body_length File.size(@tempfile.path)
+    end
+
+    if body_length
+      content_length = header("content-length").to_i
 
-    puts "Http2: Body length: #{body_length}" if @debug
-    puts "Http2: Content length: #{content_length}" if @debug
+      @http2.debug "Body length: #{body_length}" if @debug
+      @http2.debug "Content length: #{content_length}" if @debug
 
-    raise "Body does not match the given content-length: '#{body_length}', '#{content_length}'." if body_length != content_length
+      if body_length != content_length
+        raise "Body does not match the given content-length: '#{body_length}', '#{content_length}'."
+      end
+    end
   end
 end