Add CURLOPT_COOKIELIST support (#455)

* Update curb_easy.c Add `CURLOPT_COOKIELIST` support * Fix doc comments * Fix typos * Improve #cookielist docs * Add tests for CURLOPT_COOKIELIST * assert that file is written by FLUSH * fix typos * more tests
taf2 · Oct 11, 2024 · 96b874f · 96b874f
1 parent 8b01e0f
commit 96b874f
Show file tree

Hide file tree

Showing 4 changed files with 310 additions and 9 deletions.
diff --git a/curb.gemspec b/curb.gemspec
@@ -2,7 +2,7 @@ Gem::Specification.new do |s|
   s.name    = "curb"
   s.authors = ["Ross Bamford", "Todd A. Fisher"]
   s.version = '1.0.6'
-  s.date    = '2024-08-23'
+  s.date    = '2024-10-02'
   s.description = %q{Curb (probably CUrl-RuBy or something) provides Ruby-language bindings for the libcurl(3), a fully-featured client-side URL transfer library. cURL and libcurl live at http://curl.haxx.se/}
   s.email   = '[email protected]'
   s.extra_rdoc_files = ['LICENSE', 'README.markdown']
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   #### Load-time details
   s.require_paths = ['lib','ext']
   s.summary = %q{Ruby libcurl bindings}
-  s.test_files = ["tests/alltests.rb", "tests/bug_crash_on_debug.rb", "tests/bug_crash_on_progress.rb", "tests/bug_curb_easy_blocks_ruby_threads.rb", "tests/bug_curb_easy_post_with_string_no_content_length_header.rb", "tests/bug_follow_redirect_288.rb", "tests/bug_instance_post_differs_from_class_post.rb", "tests/bug_issue102.rb", "tests/bug_issue277.rb", "tests/bug_multi_segfault.rb", "tests/bug_postfields_crash.rb", "tests/bug_postfields_crash2.rb", "tests/bug_raise_on_callback.rb", "tests/bug_require_last_or_segfault.rb", "tests/bugtests.rb", "tests/helper.rb", "tests/mem_check.rb", "tests/require_last_or_segfault_script.rb", "tests/signals.rb", "tests/tc_curl.rb", "tests/tc_curl_download.rb", "tests/tc_curl_easy.rb", "tests/tc_curl_easy_resolve.rb", "tests/tc_curl_easy_setopt.rb", "tests/tc_curl_maxfilesize.rb", "tests/tc_curl_multi.rb", "tests/tc_curl_postfield.rb", "tests/tc_curl_protocols.rb", "tests/timeout.rb", "tests/timeout_server.rb", "tests/unittests.rb"]
+  s.test_files = ["tests/alltests.rb", "tests/bug_crash_on_debug.rb", "tests/bug_crash_on_progress.rb", "tests/bug_curb_easy_blocks_ruby_threads.rb", "tests/bug_curb_easy_post_with_string_no_content_length_header.rb", "tests/bug_follow_redirect_288.rb", "tests/bug_instance_post_differs_from_class_post.rb", "tests/bug_issue102.rb", "tests/bug_multi_segfault.rb", "tests/bug_postfields_crash.rb", "tests/bug_postfields_crash2.rb", "tests/bug_raise_on_callback.rb", "tests/bug_require_last_or_segfault.rb", "tests/bugtests.rb", "tests/helper.rb", "tests/mem_check.rb", "tests/require_last_or_segfault_script.rb", "tests/signals.rb", "tests/tc_curl.rb", "tests/tc_curl_download.rb", "tests/tc_curl_easy.rb", "tests/tc_curl_easy_cookielist.rb", "tests/tc_curl_easy_resolve.rb", "tests/tc_curl_easy_setopt.rb", "tests/tc_curl_maxfilesize.rb", "tests/tc_curl_multi.rb", "tests/tc_curl_postfield.rb", "tests/tc_curl_protocols.rb", "tests/timeout.rb", "tests/timeout_server.rb", "tests/unittests.rb"]
 
   s.extensions << 'ext/extconf.rb'
 

diff --git a/ext/curb_easy.c b/ext/curb_easy.c
@@ -3364,14 +3364,16 @@ static VALUE ruby_curl_easy_num_connects_get(VALUE self) {
 
 /*
  * call-seq:
- *   easy.cookielist                                => array
+ *   easy.cookielist                                => cookielist
  *
  * Retrieves the cookies curl knows in an array of strings.
  * Returned strings are in Netscape cookiejar format or in Set-Cookie format.
+ * Since 7.43.0 cookies in the Set-Cookie format without a domain name are not exported.
  *
- * See also option CURLINFO_COOKIELIST of curl_easy_getopt(3) to see how libcurl behaves.
- *
- * (requires libcurl 7.14.1 or higher, otherwise -1 is always returned).
+ * @see https://curl.se/libcurl/c/CURLINFO_COOKIELIST.html option <code>CURLINFO_COOKIELIST</code> of 
+ *   <code>curl_easy_getopt(3)</code> to see how libcurl behaves.
+ * @note requires libcurl 7.14.1 or higher, otherwise +-1+ is always returned
+ * @return [Array<String>, nil, -1] array of strings, or +nil+ if there are no cookies, or +-1+ if the libcurl version is too old
 */
 static VALUE ruby_curl_easy_cookielist_get(VALUE self) {
 #ifdef HAVE_CURLINFO_COOKIELIST
@@ -3482,9 +3484,16 @@ static VALUE ruby_curl_easy_last_error(VALUE self) {
 
 /*
  * call-seq:
- *   easy.setopt Fixnum, value  => value
+ *   easy.setopt(opt, val)  => val
  *
  * Initial access to libcurl curl_easy_setopt
+ *
+ * @param [Fixnum] opt The option to set, see +Curl::CURLOPT_*+ constants
+ * @param [Object] val
+ * @return [Object] val
+ * @raise [TypeError] if the option is not supported
+ * @note Some options - like url or cookie - aren't set directly throught +curl_easy_setopt+, but stored in the Ruby object state.
+ * @note When +curl_easy_setopt+ is called, return value is not checked here.
  */
 static VALUE ruby_curl_easy_set_opt(VALUE self, VALUE opt, VALUE val) {
   ruby_curl_easy *rbce;
@@ -3650,6 +3659,11 @@ static VALUE ruby_curl_easy_set_opt(VALUE self, VALUE opt, VALUE val) {
     curl_easy_setopt(rbce->curl, CURLOPT_SSL_SESSIONID_CACHE, NUM2LONG(val));
     break;
 #endif
+#if HAVE_CURLOPT_COOKIELIST
+  case CURLOPT_COOKIELIST: {
+	curl_easy_setopt(rbce->curl, CURLOPT_COOKIELIST, StringValueCStr(val));
+    } break;
+#endif
 #if HAVE_CURLOPT_PROXY_SSL_VERIFYHOST
   case CURLOPT_PROXY_SSL_VERIFYHOST:
     curl_easy_setopt(rbce->curl, CURLOPT_PROXY_SSL_VERIFYHOST, NUM2LONG(val));
@@ -3664,9 +3678,13 @@ static VALUE ruby_curl_easy_set_opt(VALUE self, VALUE opt, VALUE val) {
 
 /*
  * call-seq:
- *   easy.getinfo Fixnum => value
+ *   easy.getinfo(opt) => nil
  *
  * Iniital access to libcurl curl_easy_getinfo, remember getinfo doesn't return the same values as setopt
+ *
+ * @note This method is not implemented yet.
+ * @param [Fixnum] code Constant +CURLINFO_*+ from libcurl
+ * @return [nil]
  */
 static VALUE ruby_curl_easy_get_opt(VALUE self, VALUE opt) {
   return Qnil;

diff --git a/tests/helper.rb b/tests/helper.rb
@@ -80,6 +80,10 @@ def do_GET(req,res)
       res.status = 404
     elsif req.path.match(/error$/)
       res.status = 500
+    elsif req.path.match(/get_cookies$/)
+      res['Content-Type'] = "text/plain"
+      res.body = req['Cookie']
+      return
     end
     respond_with("GET#{req.query_string}",req,res)
   end
@@ -90,7 +94,18 @@ def do_HEAD(req,res)
   end
 
   def do_POST(req,res)
-    if req.query['filename'].nil?
+    if req.path.match(/set_cookies$/)
+      JSON.parse(req.body || '[]', symbolize_names: true).each do |hash|
+        cookie = WEBrick::Cookie.new(hash.fetch(:name), hash.fetch(:value))
+        cookie.domain = hash[:domain] if hash.key?(:domain)
+        cookie.expires = hash[:expires] if hash.key?(:expires)
+        cookie.path = hash[:path] if hash.key?(:path)
+        cookie.secure = hash[:secure] if hash.key?(:secure)
+        cookie.max_age = hash[:max_age] if hash.key?(:max_age)
+        res.cookies.push(cookie)
+      end
+      respond_with('OK', req, res)
+    elsif req.query['filename'].nil?
       if req.body
         params = {}
         req.body.split('&').map{|s| k,v=s.split('='); params[k] = v }

diff --git a/tests/tc_curl_easy_cookielist.rb b/tests/tc_curl_easy_cookielist.rb
@@ -0,0 +1,268 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'json'
+
+class TestCurbCurlEasyCookielist < Test::Unit::TestCase
+  def test_setopt_cookielist
+    easy = Curl::Easy.new
+    # DateTime handles time zone correctly
+    expires = (Date.today + 2).to_datetime
+    easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c1=v1; domain=localhost; expires=#{expires.httpdate};")
+    easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c2=v2; domain=localhost')
+    easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c3=v3; expires=#{expires.httpdate};")
+    easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c4=v4;')
+    easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c5=v5; domain=127.0.0.1; expires=#{expires.httpdate};")
+    easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c6=v6; domain=127.0.0.1;;')
+
+    # Since 7.43.0 cookies that were imported in the Set-Cookie format without a domain name are not exported by this option.
+    # So, before 7.43.0, c3 and c4 will be exported too; but that version is far too old for current curb version, so it's not handled here.
+    expected_cookielist = [
+      "127.0.0.1\tFALSE\t/\tFALSE\t#{expires.to_time.to_i}\tc5\tv5",
+      "127.0.0.1\tFALSE\t/\tFALSE\t0\tc6\tv6",
+      ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1",
+      ".localhost\tTRUE\t/\tFALSE\t0\tc2\tv2",
+    ]
+    assert_equal expected_cookielist, easy.cookielist
+
+    easy.url = "#{TestServlet.url}/get_cookies"
+    easy.perform
+    assert_equal 'c6=v6; c5=v5; c4=v4; c3=v3', easy.body_str
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    easy.perform
+    assert_equal 'c2=v2; c1=v1', easy.body_str
+  end
+
+  # libcurl documentation says: "This option also enables the cookie engine", but it's not tracked on the curb level
+  def test_setopt_cookielist_enables_cookie_engine
+    easy = Curl::Easy.new
+    expires = (Date.today + 2).to_datetime
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies"
+    easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c1=v1; domain=localhost; expires=#{expires.httpdate};")
+    easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }])
+    easy.perform
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    easy.post_body = nil
+    easy.perform
+
+    assert !easy.enable_cookies?
+    assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
+    assert_equal 'c2=v2; c1=v1', easy.body_str
+  end
+
+  def test_setopt_cookielist_invalid_format
+    easy = Curl::Easy.new
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    easy.setopt(Curl::CURLOPT_COOKIELIST, 'Not a cookie')
+    assert_nil easy.cookielist
+    easy.perform
+    assert_equal '', easy.body_str
+  end
+
+  def test_setopt_cookielist_netscape_format
+    easy = Curl::Easy.new
+    expires = (Date.today + 2).to_datetime
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    # Note domain changes for include subdomains
+    [
+      ['localhost', 'FALSE', '/', 'TRUE', 0, 'session_http_only', '42'].join("\t"),
+      ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '43'].join("\t"),
+      ['localhost', 'TRUE', '/', 'FALSE', expires.to_time.to_i, 'permanent', '44'].join("\t"),
+      ['.localhost', 'FALSE', '/', 'TRUE', expires.to_time.to_i, 'permanent_http_only', '45'].join("\t"),
+    ].each { |cookie| easy.setopt(Curl::CURLOPT_COOKIELIST, cookie) }
+
+    expected_cookielist = [
+      ['localhost', 'FALSE', '/', 'TRUE', 0, 'session_http_only', '42'].join("\t"),
+      ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '43'].join("\t"),
+      ['.localhost', 'TRUE', '/', 'FALSE', expires.to_time.to_i, 'permanent', '44'].join("\t"),
+      ['localhost', 'FALSE', '/', 'TRUE', expires.to_time.to_i, 'permanent_http_only', '45'].join("\t"),
+    ]
+    assert_equal expected_cookielist, easy.cookielist
+    easy.perform
+    assert_equal 'permanent_http_only=45; session_http_only=42; permanent=44; session=43', easy.body_str
+  end
+
+  # Multiple cookies and comments are not supported
+  def test_setopt_cookielist_netscape_format_mutliline
+    easy = Curl::Easy.new
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    easy.setopt(
+      Curl::CURLOPT_COOKIELIST,
+      [
+        '# Netscape HTTP Cookie File',
+        ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '42'].join("\t"),
+        '',
+      ].join("\n"),
+    )
+    assert_nil easy.cookielist
+    easy.perform
+    assert_equal '', easy.body_str
+
+    easy.setopt(
+      Curl::CURLOPT_COOKIELIST,
+      [
+        ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '42'].join("\t"),
+        ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session2', '84'].join("\t"),
+        '',
+      ].join("\n"),
+    )
+    # Only first cookie is set
+    assert_equal [".localhost\tTRUE\t/\tFALSE\t0\tsession\t42"], easy.cookielist
+    easy.perform
+    assert_equal 'session=42', easy.body_str
+  end
+
+  # ALL erases all cookies held in memory
+  # ALL was added in 7.14.1
+  def test_setopt_cookielist_command_all
+    expires = (Date.today + 2).to_datetime
+    with_permanent_and_session_cookies(expires) do |easy|
+      easy.setopt(Curl::CURLOPT_COOKIELIST, 'ALL')
+      assert_nil easy.cookielist
+      easy.perform
+      assert_equal '', easy.body_str
+    end
+  end
+
+  # SESS erases all session cookies held in memory
+  # SESS was added in 7.15.4
+  def test_setopt_cookielist_command_sess
+    expires = (Date.today + 2).to_datetime
+    with_permanent_and_session_cookies(expires) do |easy|
+      easy.setopt(Curl::CURLOPT_COOKIELIST, 'SESS')
+      assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42"], easy.cookielist
+      easy.perform
+      assert_equal 'permanent=42', easy.body_str
+    end
+  end
+
+  # FLUSH writes all known cookies to the file specified by CURLOPT_COOKIEJAR
+  # FLUSH was added in 7.17.1
+  def test_setopt_cookielist_command_flush
+    expires = (Date.today + 2).to_datetime
+    with_permanent_and_session_cookies(expires) do |easy|
+      cookiejar = File.join(Dir.tmpdir, 'curl_test_cookiejar')
+      assert !File.exist?(cookiejar)
+      begin
+        easy.cookiejar = cookiejar
+        # trick to actually set CURLOPT_COOKIEJAR
+        easy.enable_cookies = true
+        easy.perform
+        assert !File.exist?(cookiejar)
+        easy.setopt(Curl::CURLOPT_COOKIELIST, 'FLUSH')
+        expected_cookiejar = <<~COOKIEJAR
+          # Netscape HTTP Cookie File
+          # https://curl.se/docs/http-cookies.html
+          # This file was generated by libcurl! Edit at your own risk.
+          
+          .localhost	TRUE	/	FALSE	0	session	420
+          .localhost	TRUE	/	FALSE	#{expires.to_time.to_i}	permanent	42
+        COOKIEJAR
+        assert_equal expected_cookiejar, File.read(cookiejar)
+      ensure
+        # Otherwise it'll create this file again
+        easy.close
+        File.unlink(cookiejar) if File.exist?(cookiejar)
+      end
+    end
+  end
+
+  # RELOAD loads all cookies from the files specified by CURLOPT_COOKIEFILE
+  # RELOAD was added in 7.39.0
+  def test_setopt_cookielist_command_reload
+    expires = (Date.today + 2).to_datetime
+    expires_file = (Date.today + 4).to_datetime
+    with_permanent_and_session_cookies(expires) do |easy|
+      cookiefile = File.join(Dir.tmpdir, 'curl_test_cookiefile')
+      assert !File.exist?(cookiefile)
+      begin
+        cookielist = [
+          # Won't be updated, added instead
+          ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent\t84",
+          ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent_file\t84",
+          # Won't be updated, added instead
+          ".localhost\tTRUE\t/\tFALSE\t0\tsession\t840",
+          ".localhost\tTRUE\t/\tFALSE\t0\tsession_file\t840",
+          '',
+        ]
+        File.write(cookiefile, cookielist.join("\n"))
+        easy.cookiefile = cookiefile
+        # trick to actually set CURLOPT_COOKIEFILE
+        easy.enable_cookies = true
+        easy.perform
+        easy.setopt(Curl::CURLOPT_COOKIELIST, 'RELOAD')
+        expected_cookielist = [
+          ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42",
+          ".localhost\tTRUE\t/\tFALSE\t0\tsession\t420",
+          ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent\t84",
+          ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent_file\t84",
+          ".localhost\tTRUE\t/\tFALSE\t0\tsession\t840",
+          ".localhost\tTRUE\t/\tFALSE\t0\tsession_file\t840",
+        ]
+        assert_equal expected_cookielist, easy.cookielist
+        easy.perform
+        # Be careful, duplicates are not removed
+        assert_equal 'permanent_file=84; session_file=840; permanent=84; session=840; permanent=42; session=420', easy.body_str
+      ensure
+        File.unlink(cookiefile) if File.exist?(cookiefile)
+      end
+    end
+  end
+
+  def test_commands_do_not_enable_cookie_engine
+    %w[ALL SESS FLUSH RELOAD].each do |command|
+      easy = Curl::Easy.new
+      expires = (Date.today + 2).to_datetime
+      easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies"
+      easy.setopt(Curl::CURLOPT_COOKIELIST, command)
+      easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }])
+      easy.perform
+      easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+      easy.post_body = nil
+      easy.perform
+
+      assert !easy.enable_cookies?
+      assert_nil easy.cookielist
+      assert_equal '', easy.body_str
+    end
+  end
+
+
+  def test_strings_without_cookie_enable_cookie_engine
+    [
+      '',
+      '# Netscape HTTP Cookie File',
+      'no_a_cookie',
+    ].each do |command|
+      easy = Curl::Easy.new
+      expires = (Date.today + 2).to_datetime
+      easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies"
+      easy.setopt(Curl::CURLOPT_COOKIELIST, command)
+      easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }])
+      easy.perform
+      easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+      easy.post_body = nil
+      easy.perform
+
+      assert !easy.enable_cookies?
+      assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist
+      assert_equal 'c2=v2', easy.body_str
+    end
+  end
+
+  def with_permanent_and_session_cookies(expires)
+    easy = Curl::Easy.new
+    easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies"
+    easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: permanent=42; domain=localhost; expires=#{expires.httpdate};")
+    easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: session=420; domain=localhost;')
+    assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42", ".localhost\tTRUE\t/\tFALSE\t0\tsession\t420"], easy.cookielist
+    easy.perform
+    assert_equal 'permanent=42; session=420', easy.body_str
+
+    yield easy
+  end
+
+  include TestServerMethods
+
+  def setup
+    server_setup
+  end
+end