Support Net::HTTP::Persistent.write_timeout

.github/workflows/ci-test.yml

@@ -1,5 +1,9 @@
 name: "ci"
 
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+  cancel-in-progress: true
+
 on:
   push:
     branches:
@@ -8,15 +12,17 @@ on:
     types: [opened, synchronize]
     branches:
       - main
+  schedule:
+    - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5
 
 jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "3.0"
+        ruby-version: "3.2"
         bundler-cache: true
     - run: bundle exec rake rubocop
 
@@ -25,15 +31,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ["2.5", "2.6", "2.7", "3.0", "jruby", "truffleruby-head"]
+        ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"]
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{matrix.ruby-version}}
         bundler-cache: true
+        bundler: 2.3.26 # https://github.com/rubygems/rubygems/issues/6435
     - run: bundle exec rake test
 
   test-platform:
@@ -45,9 +52,10 @@ jobs:
 
     runs-on: ${{matrix.platform}}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "3.0"
+        ruby-version: "3.2"
         bundler-cache: true
+        bundler: latest
     - run: bundle exec rake test

CHANGELOG.md

@@ -1,5 +1,45 @@
 # Mechanize CHANGELOG
 
+## 2.9.1 / 2023-04-17
+
+### Update
+
+* Updated User-Agent strings to represent modern browser versions. (#612) Thank you, @takatea!
+
+
+## 2.9.0 / 2023-04-07
+
+### Requirements
+
+* Mechanize now requires Ruby 2.6 or newer.
+
+
+### Improvement
+
+* Mechanize can now parse frozen strings. (#610)
+
+
+## 2.8.5 / 2022-06-09
+
+### Security
+
+Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." See [GHSA-64qm-hrgp-pgr9](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-64qm-hrgp-pgr9) for more details.
+
+
+## 2.8.4 / 2022-01-17
+
+### Fix
+
+* `Mechanize::CookieJar#load` calls `Psych.safe_load` when using Psych >= 3.1
+
+
+## 2.8.3 / 2021-11-11
+
+### Update
+
+* Update the "Linux Firefox" user agent string to rev94 (#587) Thank you, @ncs1!
+
+
 ## 2.8.2 / 2021-08-06
 
 ### Dependencies

README.md

@@ -13,7 +13,7 @@ The Mechanize library is used for automating interaction with websites. Mechaniz
 
 ## Dependencies
 
-* Ruby >= 2.5
+* Ruby >= 2.6
 * Gems:
   * `addressable`
   * `domain_name`

examples/latest_user_agents.rb

@@ -0,0 +1,91 @@
+require 'mechanize'
+
+class LatestUAFetcher
+  attr_reader :user_agents
+
+  BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent'
+
+  def initialize
+    @agent = Mechanize.new.tap { |a| a.user_agent_alias = 'Mac Firefox' }
+    @user_agents = {}
+  end
+
+  def run
+    sleep_time = 1
+
+    puts 'get chrome UA...'
+    chrome
+    puts "sleeping... (#{sleep_time}s)"
+    sleep 1
+
+    puts 'get firefox UA...'
+    firefox
+    puts "sleeping... (#{sleep_time}s)"
+    sleep 1
+
+    puts 'get safari UA...'
+    safari
+    puts "sleeping... (#{sleep_time}s)"
+    sleep 1
+
+    puts 'get edge UA...'
+    edge
+  end
+
+  private
+
+  def edge
+    page = @agent.get("#{BASE_URL}/edge")
+
+    windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')")
+    @user_agents[:edge] = {
+      windows: windows_dom.css('+ .listing-of-useragents .code').first.text
+    }
+  end
+
+  def firefox
+    page = @agent.get("#{BASE_URL}/firefox")
+
+    desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')")
+    table_dom = desktop_dom.css('+ .listing-of-useragents')
+
+    @user_agents[:firefox] = {
+      windows: table_dom.css('td:contains("Windows")').css('+ td .code').text,
+      macOS: table_dom.css('td:contains("Macos")').css('+ td .code').text,
+      linux: table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text
+    }
+  end
+
+  def safari
+    page = @agent.get("#{BASE_URL}/safari")
+
+    macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')")
+    ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')")
+
+    @user_agents[:safari] = {
+      mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text,
+      iphone: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text,
+      ipad: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text
+    }
+  end
+
+  def chrome
+    page = @agent.get("#{BASE_URL}/chrome")
+
+    windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')")
+    linux_dom = page.css("h2:contains('Latest Chrome on Linux User Agents')")
+    macos_dom = page.css("h2:contains('Latest Chrome on macOS User Agents')")
+    android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')")
+
+    @user_agents[:chrome] = {
+      windows: windows_dom.css('+ .listing-of-useragents .code').first.text,
+      linux: linux_dom.css('+ .listing-of-useragents .code').first.text,
+      mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text,
+      android: android_dom.css('+ .listing-of-useragents .code').first.text
+    }
+  end
+end
+
+agent = LatestUAFetcher.new
+agent.run
+p agent.user_agents

examples/wikipedia_links_to_philosophy.rb

@@ -58,10 +58,10 @@ def finished?
   # the article.
 
   def follow_first_link
-    puts @title
+    puts "#{@title} (#{@page.uri})"
 
     # > p > a rejects italics
-    links = @page.root.css('.mw-content-ltr > p > a[href^="/wiki/"]')
+    links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]')
 
     # reject disambiguation and special pages, images and files
     links = links.reject do |link_node|
@@ -74,10 +74,9 @@ def follow_first_link
 
     link = links.first
 
-    unless link then
-      # disambiguation page? try the first item in the list
-      link =
-        @page.root.css('.mw-content-ltr > ul > li > a[href^="/wiki/"]').first
+    if link.nil?
+      puts "Could not parse #{@page.uri}"
+      exit 1
     end
 
     # convert a Nokogiri HTML element back to a mechanize link

lib/mechanize.rb

@@ -87,54 +87,73 @@ class Error < RuntimeError
   # description in parenthesis is for informative purposes and is not part of
   # the alias name.
   #
-  # * Linux Firefox (43.0 on Ubuntu Linux)
-  # * Linux Konqueror (3)
-  # * Linux Mozilla
-  # * Mac Firefox (43.0)
-  # * Mac Mozilla
-  # * Mac Safari (9.0 on OS X 10.11.2)
-  # * Mac Safari 4
-  # * Mechanize (default)
-  # * Windows IE 6
-  # * Windows IE 7
-  # * Windows IE 8
-  # * Windows IE 9
-  # * Windows IE 10 (Windows 8 64bit)
-  # * Windows IE 11 (Windows 8.1 64bit)
-  # * Windows Edge
-  # * Windows Mozilla
-  # * Windows Firefox (43.0)
-  # * iPhone (iOS 9.1)
-  # * iPad (iOS 9.1)
-  # * Android (5.1.1)
+  # The default User-Agent alias:
+  #
+  # * "Mechanize"
+  #
+  # Linux User-Agent aliases:
+  #
+  # * "Linux Firefox"
+  # * "Linux Konqueror"
+  # * "Linux Mozilla"
+  #
+  # Mac User-Agent aliases:
+  #
+  # * "Mac Firefox"
+  # * "Mac Mozilla"
+  # * "Mac Safari 4"
+  # * "Mac Safari"
+  #
+  # Windows User-Agent aliases:
+  #
+  # * "Windows Edge"
+  # * "Windows Firefox"
+  # * "Windows IE 6"
+  # * "Windows IE 7"
+  # * "Windows IE 8"
+  # * "Windows IE 9"
+  # * "Windows IE 10"
+  # * "Windows IE 11"
+  # * "Windows Mozilla"
+  #
+  # Mobile User-Agent aliases:
+  #
+  # * "Android"
+  # * "iPad"
+  # * "iPhone"
   #
   # Example:
   #
   #   agent = Mechanize.new
   #   agent.user_agent_alias = 'Mac Safari'
-
+  #
   AGENT_ALIASES = {
+    # TODO: use output from examples/latest_user_agents.rb as the underling data structure
     'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)",
-    'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0',
+
+    'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:112.0) Gecko/20100101 Firefox/112.0',
     'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
     'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
-    'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0',
+
+    'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13.3; rv:112.0) Gecko/20100101 Firefox/112.0',
     'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
     'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10',
-    'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9',
-    'Windows Chrome' => 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.125 Safari/537.36',
+    'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',
+
+    'Windows Chrome' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+    'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.46',
+    'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0',
     'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
     'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
     'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
     'Windows IE 10' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)',
     'Windows IE 11' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko',
-    'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586',
     'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
-    'Windows Firefox' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0',
-    'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B5110e Safari/601.1',
-    'iPad' => 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1',
-    'Android' => 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.76 Safari/537.36',
+
+    'Android' => 'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.48 Mobile Safari/537.36',
+    'iPad' => 'Mozilla/5.0 (iPad; CPU OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1',
+    'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1',
   }
 
   AGENT_ALIASES.default_proc = proc { |hash, key|
@@ -933,6 +952,21 @@ def read_timeout= read_timeout
     @agent.read_timeout = read_timeout
   end
 
+  ##
+  # Length of time to wait for data to be sent to the server
+
+  def write_timeout
+    @agent.write_timeout
+  end
+
+  ##
+  # Sets the timeout for each chunk of data to be sent to the server to
+  # +write_timeout+.  A single request may write many chunks of data.
+
+  def write_timeout= write_timeout
+    @agent.write_timeout = write_timeout
+  end
+
   ##
   # Controls how mechanize deals with redirects.  The following values are
   # allowed:

lib/mechanize/cookie_jar.rb

@@ -149,7 +149,7 @@ def load(input, *options)
       return super(input, opthash) if opthash[:format] != :yaml
 
       begin
-        data = YAML.load(input) # rubocop:disable Security/YAMLLoad
+        data = load_yaml(input)
       rescue ArgumentError
         @logger.warn "unloadable YAML cookie data discarded" if @logger
         return self
@@ -174,6 +174,18 @@ def load(input, *options)
         return self
       end
     end
+
+    private
+
+    if YAML.name == "Psych" && Gem::Requirement.new(">= 3.1").satisfied_by?(Gem::Version.new(Psych::VERSION))
+      def load_yaml(yaml)
+        YAML.safe_load(yaml, aliases: true, permitted_classes: ["Mechanize::Cookie", "Time"])
+      end
+    else
+      def load_yaml(yaml)
+        YAML.load(yaml) # rubocop:disable Security/YAMLLoad
+      end
+    end
   end
 
   class ::HTTP::CookieJar