Search endpoint accounts for Search.gov constraints (#2408)

* Handles case where Search.gov’s rate limit is exceeded

* Calculates and returns viewable page data

Since Search.gov’s offset is maxed out at 999, we cannot view all of the potential search results.

For example, if the total search results are 35k, we would only be able to view the first 1000.

The total pages and total entries now accounts for this constraint.

app/swagger/requests/search.rb

@@ -214,6 +214,27 @@ class Search
               key :'$ref', :Errors
             end
           end
+
+          response 429 do
+            key :description, 'Exceeded rate limit'
+            schema do
+              key :required, [:errors]
+
+              property :errors do
+                key :type, :array
+                items do
+                  key :required, %i[title detail code status source]
+                  property :title, type: :string, example: 'Exceeded rate limit'
+                  property :detail,
+                           type: :string,
+                           example: 'Exceeded Search.gov rate limit'
+                  property :code, type: :string, example: 'SEARCH_429'
+                  property :status, type: :string, example: '429'
+                  property :source, type: :string, example: 'Search::Service'
+                end
+              end
+            end
+          end
         end
       end
     end

config/locales/exceptions.en.yml

@@ -356,6 +356,12 @@ en:
         code: 'SEARCH_400'
         detail: 'Search.gov service responded with a Bad Request'
         status: 400
+      SEARCH_429:
+        <<: *external_defaults
+        title: Exceeded rate limit
+        code: 'SEARCH_429'
+        detail: 'Exceeded Search.gov rate limit'
+        status: 429
       VET360_502:
         <<: *external_defaults
         title: Bad Gateway

lib/search/pagination.rb

@@ -14,6 +14,15 @@ class Pagination
     #
     ENTRIES_PER_PAGE = 10
 
+    # Due to Search.gov's offset max of 999, we cannot view pages
+    # where the offset param exceeds 999.  This influences our:
+    #   - total_viewable_pages
+    #   - total_viewable_entries
+    #
+    # @see https://search.usa.gov/sites/7378/api_instructions under `offset`
+    #
+    OFFSET_LIMIT = 999
+
     attr_reader :next_offset
     attr_reader :total_entries
     attr_reader :total_pages
@@ -47,9 +56,25 @@ def pagination_object
       {
         'current_page' => [current_page, total_pages].min,
         'per_page' => ENTRIES_PER_PAGE,
-        'total_pages' => total_pages,
-        'total_entries' => total_entries
+        'total_pages' => total_viewable_pages,
+        'total_entries' => total_viewable_entries
       }
     end
+
+    def total_viewable_pages
+      [total_pages, maximum_viewable_pages].min
+    end
+
+    def maximum_viewable_pages
+      (OFFSET_LIMIT / ENTRIES_PER_PAGE.to_f).floor
+    end
+
+    def total_viewable_entries
+      [total_entries, maximum_viewable_entries].min
+    end
+
+    def maximum_viewable_entries
+      (ENTRIES_PER_PAGE * total_viewable_pages) + (ENTRIES_PER_PAGE - 1)
+    end
   end
 end

lib/search/service.rb

@@ -87,7 +87,8 @@ def handle_error(error)
       when Common::Client::Errors::ClientError
         message = parse_messages(error).first
         log_error_message(message)
-        raise_backend_exception('SEARCH_400', self.class, error) if error.status == 400
+        raise_backend_exception('SEARCH_429', self.class, error) if error.status == 429
+        raise_backend_exception('SEARCH_400', self.class, error) if error.status >= 400
       else
         raise error
       end

spec/lib/search/pagination_spec.rb

@@ -80,4 +80,39 @@
       expect(subject.object).to include('current_page' => 9)
     end
   end
+
+  context 'when Search.govs total entries exceed the OFFSET_LIMIT to view them' do
+    let(:raw_body) do
+      {
+        'web' =>
+          {
+            'total' => 35_123,
+            'next_offset' => 20
+          }
+      }
+    end
+    subject { described_class.new(raw_body) }
+
+    it 'sets total_pages to the maximum viewable number of pages' do
+      expect(subject.object['total_pages']).to eq 99
+    end
+
+    it 'sets total_entries to the maximum viewable number of entries' do
+      expect(subject.object['total_entries']).to eq 999
+    end
+
+    context 'when the ENTRIES_PER_PAGE is set to its max of 50' do
+      before do
+        stub_const('Search::Pagination::ENTRIES_PER_PAGE', 50)
+      end
+
+      it 'sets total_pages to the maximum viewable number of pages' do
+        expect(subject.object['total_pages']).to eq 19
+      end
+
+      it 'sets total_entries to the maximum viewable number of entries' do
+        expect(subject.object['total_entries']).to eq 999
+      end
+    end
+  end
 end

spec/lib/search/service_spec.rb

@@ -77,5 +77,17 @@
         end
       end
     end
+
+    context 'when exceeding the Search.gov rate limit' do
+      it 'raises an exception', :aggregate_failures do
+        VCR.use_cassette('search/exceeds_rate_limit', VCR::MATCH_EVERYTHING) do
+          expect { subject.results }.to raise_error do |e|
+            expect(e).to be_a(Common::Exceptions::BackendServiceException)
+            expect(e.status_code).to eq(429)
+            expect(e.errors.first.code).to eq('SEARCH_429')
+          end
+        end
+      end
+    end
   end
 end

spec/request/swagger_spec.rb

@@ -1713,6 +1713,14 @@
           end
         end
       end
+
+      context 'when the Search.gov rate limit has been exceeded' do
+        it 'returns a 429 with error details' do
+          VCR.use_cassette('search/exceeds_rate_limit') do
+            expect(subject).to validate(:get, '/v0/search', 429, '_query_string' => 'query=benefits')
+          end
+        end
+      end
     end
   end