Feature/nrmi 384 fix api syncs

app/jobs/inactive_urn_list_api_sync_job.rb

@@ -1,6 +1,6 @@
 class InactiveUrnListApiSyncJob < ApplicationJob
   def perform
-    rows = UrnLists::ApiClient.new.fetch_inactive_customers
+    rows = UrnLists::ApiClient.new.fetch_inactive_rows
 
     UrnLists::ImportInactiveCustomers.new(rows: rows).call
   end

app/jobs/urn_list_api_sync_job.rb

@@ -2,7 +2,7 @@ class UrnListApiSyncJob < ApplicationJob
   def perform
     urn_list = UrnList.create!(aasm_state: :pending, source: 'api_import')
 
-    rows = UrnLists::ApiClient.new.fetch_customers
+    rows = UrnLists::ApiClient.new.fetch_rows
     count = UrnLists::ImportCustomers.new(rows: rows).call
 
     urn_list.update!(

app/services/urn_lists/api_client.rb

@@ -6,45 +6,71 @@ module UrnLists
   class ApiClient
     class ApiError < StandardError; end
 
-    def fetch_customers
-      token = fetch_access_token
-      fetch_urn_list(token)
+    TOP_COUNT = 1000
+
+    def fetch_rows
+      fetch_paginated_rows(
+        base_url: active_urns_url,
+        params: {
+          'api-version' => '2016-10-01',
+          'sp' => '/triggers/manual/run',
+          'sv' => '1.0',
+          'filter' => "Published eq 'True'"
+        },
+        error_message: 'Failed to fetch URN list'
+      )
     end
 
-    def fetch_inactive_customers
-      token = fetch_access_token
-      fetch_inactive_urn_list(token)
+    def fetch_inactive_rows
+      fetch_paginated_rows(
+        base_url: inactive_urns_url,
+        params: {
+          'api-version' => '2016-10-01',
+          'sp' => '/triggers/manual/run',
+          'sv' => '1.0'
+        },
+        error_message: 'Failed to fetch inactive URN list'
+      )
     end
 
     private
 
-    def fetch_access_token
-      uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL'))
+    def fetch_paginated_rows(base_url:, params:, error_message:)
+      token = fetch_access_token
 
-      response = Net::HTTP.post_form(uri, {
-                                       grant_type: 'client_credentials',
-        client_id: ENV.fetch('MDM_API_CLIENT_ID'),
-        client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'),
-        scope: ENV.fetch('MDM_API_SCOPE')
-                                     })
+      all_rows = []
+      skip = 0
 
-      raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+      loop do
+        rows = fetch_page(
+          token: token,
+          base_url: base_url,
+          params: params,
+          top_count: TOP_COUNT,
+          skip: skip,
+          error_message: error_message
+        )
 
-      body = JSON.parse(response.body)
-      body.fetch('access_token')
-    end
+        break if rows.empty?
 
-    def fetch_urn_list(token)
-      base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/'
-      params = {
-        'api-version' => '2016-10-01',
-        'sp' => '/triggers/manual/run',
-        'sv' => '1.0',
-        'filter' => "Published eq 'True'"
-      }
+        all_rows.concat(rows)
+        break if rows.size < TOP_COUNT
 
+        skip += TOP_COUNT
+      end
+
+      all_rows
+    end
+
+    # rubocop:disable Metrics/ParameterLists
+    def fetch_page(token:, base_url:, params:, top_count:, skip:, error_message:)
       uri = URI(base_url)
-      uri.query = URI.encode_www_form(params)
+      uri.query = URI.encode_www_form(
+        params.merge(
+          'TopCount' => top_count,
+          'SkipCount' => skip
+        )
+      )
 
       request = Net::HTTP::Get.new(uri.to_s)
       request['Authorization'] = "Bearer #{token}"
@@ -54,43 +80,42 @@ def fetch_urn_list(token)
         http.request(request)
       end
 
-      raise ApiError, "Failed to fetch URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+      raise ApiError, "#{error_message}: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
 
       rows = JSON.parse(response.body)
       validate_rows!(rows)
       rows
     end
+    # rubocop:enable Metrics/ParameterLists
 
-    def fetch_inactive_urn_list(token)
-      base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/'
-      params = {
-        'api-version' => '2016-10-01',
-        'sp' => '/triggers/manual/run',
-        'sv' => '1.0'
-      }
-
-      uri = URI(base_url)
-      uri.query = URI.encode_www_form(params)
-
-      request = Net::HTTP::Get.new(uri.to_s)
-      request['Authorization'] = "Bearer #{token}"
-      request['Accept'] = 'application/json'
+    def fetch_access_token
+      uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL'))
 
-      response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
-        http.request(request)
-      end
+      response = Net::HTTP.post_form(uri, {
+                                       grant_type: 'client_credentials',
+        client_id: ENV.fetch('MDM_API_CLIENT_ID'),
+        client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'),
+        scope: ENV.fetch('MDM_API_SCOPE')
+                                     })
 
-      raise ApiError, "Failed to fetch inactive URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+      raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
 
-      rows = JSON.parse(response.body)
-      validate_rows!(rows)
-      rows
+      body = JSON.parse(response.body)
+      body.fetch('access_token')
     end
 
     def validate_rows!(rows)
       return if rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) }
 
       raise ApiError, 'Invalid URN list format: expected an array of objects'
     end
+
+    def active_urns_url
+      'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/'
+    end
+
+    def inactive_urns_url
+      'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/'
+    end
   end
 end

spec/jobs/urn_list_api_sync_job_spec.rb

@@ -23,7 +23,7 @@
       ]
     end
 
-    let(:api_client_service) { double('UrnLists::ApiClient', fetch_customers: rows) }
+    let(:api_client_service) { double('UrnLists::ApiClient', fetch_rows: rows) }
     let(:import_customers_service) { double('UrnLists::ImportCustomers', call: rows.count) }
 
     before do
@@ -36,7 +36,7 @@
         described_class.perform_now
       end.to change(UrnList, :count).by(1)
 
-      expect(api_client_service).to have_received(:fetch_customers)
+      expect(api_client_service).to have_received(:fetch_rows)
       expect(import_customers_service).to have_received(:call)
 
       urn_list = UrnList.last
@@ -47,7 +47,7 @@
     end
 
     it 'marks the urn list as failed when the api call fails' do
-      allow(api_client_service).to receive(:fetch_customers).and_raise(StandardError.new('token failed'))
+      allow(api_client_service).to receive(:fetch_rows).and_raise(StandardError.new('token failed'))
 
       expect do
         described_class.perform_now

spec/services/urn_lists/api_client_spec.rb

@@ -1,18 +1,20 @@
 require 'rails_helper'
 
 RSpec.describe UrnLists::ApiClient do
-  describe '#fetch_customers' do
+  describe '#fetch_rows' do
+    let(:top_count) { described_class::TOP_COUNT }
+
     before do
       stub_request(:post, 'https://example.com/oauth/token')
         .with(
           body: { 'client_id' => 'test_client_id', 'client_secret' => 'test_client_secret',
 'grant_type' => 'client_credentials', 'scope' => 'test_scope' },
           headers: {
             'Accept' => '*/*',
-           'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-           'Content-Type' => 'application/x-www-form-urlencoded',
-           'Host' => 'example.com',
-           'User-Agent' => 'Ruby'
+            'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'Content-Type' => 'application/x-www-form-urlencoded',
+            'Host' => 'example.com',
+            'User-Agent' => 'Ruby'
           }
         )
         .to_return(
@@ -21,13 +23,13 @@
           headers: { 'Content-Type' => 'application/json' }
         )
 
-      stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0")
+      stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?SkipCount=0&TopCount=1000&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0")
         .with(
           headers: {
             'Accept' => 'application/json',
-          'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-          'Authorization' => 'Bearer abc123',
-          'User-Agent' => 'Ruby'
+            'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'Authorization' => 'Bearer abc123',
+            'User-Agent' => 'Ruby'
           }
         )
         .to_return(
@@ -53,7 +55,7 @@
 
     it 'fetches and returns customer data' do
       client = described_class.new
-      customers = client.fetch_customers
+      customers = client.fetch_rows
 
       expect(customers.size).to eq(1)
       expect(customers.first['urn']).to eq(10009655)
@@ -62,5 +64,71 @@
       expect(customers.first['sector']).to eq('central_government')
       expect(customers.first['published']).to eq(true)
     end
+
+    context 'when the API returns multiple pages' do
+      let(:first_page_rows) do
+        Array.new(top_count) do |i|
+          {
+            urn: 10009655 + i,
+            name: "Customer #{i}",
+            postcode: 'L3 9PP',
+            sector: 'central_government',
+            published: true
+          }
+        end
+      end
+
+      let(:second_page_rows) do
+        [
+          {
+            urn: 10009655 + top_count,
+            name: "Customer #{top_count}",
+            postcode: 'L3 9PP',
+            sector: 'central_government',
+            published: true
+          }
+        ]
+      end
+
+      before do
+        stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=0&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0")
+          .with(
+            headers: {
+              'Accept' => 'application/json',
+              'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+              'Authorization' => 'Bearer abc123',
+              'User-Agent' => 'Ruby'
+            }
+          )
+          .to_return(
+            status: 200,
+            body: first_page_rows.to_json,
+            headers: { 'Content-Type' => 'application/json' }
+          )
+
+        stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=#{top_count}&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0")
+          .with(
+            headers: {
+              'Accept' => 'application/json',
+              'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+              'Authorization' => 'Bearer abc123',
+              'User-Agent' => 'Ruby'
+            }
+          )
+          .to_return(
+            status: 200,
+            body: second_page_rows.to_json,
+            headers: { 'Content-Type' => 'application/json' }
+          )
+      end
+
+      it 'fetches each page and returns combined rows' do
+        rows = described_class.new.fetch_rows
+
+        expect(rows.count).to eq(top_count + 1)
+        expect(rows.first['urn']).to eq(10009655)
+        expect(rows.last['urn']).to eq(10009655 + top_count)
+      end
+    end
   end
 end