Fix University of Twente scraper (0 events ingested)

lib/ingestors/taxila/utwente_ingestor.rb

@@ -11,6 +11,14 @@ def self.config
         }
       end
 
+      # The events page is a client-rendered app that fetches its data from this fixed WebHare
+      # JSON-RPC endpoint - it doesn't move when the page URL does. OBJID identifies the events
+      # section in UT's CMS; it would only need updating if UT restructures their site again.
+      RPC_URL = 'https://www.utwente.nl/wh_services/utwente_base/rpc/GetNEOItems'
+      OBJID = 414426
+      PAGE_SIZE = 10
+      MAX_PAGES = 50 # safety cap in case the API never reports moreitems: false
+
       def read(url)
         begin
           process_utwente(url)
@@ -25,35 +33,41 @@ def read(url)
       private
 
       def process_utwente(url)
-        response = get_json_response(url, method: :post, referrer: url,
-          headers: {
-            content_type: :json,
-            accept: :json
-          },
-          payload: {
-            id: 1,
-            method: 'GetItems',
-            # there may be an issue with multiple categories here...
-            params: [{ categories: url.split('=').last }]
-          }.to_json)
-
-        response['result']['items'].each do |item|
-          event = OpenStruct.new
-
-          event.title = item['title']
-          event.url = item['link']
-          event.start, event.end = parse_dates(item['dateformatted'])
-          event.set_default_times
-          event.venue = item['location']
-
-          event.keywords = item['tags'].map{ |t| t['tag'] }
-          event.description = convert_description item['description']
-          event.timezone = 'Amsterdam'
-          event.organizer = 'University of Twente'
-          event.source = 'University of Twente'
-          add_event(event)
-        rescue Exception => e
-          @messages << "Extract event fields failed with: #{e.message}"
+        skip = 0
+        MAX_PAGES.times do
+          response = get_json_response(RPC_URL, method: :post, referrer: url,
+            headers: {
+              content_type: :json,
+              accept: :json
+            },
+            payload: {
+              id: 1,
+              method: 'GetNEOItems',
+              params: [OBJID, 'event', '', nil, { skip: skip, archive: false, tag: 0 }]
+            }.to_json)
+
+          result = response['result']
+          result['items'].each do |item|
+            event = OpenStruct.new
+
+            event.title = item['title']
+            event.url = item['link']
+            event.start, event.end = parse_dates(item['dateformatted'])
+            event.set_default_times
+            event.venue = item['location']
+
+            event.keywords = item['tags'].map{ |t| t['tag'] }
+            event.description = convert_description item['description']
+            event.timezone = 'Amsterdam'
+            event.organizer = 'University of Twente'
+            event.source = 'University of Twente'
+            add_event(event)
+          rescue Exception => e
+            @messages << "Extract event fields failed with: #{e.message}"
+          end
+
+          break unless result['moreitems']
+          skip += PAGE_SIZE
         end
       end
     end

test/unit/ingestors/taxila/utwente_ingestor_test.rb

@@ -14,20 +14,20 @@ class UtwenteIngestorTest < ActiveSupport::TestCase
 
   test 'can ingest events from utwente' do
     source = @content_provider.sources.build(
-      url: 'https://www.utwente.nl/en/events/?categories=417878',
+      url: 'https://www.utwente.nl/en/about-us/news-events-ceremonies/events/',
       method: 'utwente',
       enabled: true
     )
 
     ingestor = Ingestors::Taxila::UtwenteIngestor.new
 
     # check event doesn't
-    new_title = 'Risk & Resilience Festival'
-    new_url = 'https://www.utwente.nl/en/events/2023/11/925436/risk-resilience-festival'
+    new_title = 'Let’s make peace work'
+    new_url = 'https://www.utwente.nl/en/about-us/news-events-ceremonies/events/2026/9/996952/lets-make-peace-work'
     refute Event.where(title: new_title, url: new_url).any?
 
-    # run task
-    assert_difference 'Event.count', 1 do
+    # run task, cassette covers 2 pages (skip: 0, then skip: 10) to exercise pagination
+    assert_difference 'Event.count', 2 do
       freeze_time(2019) do
         VCR.use_cassette('ingestors/utwente') do
           ingestor.read(source.url)
@@ -36,9 +36,9 @@ class UtwenteIngestorTest < ActiveSupport::TestCase
       end
     end
 
-    assert_equal 1, ingestor.events.count
+    assert_equal 2, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 1, ingestor.stats[:events][:added]
+    assert_equal 2, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -51,8 +51,11 @@ class UtwenteIngestorTest < ActiveSupport::TestCase
     # check other fields
     assert_equal 'Amsterdam', event.timezone
     assert_equal 'University of Twente', event.organizer
-    assert_equal Time.zone.parse('Thu, 09 Nov 2023 09:00:00 +0000'), event.start
-    assert_equal Time.zone.parse('Thu, 09 Nov 2023 17:00:00 +0000'), event.end
-    assert_equal 'Waaier', event.venue
+    assert_equal Time.zone.parse('Tue, 22 Sep 2026 19:30:00 +0000'), event.start
+    assert_equal Time.zone.parse('Tue, 22 Sep 2026 21:00:00 +0000'), event.end
+    assert_equal 'Vrijhof - Amphitheater', event.venue
+
+    # second page came through too
+    assert Event.where(title: 'Founding Father of ASML – Martin van den Brink').any?
   end
 end