From 23ae89cd49a836329326c0152ed776b45812fd5b Mon Sep 17 00:00:00 2001 From: Alexandra Ioan Date: Wed, 16 Sep 2026 14:00:03 +0200 Subject: [PATCH] Fix University of Twente scraper (0 events ingested) --- lib/ingestors/taxila/utwente_ingestor.rb | 72 +++++++++------ .../ingestors/taxila/utwente_ingestor_test.rb | 23 ++--- test/vcr_cassettes/ingestors/utwente.yml | 90 ++++++++++++++----- 3 files changed, 122 insertions(+), 63 deletions(-) diff --git a/lib/ingestors/taxila/utwente_ingestor.rb b/lib/ingestors/taxila/utwente_ingestor.rb index b2254fbb3..741ef772d 100644 --- a/lib/ingestors/taxila/utwente_ingestor.rb +++ b/lib/ingestors/taxila/utwente_ingestor.rb @@ -11,6 +11,14 @@ def self.config } end + # The events page is a client-rendered app that fetches its data from this fixed WebHare + # JSON-RPC endpoint - it doesn't move when the page URL does. OBJID identifies the events + # section in UT's CMS; it would only need updating if UT restructures their site again. + RPC_URL = 'https://www.utwente.nl/wh_services/utwente_base/rpc/GetNEOItems' + OBJID = 414426 + PAGE_SIZE = 10 + MAX_PAGES = 50 # safety cap in case the API never reports moreitems: false + def read(url) begin process_utwente(url) @@ -25,35 +33,41 @@ def read(url) private def process_utwente(url) - response = get_json_response(url, method: :post, referrer: url, - headers: { - content_type: :json, - accept: :json - }, - payload: { - id: 1, - method: 'GetItems', - # there may be an issue with multiple categories here... - params: [{ categories: url.split('=').last }] - }.to_json) - - response['result']['items'].each do |item| - event = OpenStruct.new - - event.title = item['title'] - event.url = item['link'] - event.start, event.end = parse_dates(item['dateformatted']) - event.set_default_times - event.venue = item['location'] - - event.keywords = item['tags'].map{ |t| t['tag'] } - event.description = convert_description item['description'] - event.timezone = 'Amsterdam' - event.organizer = 'University of Twente' - event.source = 'University of Twente' - add_event(event) - rescue Exception => e - @messages << "Extract event fields failed with: #{e.message}" + skip = 0 + MAX_PAGES.times do + response = get_json_response(RPC_URL, method: :post, referrer: url, + headers: { + content_type: :json, + accept: :json + }, + payload: { + id: 1, + method: 'GetNEOItems', + params: [OBJID, 'event', '', nil, { skip: skip, archive: false, tag: 0 }] + }.to_json) + + result = response['result'] + result['items'].each do |item| + event = OpenStruct.new + + event.title = item['title'] + event.url = item['link'] + event.start, event.end = parse_dates(item['dateformatted']) + event.set_default_times + event.venue = item['location'] + + event.keywords = item['tags'].map{ |t| t['tag'] } + event.description = convert_description item['description'] + event.timezone = 'Amsterdam' + event.organizer = 'University of Twente' + event.source = 'University of Twente' + add_event(event) + rescue Exception => e + @messages << "Extract event fields failed with: #{e.message}" + end + + break unless result['moreitems'] + skip += PAGE_SIZE end end end diff --git a/test/unit/ingestors/taxila/utwente_ingestor_test.rb b/test/unit/ingestors/taxila/utwente_ingestor_test.rb index 977eea4ca..60fe84c5f 100644 --- a/test/unit/ingestors/taxila/utwente_ingestor_test.rb +++ b/test/unit/ingestors/taxila/utwente_ingestor_test.rb @@ -14,7 +14,7 @@ class UtwenteIngestorTest < ActiveSupport::TestCase test 'can ingest events from utwente' do source = @content_provider.sources.build( - url: 'https://www.utwente.nl/en/events/?categories=417878', + url: 'https://www.utwente.nl/en/about-us/news-events-ceremonies/events/', method: 'utwente', enabled: true ) @@ -22,12 +22,12 @@ class UtwenteIngestorTest < ActiveSupport::TestCase ingestor = Ingestors::Taxila::UtwenteIngestor.new # check event doesn't - new_title = 'Risk & Resilience Festival' - new_url = 'https://www.utwente.nl/en/events/2023/11/925436/risk-resilience-festival' + new_title = 'Let’s make peace work' + new_url = 'https://www.utwente.nl/en/about-us/news-events-ceremonies/events/2026/9/996952/lets-make-peace-work' refute Event.where(title: new_title, url: new_url).any? - # run task - assert_difference 'Event.count', 1 do + # run task, cassette covers 2 pages (skip: 0, then skip: 10) to exercise pagination + assert_difference 'Event.count', 2 do freeze_time(2019) do VCR.use_cassette('ingestors/utwente') do ingestor.read(source.url) @@ -36,9 +36,9 @@ class UtwenteIngestorTest < ActiveSupport::TestCase end end - assert_equal 1, ingestor.events.count + assert_equal 2, ingestor.events.count assert ingestor.materials.empty? - assert_equal 1, ingestor.stats[:events][:added] + assert_equal 2, ingestor.stats[:events][:added] assert_equal 0, ingestor.stats[:events][:updated] assert_equal 0, ingestor.stats[:events][:rejected] @@ -51,8 +51,11 @@ class UtwenteIngestorTest < ActiveSupport::TestCase # check other fields assert_equal 'Amsterdam', event.timezone assert_equal 'University of Twente', event.organizer - assert_equal Time.zone.parse('Thu, 09 Nov 2023 09:00:00 +0000'), event.start - assert_equal Time.zone.parse('Thu, 09 Nov 2023 17:00:00 +0000'), event.end - assert_equal 'Waaier', event.venue + assert_equal Time.zone.parse('Tue, 22 Sep 2026 19:30:00 +0000'), event.start + assert_equal Time.zone.parse('Tue, 22 Sep 2026 21:00:00 +0000'), event.end + assert_equal 'Vrijhof - Amphitheater', event.venue + + # second page came through too + assert Event.where(title: 'Founding Father of ASML – Martin van den Brink').any? end end diff --git a/test/vcr_cassettes/ingestors/utwente.yml b/test/vcr_cassettes/ingestors/utwente.yml index 57ec310e7..c038daf99 100644 --- a/test/vcr_cassettes/ingestors/utwente.yml +++ b/test/vcr_cassettes/ingestors/utwente.yml @@ -2,19 +2,19 @@ http_interactions: - request: method: post - uri: https://www.utwente.nl/en/events/?categories=417878 + uri: https://www.utwente.nl/wh_services/utwente_base/rpc/GetNEOItems body: encoding: UTF-8 - string: '{"id":1,"method":"GetItems","params":[{"categories":"417878"}]}' + string: '{"id":1,"method":"GetNEOItems","params":[414426,"event","",null,{"skip":0,"archive":false,"tag":0}]}' headers: Accept: - application/json User-Agent: - - rest-client/2.1.0 (linux x86_64) ruby/3.0.5p211 + - rest-client/2.1.0 (linux x86_64) ruby/3.4.9 Content-Type: - application/json Content-Length: - - '63' + - '105' Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 Host: @@ -26,8 +26,6 @@ http_interactions: headers: Server: - nginx - Date: - - Thu, 09 Feb 2023 09:50:07 GMT Content-Type: - application/json Transfer-Encoding: @@ -43,25 +41,69 @@ http_interactions: X-Xss-Protection: - 1; mode=block Content-Security-Policy: - - frame-ancestors 'self' https://webhare.utwente.nl https://portal-test.utsp.utwente.nl + - frame-ancestors 'self' https://webhare.utwente.nl Cache-Control: - no-cache body: encoding: ASCII-8BIT - string: '{"error":null,"id":1,"result":{"emptyfilters":false,"items":[{"addthisdata":{"allday":true,"description":"

Indestructible adaptability and resilience are crucial in - this unpredictable world and uncertain time.\u00A0Risk and resilience management - is - especially now - invaluable. The festival consists of interactive and - instructive sessions. In these sessions, scientists, risk managers, professionals - and students will discuss safety with each other. Looking for pressing questions - that need to be answered.<\/p>","enddate":"09/11/2023 23:59","startdate":"09/11/2023 - 00:00"},"dateformatted":"Thu 9 Nov 2023","description":"Indestructible adaptability - and resilience are crucial in this unpredictable world and uncertain time. - Risk and resilience management is - especially now - invaluable. The festival - consists of interactive and instructive sessions. In these sessions, scientists, - risk managers, professionals and students will discuss safety with each other. - Looking for pressing questions that need to be answered.","id":925436,"link":"https://www.utwente.nl/en/events/2023/11/925436/risk-resilience-festival","location":"Waaier","photolink":"https://1348661504.rsc.cdn77.org/.uc/ic25c7d48010254b8df01274d47032370cbd8e18df7be0701c3a401ee0080/data.jpg","photolink_2x":"https://1348661504.rsc.cdn77.org/.uc/i58f2af55010254b8df01274d47032370cbd8e18df7be0701c34803dc0180/data.jpg","subtitle":"","summaryphotolink":"https://1348661504.rsc.cdn77.org/.uc/if4fe220801025dcc5701274d4703f2f55d25a150f54d0701c33a003a0080/data.png","summaryphotolink_2x":"https://1348661504.rsc.cdn77.org/.uc/ibf507df401025dcc5701274d4703f2f55d25a150f54d0701c37400740080/data.png","tags":[{"filter":"categories","id":417877,"tag":"Conferences - & Symposia"},{"filter":"categories","id":417884,"tag":"Festivals"},{"filter":"categories","id":417878,"tag":"Lectures"}],"title":"Risk - & Resilience Festival"}],"links":[{"id":925436,"link":"2023/11/925436/risk-resilience-festival"}],"moreitems":false,"totalresults":1}}' - recorded_at: Tue, 01 Jan 2019 00:00:00 GMT -recorded_with: VCR 6.1.0 + string: '{"error":null,"id":1,"result":{"archive":false,"items":[{"addthisdata":{"allday":false,"description":"

Our world today is plagued by numerous wars and conflicts.<\/p>","enddate":"22/09/2026 + 21:00","startdate":"22/09/2026 19:30"},"dateformatted":"Tue 22 Sep 2026 19:30 + - 21:00","description":"Our world today is plagued by numerous wars and conflicts.","id":996952,"link":"https://www.utwente.nl/en/about-us/news-events-ceremonies/events/2026/9/996952/lets-make-peace-work","location":"Vrijhof + - Amphitheater","photoalt":"","photolink":"https://utwente.becdn.net/.wh/ea/uc/i0/data.avif","photolink_2x":"https://utwente.becdn.net/.wh/ea/uc/i1/data.avif","subtitle":"Lecture","summaryphotoalt":"","summaryphotolink":"https://utwente.becdn.net/.wh/ea/uc/i2/data.avif","summaryphotolink_2x":"https://utwente.becdn.net/.wh/ea/uc/i3/data.avif","tags":[{"filter":"categories","id":417879,"tag":"Studium + Generale"}],"title":"Let’s make peace work"}],"links":[{"id":996952,"suburl":"2026/9/996952/lets-make-peace-work"}],"moreitems":true,"skip":0,"tag":0,"totalresults":2}}' + recorded_at: Wed, 16 Sep 2026 12:00:00 GMT +- request: + method: post + uri: https://www.utwente.nl/wh_services/utwente_base/rpc/GetNEOItems + body: + encoding: UTF-8 + string: '{"id":1,"method":"GetNEOItems","params":[414426,"event","",null,{"skip":10,"archive":false,"tag":0}]}' + headers: + Accept: + - application/json + User-Agent: + - rest-client/2.1.0 (linux x86_64) ruby/3.4.9 + Content-Type: + - application/json + Content-Length: + - '106' + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Host: + - www.utwente.nl + response: + status: + code: 200 + message: OK + headers: + Server: + - nginx + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Vary: + - Accept-Encoding + Strict-Transport-Security: + - max-age=31536000 + X-Frame-Options: + - SAMEORIGIN + X-Xss-Protection: + - 1; mode=block + Content-Security-Policy: + - frame-ancestors 'self' https://webhare.utwente.nl + Cache-Control: + - no-cache + body: + encoding: ASCII-8BIT + string: '{"error":null,"id":1,"result":{"archive":false,"items":[{"addthisdata":{"allday":false,"description":"

After finishing his study Applied Physics.<\/p>","enddate":"29/09/2026 + 21:00","startdate":"29/09/2026 19:30"},"dateformatted":"Tue 29 Sep 2026 19:30 + - 21:00","description":"After finishing his study Applied Physics.","id":997304,"link":"https://www.utwente.nl/en/about-us/news-events-ceremonies/events/2026/9/997304/founding-father-of-asml-martin-van-den-brink","location":"Vrijhof + - Amphitheater","photoalt":"","photolink":"https://utwente.becdn.net/.wh/ea/uc/i4/data.avif","photolink_2x":"https://utwente.becdn.net/.wh/ea/uc/i5/data.avif","subtitle":"Interview","summaryphotoalt":"","summaryphotolink":"https://utwente.becdn.net/.wh/ea/uc/i6/data.avif","summaryphotolink_2x":"https://utwente.becdn.net/.wh/ea/uc/i7/data.avif","tags":[{"filter":"categories","id":417879,"tag":"Studium + Generale"}],"title":"Founding Father of ASML – Martin van den Brink"}],"links":[{"id":997304,"suburl":"2026/9/997304/founding-father-of-asml-martin-van-den-brink"}],"moreitems":false,"skip":10,"tag":0,"totalresults":2}}' + recorded_at: Wed, 16 Sep 2026 12:00:01 GMT +recorded_with: VCR 6.2.0