sal-uva · Sep 16, 2026 · Sep 16, 2026 · Sep 17, 2026 · Sep 17, 2026 · Sep 17, 2026
diff --git a/AGENTS.md b/AGENTS.md
@@ -118,7 +118,7 @@ Invocation (the plain `docker compose` command uses `docker-compose.yml`):
 - **Extensions**: Installed under `config/extensions/`. Each extension can include its own `requirements.txt` (auto-installed by `setup.py`). Enabled/disabled via `extensions.enabled` setting.
 
 ## Versioning and Migrations
-- The current version is stored in the `VERSION` file (currently `1.53`). **Do not edit this file casually.**
+- The current version is stored in the `VERSION` file. **Do not edit this file casually.**
 - `config/.current-version` tracks the last-migrated version for the running instance.
 - `helper-scripts/migrate.py` compares `VERSION` to `.current-version` and runs the appropriate `migrate-X.XX-X.XX.py` scripts from `helper-scripts/migrate/` in sequence.
 - Migration scripts handle database schema changes, data transformations, and config updates between versions.

diff --git a/VERSION b/VERSION
@@ -1,4 +1,4 @@
-1.57
+1.58
 
 This file should not be modified. It is used by 4CAT to determine whether it
 needs to run migration scripts to e.g. update the database structure to a more

diff --git a/backend/lib/manager.py b/backend/lib/manager.py
@@ -236,8 +236,11 @@ def request_interrupt(self, interrupt_level, job):
 		for queue_id, worker in self.iterate_active_workers():
 			if worker.job.data["id"] == job.data["id"]:
 				# first cancel any interruptable postgres queries for this job's worker
-				while True:
-					active_queries = self.queue.get_all_jobs("cancel-pg-query", remote_id=worker.db.appname, restrict_claimable=False)
+				# a worker that has not connected to the database yet (it is still
+				# starting up, or failed to connect) has no queries to cancel
+				worker_db = getattr(worker, "db", None)
+				while worker_db:
+					active_queries = self.queue.get_all_jobs("cancel-pg-query", remote_id=worker_db.appname, restrict_claimable=False)
 					if not active_queries:
 						# all cancellation jobs have been run
 						break

diff --git a/backend/lib/processor.py b/backend/lib/processor.py
@@ -10,7 +10,6 @@
 import abc
 import csv
 import os
-import re
 import time
 
 from pathlib import PurePath, Path
@@ -19,7 +18,7 @@
 from common.lib.dataset import DataSet, StatusType
 from common.lib.compatibility import Compatibility
 from common.lib.fourcat_module import FourcatModule
-from common.lib.helpers import get_software_commit, remove_nuls, send_email, hash_to_md5
+from common.lib.helpers import get_software_commit, remove_nuls, send_email, hash_to_md5, looks_like_email_address
 from common.lib.exceptions import (WorkerInterruptedException, ProcessorInterruptedException, ProcessorException,
                                    DataSetException, MapItemException, AnnotationException)
 from common.config_manager import ConfigWrapper
@@ -372,7 +371,17 @@ def after_process(self):
                     # already finished, could happen (though it shouldn't)
                     pass
 
-                surrogate.update_status(self.dataset.get_status())
+                # status updates of the steps are copied to the preset, so it usually has this status already;
+                # setting it again would only repeat the line in the preset's log
+                if surrogate.get_status() != self.dataset.get_status():
+                    surrogate.update_status(self.dataset.get_status())
+
+                # the steps of a preset are hidden, and their logs have more details than the preset's own
+                surrogate.log("Each step of this preset has its own log, with more details (e.g. why individual items failed):")
+                for step in surrogate.get_all_children():
+                    step_processor = step.get_producer_processor()
+                    step_name = step_processor.title if step_processor else step.type
+                    surrogate.log(f"  {step_name}: {step.get_log_url()}")
 
             except DataSetException:
                 # dataset with key to attach to doesn't exist...
@@ -390,17 +399,7 @@ def after_process(self):
         if self.config.get('mail.server') and self.dataset.get_parameters().get("email-complete", False):
             owner = self.dataset.get_parameters().get("email-complete", False)
             # Check that username is email address
-            if re.match(r"[^@]+\@.*?\.[a-zA-Z]+", owner):
-                from email.mime.multipart import MIMEMultipart
-                from email.mime.text import MIMEText
-                from smtplib import SMTPException
-                import socket
-                import html2text
-
-        if self.config.get('mail.server') and self.dataset.get_parameters().get("email-complete", False):
-            owner = self.dataset.get_parameters().get("email-complete", False)
-            # Check that username is email address
-            if re.match(r"[^@]+\@.*?\.[a-zA-Z]+", owner):
+            if looks_like_email_address(owner):
                 from email.mime.multipart import MIMEMultipart
                 from email.mime.text import MIMEText
                 from smtplib import SMTPException

diff --git a/backend/lib/proxied_requests.py b/backend/lib/proxied_requests.py
@@ -24,21 +24,45 @@ def wrapper(self, *args, **kwargs):
 
     return wrapper
 
+
+def remove_proxy_credentials(proxy_url):
+    """
+    Remove the username and password from a proxy URL
+
+    Proxy URLs can contain a username and password, as in
+    `http://user:password@host:port`. Processors may write the proxy they
+    were given to dataset logs or results that users can see, and the backend
+    log may be forwarded elsewhere (e.g. to Slack), so use this on any proxy
+    URL that is passed on to processors or written to a log.
+
+    :param str|None proxy_url:  Proxy URL
+    :return str|None:  The same URL, without anything before the last `@`
+    """
+    if not proxy_url:
+        return proxy_url
+
+    scheme, separator, rest = proxy_url.partition("://")
+    if not separator:
+        scheme, rest = "", proxy_url
+
+    return scheme + separator + rest.rpartition("@")[2]
+
 class FailedProxiedRequest:
     """
     A delegated request that has failed for whatever reason
 
     The failure context (usually the exception) is stored in the `context`
     property. We also keep track of the proxy URL that serviced the request so
-    downstream consumers can make informed retry decisions.
+    downstream consumers can make informed retry decisions. Any username and
+    password are removed from that URL.
     """
 
     context = None
     proxy_url = None
 
     def __init__(self, context=None, proxy_url=None):
         self.context = context
-        self.proxy_url = proxy_url
+        self.proxy_url = remove_proxy_credentials(proxy_url)
 
 
 class NoProxiesAvailableError(Exception):
@@ -175,7 +199,7 @@ def release_cooled_off(self):
                     and request.timestamp_finished < time.time() - self.COOLOFF
                 ):
                     self.log.debug(
-                        f"Releasing proxy {self.proxy_url} for host name {hostname}"
+                        f"Releasing proxy {remove_proxy_credentials(self.proxy_url)} for host name {hostname}"
                     )
                     self.hostnames[hostname].running.remove(request)
 
@@ -214,7 +238,7 @@ def claim_for(self, url):
             request.timestamp_finished = 0
             self.hostnames[hostname].running.append(request)
             self.log.debug(
-                f"Claiming proxy {self.proxy_url} for host name {hostname} ({len(self.hostnames[hostname].running)} of {self.MAX_CONCURRENT_PER_HOST} for host)"
+                f"Claiming proxy {remove_proxy_credentials(self.proxy_url)} for host name {hostname} ({len(self.hostnames[hostname].running)} of {self.MAX_CONCURRENT_PER_HOST} for host)"
             )
             return self
         else:
@@ -465,11 +489,11 @@ def _update_proxy_pool(self):
                     if proxy_url in self.proxy_health:
                         del self.proxy_health[proxy_url]
                     self.proxies_pending_removal.discard(proxy_url)
-                    self.log.info(f"Removed proxy {proxy_url} (no longer in config)")
+                    self.log.info(f"Removed proxy {remove_proxy_credentials(proxy_url)} (no longer in config)")
                 else:
                     # Has active requests - mark for removal but don't delete yet
                     self.proxies_pending_removal.add(proxy_url)
-                    self.log.warning(f"Proxy {proxy_url} marked for removal (has active requests, will not accept new requests)")
+                    self.log.warning(f"Proxy {remove_proxy_credentials(proxy_url)} marked for removal (has active requests, will not accept new requests)")
 
         # Update settings for existing proxies
         for proxy_url in proxies_to_keep:
@@ -484,7 +508,7 @@ def _update_proxy_pool(self):
         # Add new proxies
         for proxy_url in proxies_to_add:
             self._add_proxy_to_pool(proxy_url)
-            self.log.info(f"Added new proxy {proxy_url}")
+            self.log.info(f"Added new proxy {remove_proxy_credentials(proxy_url)}")
 
         if proxies_to_add or proxies_to_remove:
             self.log.info(f"Proxy pool updated: {len(self.proxy_pool)} total proxies "
@@ -635,15 +659,16 @@ def manage_requests(self):
                             if proxy_url in self.proxy_health:
                                 del self.proxy_health[proxy_url]
                             self.proxies_pending_removal.discard(proxy_url)
-                            self.log.info(f"Removed proxy {proxy_url} (completed all active requests)")
+                            self.log.info(f"Removed proxy {remove_proxy_credentials(proxy_url)} (completed all active requests)")
                     try:
                         response = url_metadata.proxied.request.result()
                         # annotate the response so processors can see which
-                        # proxy (if any) handled the request
+                        # proxy (if any) handled the request, without its
+                        # username and password
                         setattr(
                             response,
                             "_4cat_proxy",
-                            url_metadata.proxied.proxy.proxy_url,
+                            remove_proxy_credentials(url_metadata.proxied.proxy.proxy_url),
                         )
                         url_metadata.proxied.result = response
 
@@ -654,7 +679,7 @@ def manage_requests(self):
                         probe_succeeded = url_metadata.proxied.proxy.probe()
                         if probe_succeeded:
                             self.log.debug(
-                                f"Proxy {proxy_url} returned a proxy error for {url}, "
+                                f"Proxy {remove_proxy_credentials(proxy_url)} returned a proxy error for {url}, "
                                 f"but health probe to {url_metadata.proxied.proxy.healthcheck_url} succeeded; "
                                 f"passing original error back to requester"
                             )
@@ -667,7 +692,7 @@ def manage_requests(self):
                             if proxy_url not in self.proxy_warnings_logged:
                                 self.proxy_warnings_logged.add(proxy_url)
                                 self.log.warning(
-                                    f"Proxy {proxy_url} marked as unhealthy due to connection failure and failed health probe "
+                                    f"Proxy {remove_proxy_credentials(proxy_url)} marked as unhealthy due to connection failure and failed health probe "
                                     f"({url_metadata.proxied.proxy.healthcheck_url}): {str(e)}"
                                 )
 

diff --git a/backend/workers/api.py b/backend/workers/api.py
@@ -1,3 +1,4 @@
+import traceback
 import socket
 import time
 import json
@@ -141,18 +142,28 @@ def api_response(self, client, address):
 
 		self.manager.log.debug("Received API request from %s:%s" % address)
 
+		request = None
 		try:
 			payload = json.loads(buffer)
 			if "request" not in payload:
 				raise InternalAPIException
 
-			response = self.process_request(payload["request"], payload)
+			request = payload["request"]
+			response = self.process_request(request, payload)
 			if not response:
 				raise InternalAPIException
 
 			response = json.dumps({"error": False, "response": response})
 		except (json.JSONDecodeError, InternalAPIException):
-			response = json.dumps({"error": "Invalid JSON"})
+			response = json.dumps({"error": "Invalid JSON", "response": None})
+		except Exception as e:
+			# an error while answering one request should not stop the API
+			# worker, because then nothing can reach the backend until it is
+			# restarted - so log it, answer with an error, and keep listening
+			stack = traceback.extract_tb(e.__traceback__)
+			location = "->".join([frame.filename.split("/").pop() + ":" + str(frame.lineno) for frame in stack])
+			self.manager.log.error(f"API request {request} raised exception {e.__class__.__name__}: {e} at {location}", frame=stack)
+			response = json.dumps({"error": f"Internal error ({e.__class__.__name__})", "response": None})
 
 		try:
 			response = client.sendall(response.encode("ascii"))

diff --git a/common/assets/llms.json b/common/assets/llms.json
@@ -1,4 +1,25 @@
 [
+  {
+    "model": "gpt-5.6-sol",
+    "name": "[OpenAI] GPT-5.6 Sol",
+    "model_card": "https://developers.openai.com/api/docs/models/gpt-5.6-sol",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-terra",
+    "name": "[OpenAI] GPT-5.6 Terra",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-terra",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-luna",
+    "name": "[OpenAI] GPT-5.6 Luna",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-luna",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.5",
+    "name": "[OpenAI] GPT-5.5",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.5",
+    "provider": "openai"
+  },
   {
     "model": "gpt-5.5",
     "name": "[OpenAI] GPT-5.5",
@@ -48,44 +69,32 @@
     "provider": "google"
   },
   {
-    "model": "gemini-3.5-flash",
-    "name": "[Google] Gemini 3.5 Flash",
-    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash",
+    "model": "gemini-3.8-flash",
+    "name": "[Google] Gemini 3.8 Flash",
+    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash",
     "provider": "google"
   },
   {
-    "model": "gemini-3-flash-preview",
-    "name": "[Google] Gemini 3 Flash",
-    "model_card": "https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-flash",
+    "model": "gemini-3.5-flash-lite-preview",
+    "name": "[Google] Gemini 3.5 Flash Lite",
+    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-5-flash-lite",
     "provider": "google"
   },
   {
-    "model": "gemini-3.1-flash-lite-preview",
-    "name": "[Google] Gemini 3.1 Flash Lite",
-    "provider": "google",
-    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-flash-lite"
-  },
-  {
-    "model": "claude-fable-5",
-    "name": "[Anthropic] Claude Fable 5",
+    "model": "claude-fable-5-1",
+    "name": "[Anthropic] Claude Fable 5.1",
     "model_card": "https://www.anthropic.com/claude/fable",
     "provider": "anthropic"
   },
   {
-    "model": "claude-opus-4-8",
-    "name": "[Anthropic] Claude Opus 4.8",
-    "model_card": "https://www.anthropic.com/claude/opus",
-    "provider": "anthropic"
-  },
-  {
-    "model": "claude-opus-4-6",
-    "name": "[Anthropic] Claude Opus 4.6",
+    "model": "claude-opus-5",
+    "name": "[Anthropic] Claude Opus 5",
     "model_card": "https://www.anthropic.com/claude/opus",
     "provider": "anthropic"
   },
   {
-    "model": "claude-sonnet-4-6",
-    "name": "[Anthropic] Claude Sonnet 4.6 (latest)",
+    "model": "claude-sonnet-5",
+    "name": "[Anthropic] Claude Sonnet 5 (latest)",
     "model_card": "https://www.anthropic.com/claude/sonnet",
     "provider": "anthropic"
   },

diff --git a/common/assets/prompt_library.json b/common/assets/prompt_library.json
@@ -506,5 +506,29 @@
     "authors": "OpenAI (2023)",
     "paper": "https://help.openai.com/en/articles/6654000-best-practices-for-prompt-engineering-with-openai-api",
     "location_of_input": "replace"
+  },
+  {
+    "name": "political party affiliation",
+    "label": "party",
+    "prompt": [
+      "You will be given a set of Twitter posts from different <country> politicians, sent during the two months preceding the <year> <country> election, that is, between <date>, and <date>. Your task is to use your knowledge of <country> politics to make an educated guess on whether the poster is a <party1> or <party2>. Respond either ‘<party1>’ or ‘<party2>’. If the message does not have enough information for an educated guess, just make your best guess.",
+      "",
+      "Post: [body]"
+    ],
+    "authors": "Törnberg (2024)",
+    "paper": "https://journals.sagepub.com/doi/10.1177/08944393241286471",
+    "location_of_input": "replace"
+  },
+  {
+    "name": "Narrative and stance",
+    "label": "narrative",
+    "prompt": [
+      "You are a narratologist tasked with mapping out five narratives of the provided text and categorizing the text about the Russia–Ukraine war. Please output a labeled list with two items: (1) the narrative in English (summarized within 20 words) and (2) the stance of the narrative (pro-Russian, pro-Ukraine, or neutral). Below are some examples of the potential coded languages of pro-Russian, pro-Ukraine, and neutral narratives for your reference: (1) Pro-Russian refers to Ukraine soldiers as 乌纳 (UkNazi) or 乌贼 (squid); (2) Pro-Ukraine criticizes Russian army with sarcasm 菜鹅 (Veggie Goose/Weak Russian Army), 晋军 (Jin Dynasty/Putin’s Army), 晋凉 (Cold Putin/Putin is over), 鹅 粉 (Fans of the Goose/Fans of Putin); and (3) Neutral provides factual updates and describes battlefield developments without assigning blame.",
+      "",
+      "Text: [body]"
+    ],
+    "authors": "Rogers & Zhang (2024)",
+    "paper": "https://journals.sagepub.com/doi/10.1177/20563051241254379",
+    "location_of_input": "replace"
   }
 ]
diff --git a/common/lib/config_definition.py b/common/lib/config_definition.py
@@ -636,13 +636,13 @@
                 "type": UserInput.OPTION_TEXT,
                 "help": "Authentication Header",
                 "default": "",
-                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value.",
+                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value. For a bearer token you can write either 'Authorization' with 'Bearer <key>' as the value, or 'Authorization: Bearer' with just the key.",
             },
             "auth_key": {
                 "type": UserInput.OPTION_TEXT,
                 "default": "",
                 "help": "Authentication Key",
-                "tooltip": "The API key to access the LLM server, if required.",
+                "tooltip": "The API key to access the LLM server, if required. 'Bearer <key>' for bearer tokens.",
             },
         }
     },