[To Staging] Upgrade Django to 3.2 and Python to 3.10

.dockerignore

@@ -1,2 +1,9 @@
 .env
 .env.*
+.venv*/
+.uv-python/
+.uv-bin/
+__pycache__/
+*.pyc
+.git/
+.esdata/

.github/workflows/run_tests.yml

@@ -27,7 +27,7 @@ jobs:
         ports:
           - 9200:9200
       db:
-        image: mysql:8.0
+        image: mysql:8.4
         env:
           MYSQL_DATABASE: "rorapi"
           MYSQL_USER: "ror_user"
@@ -44,16 +44,21 @@ jobs:
       - name: Set up Python environment
         uses: actions/setup-python@v6
         with:
-          python-version: "3.8"
+          python-version: "3.10"
           cache: "pip"
       - name: Install requirements
         working-directory: ./ror-api
         run: |
           # python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install yapf
+          pip check
 
           python manage.py collectstatic --noinput
+      - name: Run system checks
+        working-directory: ./ror-api
+        run: |
+          python -Wd manage.py check
+          python manage.py makemigrations --check --dry-run
       - name: Load test data
         working-directory: ./ror-api
         run: |

.gitignore

@@ -7,3 +7,6 @@ __pycache__/
 *.pyc
 rorapi/data/**
 esdata/**
+.venv310/
+.uv-python/
+.uv-bin/

Dockerfile

@@ -1,22 +1,27 @@
-FROM phusion/passenger-full:1.0.12
-MAINTAINER Martin Fenner "mfenner@datacite.org"
-
+FROM phusion/passenger-python310:3.2.0
 # Set correct environment variables
-ENV HOME /home/app
+ENV HOME=/home/app
 
 # Allow app user to read /etc/container_environment
 RUN usermod -a -G docker_env app
 
 # Use baseimage-docker's init process
 CMD ["/sbin/my_init"]
 
-# Update installed APT packages, clean up when done
-RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \
-    apt update && apt install -y ca-certificates && \
-    mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \
+# Update installed APT packages, clean up when done.
+# Keep /usr/bin/python as the image's python3.10 symlink (do not retarget to system python3).
+RUN apt-get update && \
     apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \
-    apt-get clean && \
-    apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        wget \
+        unzip \
+        tzdata \
+        libmagic1 \
+        default-libmysqlclient-dev \
+        libcairo2-dev \
+        pkg-config \
+        build-essential && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Enable Passenger and Nginx and remove the default site
@@ -26,9 +31,6 @@ RUN rm -f /etc/service/nginx/down && \
 COPY vendor/docker/webapp.conf /etc/nginx/sites-enabled/webapp.conf
 COPY vendor/docker/00_app_env.conf /etc/nginx/conf.d/00_app_env.conf
 
-# Use Amazon NTP servers
-COPY vendor/docker/ntp.conf /etc/ntp.conf
-
 # Copy webapp folder
 COPY . /home/app/webapp/
 RUN chown -R app:app /home/app/webapp && \
@@ -45,13 +47,10 @@ COPY vendor/docker/10_ssh.sh /etc/my_init.d/10_ssh.sh
 # workdir
 WORKDIR /home/app/webapp
 
-# point /usr/bin/python to Python3
-RUN ln -s -f /usr/bin/python3 /usr/bin/python
-
-# install Python packages
-RUN pip3 install --no-cache-dir --upgrade pip
-RUN pip3 install --no-cache-dir -r requirements.txt
-RUN pip3 install yapf
+# Install pip for Python 3.10 and install Python packages into that interpreter
+RUN python -m ensurepip --upgrade && \
+    python -m pip install --no-cache-dir --upgrade pip && \
+    python -m pip install --no-cache-dir -r requirements.txt
 
 # collect static files for Django
 ENV DJANGO_SKIP_DB_CHECK=True

docker-compose.yml

@@ -18,7 +18,7 @@ services:
     volumes:
       - esdata:/usr/share/elasticsearch/data
   db:
-    image: mysql:8.0
+    image: mysql:8.4
     volumes:
       - mysql_data:/var/lib/mysql
     env_file:
@@ -45,4 +45,4 @@ services:
 volumes:
   mysql_data:
   esdata:
-    driver: local
+    driver: local

requirements.txt

@@ -1,24 +1,23 @@
-Django==2.2.28
+Django==3.2.25
 elasticsearch_dsl==7.4.1
 geonamescache==1.3.0
 requests==2.32.4
 requests-aws4auth==0.9
 mock==3.0.5
 base32_crockford==0.3.0
 elasticsearch==7.10.1
-djangorestframework==3.11.2
+djangorestframework==3.12.4
 coreapi==2.3.3
-django-prometheus==1.0.15
-sentry-sdk==0.12.2
+setuptools>=40,<81
+django-prometheus==2.2.0
+sentry-sdk==1.45.1
 python-dotenv==0.10.3
-django-cors-headers==3.1.0
+django-cors-headers==3.13.0
 unidecode==1.1.1
-fuzzywuzzy==0.18.0
-python-Levenshtein==0.12.1
-statsmodels==0.10.2
+statsmodels==0.14.4
 boto3
 pandas==1.4.1
-numpy==1.22
+numpy==1.22.4
 titlecase==2.3
 update_address @ git+https://github.com/ror-community/update_address.git
 launchdarkly-server-sdk==7.6.1
@@ -29,4 +28,4 @@ rapidfuzz==3.6.1
 mysqlclient==2.2.7
 bleach==6.0.0
 pycountry==22.3.5
-django-ses==3.5.0
+django-ses==3.5.0

rorapi/common/matching.py

@@ -10,7 +10,7 @@
 
 from collections import namedtuple
 from functools import lru_cache
-from fuzzywuzzy import fuzz
+from rapidfuzz import fuzz
 from itertools import groupby
 
 MIN_CHOSEN_SCORE = 0.9

rorapi/common/urls.py

@@ -1,27 +1,26 @@
-from django.conf.urls import url, include
-from django.urls import path, re_path
+from django.urls import include, re_path
 from rest_framework.documentation import include_docs_urls
 from  . import views
 from rorapi.common.views import (
     HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView,ValidateClientView)
 
 urlpatterns = [
     # Health check
-    url(r"^(?P<version>v2)\/heartbeat$", HeartbeatView.as_view()),
-    url(r"^heartbeat$", HeartbeatView.as_view()),
+    re_path(r"^(?P<version>v2)\/heartbeat$", HeartbeatView.as_view()),
+    re_path(r"^heartbeat$", HeartbeatView.as_view()),
     # Using REST API
-    url(r"^(?P<version>v2)\/generateaddress\/(?P<geonamesid>[0-9]+)", GenerateAddress.as_view()),
-    path('generateaddress/<str:geonamesid>', GenerateAddress.as_view()),
-    url(r"^generateid$", GenerateId.as_view()),
+    re_path(r"^(?P<version>v2)\/generateaddress\/(?P<geonamesid>[0-9]+)", GenerateAddress.as_view()),
+    re_path(r"^generateaddress\/(?P<geonamesid>[^/]+)$", GenerateAddress.as_view()),
+    re_path(r"^generateid$", GenerateId.as_view()),
     re_path(r"^(?P<version>v2)\/bulkupdate$", BulkUpdate.as_view()),
     re_path(r"^(?P<version>v2)\/register$", ClientRegistrationView.as_view()),
-    path('validate-client-id/<str:client_id>/', ValidateClientView.as_view()),
-    url(r"^(?P<version>v2)\/indexdata/(?P<branch>.*)", IndexData.as_view()),
-    url(r"^(?P<version>v2)\/indexdatadump\/(?P<filename>v(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P<dataenv>(test|prod))$", IndexDataDump.as_view()),
-    url(r"^(?P<version>v2)\/", include(views.organizations_router.urls)),
-    url(r"^", include(views.organizations_router.urls)),
-    url(r"^docs/", include_docs_urls(title="Research Organization Registry")),
+    re_path(r"^validate-client-id\/(?P<client_id>[^/]+)\/$", ValidateClientView.as_view()),
+    re_path(r"^(?P<version>v2)\/indexdata/(?P<branch>.*)", IndexData.as_view()),
+    re_path(r"^(?P<version>v2)\/indexdatadump\/(?P<filename>v(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P<dataenv>(test|prod))$", IndexDataDump.as_view()),
+    re_path(r"^(?P<version>v2)\/", include(views.organizations_router.urls)),
+    re_path(r"^", include(views.organizations_router.urls)),
+    re_path(r"^docs/", include_docs_urls(title="Research Organization Registry")),
     # Prometheus
-    url("", include("django_prometheus.urls")),
+    re_path("", include("django_prometheus.urls")),
 
 ]