diff --git a/.dockerignore b/.dockerignore index 665da455..9e72c878 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,9 @@ .env .env.* +.venv*/ +.uv-python/ +.uv-bin/ +__pycache__/ +*.pyc +.git/ +.esdata/ diff --git a/.env.travis b/.env.travis deleted file mode 100644 index e69de29b..00000000 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index eef646b3..8b21e21f 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -27,7 +27,7 @@ jobs: ports: - 9200:9200 db: - image: mysql:8.0 + image: mysql:8.4 env: MYSQL_DATABASE: "rorapi" MYSQL_USER: "ror_user" @@ -44,16 +44,21 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v6 with: - python-version: "3.8" + python-version: "3.10" cache: "pip" - name: Install requirements working-directory: ./ror-api run: | # python -m pip install --upgrade pip pip install -r requirements.txt - pip install yapf + pip check python manage.py collectstatic --noinput + - name: Run system checks + working-directory: ./ror-api + run: | + python -Wd manage.py check + python manage.py makemigrations --check --dry-run - name: Load test data working-directory: ./ror-api run: | diff --git a/.gitignore b/.gitignore index c53a9f02..d4525307 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ __pycache__/ *.pyc rorapi/data/** esdata/** +.venv310/ +.uv-python/ +.uv-bin/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 22289d80..00000000 --- a/.travis.yml +++ /dev/null @@ -1,96 +0,0 @@ -language: python -python: - - "3.6" - -sudo: required - -services: - - docker - -env: - - DOCKER_COMPOSE_VERSION=1.23.2 - -before_install: - - pip3 install yapf - - yapf -d -r . - -install: - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - sudo pip install awscli - -before_script: - - cp .env.travis .env - - docker-compose up -d - - sleep 120 - - docker-compose exec web python manage.py createindex - - docker-compose exec web python manage.py upgrade - - docker-compose exec web python manage.py setup - -script: - - docker-compose exec web python manage.py test rorapi.tests - - docker-compose exec web python manage.py test rorapi.tests_integration - - docker-compose exec web python manage.py test rorapi.tests_functional - -after_success: - - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY - - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_KEY - - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; - - REPO=rorcommunity/ror-api; - - AUTO_DEPLOY=false; - - if [ "${TRAVIS_TAG?}" ]; then - docker build -f Dockerfile -t $REPO:$TRAVIS_TAG .; - docker push $REPO:$TRAVIS_TAG; - echo "Pushed to" $REPO:$TRAVIS_TAG; - AUTO_DEPLOY=true; - elif [[ "$TRAVIS_BRANCH" == "dev" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then - docker build -f Dockerfile -t $REPO:$TRAVIS_BRANCH .; - docker push $REPO:$TRAVIS_BRANCH; - echo "Pushed to" $REPO:$TRAVIS_BRANCH; - AUTO_DEPLOY=true; - else - docker build -f Dockerfile -t $REPO:$TRAVIS_BRANCH .; - docker push $REPO:$TRAVIS_BRANCH; - echo "Pushed to" $REPO:$TRAVIS_BRANCH; - fi - - - if [ "$AUTO_DEPLOY" == "true" ]; then - wget https://github.com/jwilder/dockerize/releases/download/v0.6.0/dockerize-linux-amd64-v0.6.0.tar.gz; - tar -xzvf dockerize-linux-amd64-v0.6.0.tar.gz; - rm dockerize-linux-amd64-v0.6.0.tar.gz; - export GIT_SHA=$(git rev-parse --short HEAD); - export GIT_TAG=$(git describe --tags $(git rev-list --tags --max-count=1)); - git clone "https://${TRAVIS_SECURE_TOKEN}@github.com/ror-community/new-deployment.git"; - - sentry-cli releases new ror-api:${GIT_TAG} --finalize --project ror-api; - - if [ "${TRAVIS_TAG?}" ]; then - ./dockerize -template vendor/docker/_ror-api.auto.tfvars.tmpl:new-deployment/ror/services/api/_ror-api.auto.tfvars; - sentry-cli releases deploys ror-ap:${GIT_TAG} new -e production; - else - ./dockerize -template vendor/docker/_ror-api-dev.auto.tfvars.tmpl:new-deployment/ror/services/api/_ror-api-dev.auto.tfvars; - sentry-cli releases deploys ror-api:${GIT_TAG} new -e dev; - fi - - sentry-cli releases set-commits --auto ror-api:${GIT_TAG}; - - cd new-deployment; - git remote; - git config user.email ${DOCKER_EMAIL}; - git config user.name ${DOCKER_USERNAME}; - - if [ "${TRAVIS_TAG?}" ]; then - git add ror/services/api/_ror-api.auto.tfvars; - git commit -m "Adding ror-api git variables for commit tagged ${TRAVIS_TAG?}"; - else - git add ror/services/api/_ror-api-dev.auto.tfvars; - git commit -m "Adding ror-api git variables for latest commit on branch $TRAVIS_BRANCH"; - fi - - git push "https://${TRAVIS_SECURE_TOKEN}@github.com/ror-community/new-deployment.git" master; - fi - -notifications: - email: false diff --git a/Dockerfile b/Dockerfile index 09b18d61..eaf220f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,6 @@ -FROM phusion/passenger-full:1.0.12 -MAINTAINER Martin Fenner "mfenner@datacite.org" - +FROM phusion/passenger-python310:3.2.0 # Set correct environment variables -ENV HOME /home/app +ENV HOME=/home/app # Allow app user to read /etc/container_environment RUN usermod -a -G docker_env app @@ -10,13 +8,20 @@ RUN usermod -a -G docker_env app # Use baseimage-docker's init process CMD ["/sbin/my_init"] -# Update installed APT packages, clean up when done -RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \ - apt update && apt install -y ca-certificates && \ - mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \ +# Update installed APT packages, clean up when done. +# Keep /usr/bin/python as the image's python3.10 symlink (do not retarget to system python3). +RUN apt-get update && \ apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \ - apt-get clean && \ - apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + unzip \ + tzdata \ + libmagic1 \ + default-libmysqlclient-dev \ + libcairo2-dev \ + pkg-config \ + build-essential && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Enable Passenger and Nginx and remove the default site @@ -26,9 +31,6 @@ RUN rm -f /etc/service/nginx/down && \ COPY vendor/docker/webapp.conf /etc/nginx/sites-enabled/webapp.conf COPY vendor/docker/00_app_env.conf /etc/nginx/conf.d/00_app_env.conf -# Use Amazon NTP servers -COPY vendor/docker/ntp.conf /etc/ntp.conf - # Copy webapp folder COPY . /home/app/webapp/ RUN chown -R app:app /home/app/webapp && \ @@ -45,13 +47,10 @@ COPY vendor/docker/10_ssh.sh /etc/my_init.d/10_ssh.sh # workdir WORKDIR /home/app/webapp -# point /usr/bin/python to Python3 -RUN ln -s -f /usr/bin/python3 /usr/bin/python - -# install Python packages -RUN pip3 install --no-cache-dir --upgrade pip -RUN pip3 install --no-cache-dir -r requirements.txt -RUN pip3 install yapf +# Install pip for Python 3.10 and install Python packages into that interpreter +RUN python -m ensurepip --upgrade && \ + python -m pip install --no-cache-dir --upgrade pip && \ + python -m pip install --no-cache-dir -r requirements.txt # collect static files for Django ENV DJANGO_SKIP_DB_CHECK=True diff --git a/docker-compose.yml b/docker-compose.yml index 55bc82c4..e123173f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: volumes: - esdata:/usr/share/elasticsearch/data db: - image: mysql:8.0 + image: mysql:8.4 volumes: - mysql_data:/var/lib/mysql env_file: @@ -45,4 +45,4 @@ services: volumes: mysql_data: esdata: - driver: local + driver: local \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9003df71..72739425 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==2.2.28 +Django==3.2.25 elasticsearch_dsl==7.4.1 geonamescache==1.3.0 requests==2.32.4 @@ -6,19 +6,18 @@ requests-aws4auth==0.9 mock==3.0.5 base32_crockford==0.3.0 elasticsearch==7.10.1 -djangorestframework==3.11.2 +djangorestframework==3.12.4 coreapi==2.3.3 -django-prometheus==1.0.15 -sentry-sdk==0.12.2 +setuptools>=40,<81 +django-prometheus==2.2.0 +sentry-sdk==1.45.1 python-dotenv==0.10.3 -django-cors-headers==3.1.0 +django-cors-headers==3.13.0 unidecode==1.1.1 -fuzzywuzzy==0.18.0 -python-Levenshtein==0.12.1 -statsmodels==0.10.2 +statsmodels==0.14.4 boto3 pandas==1.4.1 -numpy==1.22 +numpy==1.22.4 titlecase==2.3 update_address @ git+https://github.com/ror-community/update_address.git launchdarkly-server-sdk==7.6.1 @@ -29,4 +28,4 @@ rapidfuzz==3.6.1 mysqlclient==2.2.7 bleach==6.0.0 pycountry==22.3.5 -django-ses==3.5.0 \ No newline at end of file +django-ses==3.5.0 diff --git a/rorapi/common/matching.py b/rorapi/common/matching.py index c3c8ab17..1e772ea1 100644 --- a/rorapi/common/matching.py +++ b/rorapi/common/matching.py @@ -10,7 +10,7 @@ from collections import namedtuple from functools import lru_cache -from fuzzywuzzy import fuzz +from rapidfuzz import fuzz from itertools import groupby MIN_CHOSEN_SCORE = 0.9 diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py index 581a0214..b7c67a96 100644 --- a/rorapi/common/urls.py +++ b/rorapi/common/urls.py @@ -1,5 +1,4 @@ -from django.conf.urls import url, include -from django.urls import path, re_path +from django.urls import include, re_path from rest_framework.documentation import include_docs_urls from . import views from rorapi.common.views import ( @@ -7,21 +6,21 @@ urlpatterns = [ # Health check - url(r"^(?Pv2)\/heartbeat$", HeartbeatView.as_view()), - url(r"^heartbeat$", HeartbeatView.as_view()), + re_path(r"^(?Pv2)\/heartbeat$", HeartbeatView.as_view()), + re_path(r"^heartbeat$", HeartbeatView.as_view()), # Using REST API - url(r"^(?Pv2)\/generateaddress\/(?P[0-9]+)", GenerateAddress.as_view()), - path('generateaddress/', GenerateAddress.as_view()), - url(r"^generateid$", GenerateId.as_view()), + re_path(r"^(?Pv2)\/generateaddress\/(?P[0-9]+)", GenerateAddress.as_view()), + re_path(r"^generateaddress\/(?P[^/]+)$", GenerateAddress.as_view()), + re_path(r"^generateid$", GenerateId.as_view()), re_path(r"^(?Pv2)\/bulkupdate$", BulkUpdate.as_view()), re_path(r"^(?Pv2)\/register$", ClientRegistrationView.as_view()), - path('validate-client-id//', ValidateClientView.as_view()), - url(r"^(?Pv2)\/indexdata/(?P.*)", IndexData.as_view()), - url(r"^(?Pv2)\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), - url(r"^(?Pv2)\/", include(views.organizations_router.urls)), - url(r"^", include(views.organizations_router.urls)), - url(r"^docs/", include_docs_urls(title="Research Organization Registry")), + re_path(r"^validate-client-id\/(?P[^/]+)\/$", ValidateClientView.as_view()), + re_path(r"^(?Pv2)\/indexdata/(?P.*)", IndexData.as_view()), + re_path(r"^(?Pv2)\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), + re_path(r"^(?Pv2)\/", include(views.organizations_router.urls)), + re_path(r"^", include(views.organizations_router.urls)), + re_path(r"^docs/", include_docs_urls(title="Research Organization Registry")), # Prometheus - url("", include("django_prometheus.urls")), + re_path("", include("django_prometheus.urls")), ] diff --git a/rorapi/settings.py b/rorapi/settings.py index 3ee7aefa..1ae61ff1 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -4,10 +4,10 @@ Generated by 'django-admin startproject' using Django 2.2. For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ +https://docs.djangoproject.com/en/3.2/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ +https://docs.djangoproject.com/en/3.2/ref/settings/ """ import os @@ -34,7 +34,7 @@ load_dotenv() # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get( @@ -66,7 +66,6 @@ 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'corsheaders.middleware.CorsMiddleware', 'rorapi.middleware.cors.AlwaysAllowOriginMiddleware', - 'django.middleware.common.CommonMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -97,16 +96,19 @@ WSGI_APPLICATION = 'rorapi.wsgi.application' +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer', ), 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.URLPathVersioning', 'DEFAULT_VERSION': 'v2', 'ALLOWED_VERSIONS': ['v2'], + 'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema', } # Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases if 'collectstatic' in sys.argv and os.environ.get('DJANGO_SKIP_DB_CHECK') == 'True': DATABASES = { @@ -127,12 +129,12 @@ } # Password validation -# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [] # Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ +# https://docs.djangoproject.com/en/3.2/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -140,15 +142,13 @@ USE_I18N = True -USE_L10N = True - USE_TZ = True -CORS_ORIGIN_ALLOW_ALL = True +CORS_ALLOW_ALL_ORIGINS = True CORS_ALLOW_HEADERS = list(default_headers) + ['Client-Id'] # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ +# https://docs.djangoproject.com/en/3.2/howto/static-files/ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static/') diff --git a/rorapi/tests/tests_unit/tests_matching_v2.py b/rorapi/tests/tests_unit/tests_matching_v2.py index 1e871acf..8029c48c 100644 --- a/rorapi/tests/tests_unit/tests_matching_v2.py +++ b/rorapi/tests/tests_unit/tests_matching_v2.py @@ -184,39 +184,39 @@ class SimilarityTestCase(SimpleTestCase): V2_VERSION = 'v2' def test_get_similarity(self): - self.assertEqual( + self.assertAlmostEqual( get_similarity('University of Excellence', - 'University of Excellence'), 1) - self.assertEqual( + 'University of Excellence'), 1, places=2) + self.assertAlmostEqual( get_similarity('univ. of excellençë', 'Univërsity of Excellence'), - 1) - self.assertEqual( + 1, places=2) + self.assertAlmostEqual( get_similarity('of Excellence University', - 'University of Excellence'), 1) - self.assertEqual( + 'University of Excellence'), 1, places=2) + self.assertAlmostEqual( get_similarity('of excellençë univ', 'University of Excellence'), - 1) - self.assertEqual( + 1, places=2) + self.assertAlmostEqual( get_similarity('Excellence University', - 'University of Excellence'), 0.93) - self.assertEqual( + 'University of Excellence'), 0.93, places=2) + self.assertAlmostEqual( get_similarity('excellençë univ', 'University of Excellence'), - 0.93) - self.assertEqual( + 0.93, places=2) + self.assertAlmostEqual( get_similarity('University of Exçellence', - 'University of Excellence (Gallifrey)'), 1) - self.assertEqual( + 'University of Excellence (Gallifrey)'), 1, places=2) + self.assertAlmostEqual( get_similarity('University of Excellence and Brilliance', - 'University of Excellence'), 0.76) - self.assertEqual( + 'University of Excellence'), 0.76, places=2) + self.assertAlmostEqual( get_similarity('University of Excellence (and Brilliance)', - 'University of Excellence'), 1) - self.assertEqual( + 'University of Excellence'), 1, places=2) + self.assertAlmostEqual( get_similarity('University of Excellence School of Perseverance', - 'University of Excellence'), 1) - self.assertEqual( + 'University of Excellence'), 1, places=2) + self.assertAlmostEqual( get_similarity('University of Excellence Mediocrity Hospital', - 'University of Excellence'), 1) + 'University of Excellence'), 1, places=2) def test_get_score(self): empty = { diff --git a/rorapi/v2/tests.py b/rorapi/v2/tests.py index 5e2c9ef3..02b18058 100644 --- a/rorapi/v2/tests.py +++ b/rorapi/v2/tests.py @@ -1,11 +1,12 @@ from django.test import TestCase -from .models.client import Client +from rorapi.v2.models import Client class ClientTests(TestCase): def test_client_registration(self): client = Client.objects.create(email='test@example.com') self.assertIsNotNone(client.client_id) - def test_rate_limiting(self): - response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") - self.assertEqual(response.status_code, 429) + def test_validate_client_id(self): + response = self.client.get('/validate-client-id/INVALID_ID/') + self.assertEqual(response.status_code, 200) + self.assertFalse(response.json()['valid']) diff --git a/vendor/docker/ntp.conf b/vendor/docker/ntp.conf deleted file mode 100644 index b0752211..00000000 --- a/vendor/docker/ntp.conf +++ /dev/null @@ -1,4 +0,0 @@ -server 0.amazon.pool.ntp.org iburst -server 1.amazon.pool.ntp.org iburst -server 2.amazon.pool.ntp.org iburst -server 3.amazon.pool.ntp.org iburst diff --git a/vendor/docker/webapp.conf b/vendor/docker/webapp.conf index 09e0ba46..5319824a 100644 --- a/vendor/docker/webapp.conf +++ b/vendor/docker/webapp.conf @@ -4,7 +4,7 @@ server { root /home/app/webapp/; passenger_enabled on; - passenger_python /usr/bin/python; + passenger_python /usr/bin/python3.10; passenger_app_root /home/app/webapp/; passenger_user app; passenger_app_type wsgi;