Upgrade Django and dependencies; update Dockerfile and CI configurations

.github/workflows/dep_check.yml

@@ -36,9 +36,12 @@ jobs:
       - name: Install Python dependencies (Dev)
         run: |
           python3 -m pip install --upgrade pip "setuptools<81" wheel
-          python3 -m pip install --no-build-isolation -r requirements-dev.txt
+          python3 -m pip install --no-build-isolation --only-binary=numpy -r requirements-dev.txt
 
       - name: Install Python dependencies (Prod)
         run: |
           python3 -m pip install --upgrade pip "setuptools<81" wheel
-          python3 -m pip install --no-build-isolation -r requirements-dev.txt
+          python3 -m pip install --no-build-isolation --only-binary=numpy -r requirements.txt
+
+      - name: Check browser startup and API documentation
+        run: python3 tark/manage.py test tark.tests.test_browser_startup --settings=tark.settings.base

Dockerfile

@@ -1,9 +1,10 @@
 # Use an official Python runtime as a parent image
-FROM python:3.10-bookworm
+FROM python:3.12-bookworm
 
 # Set environment variables
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONWARNINGS="ignore:pkg_resources is deprecated as an API:UserWarning:coreapi.utils"
 
 # Create a user to run your application
 RUN groupadd -r myuser && useradd -r -g myuser myuser
@@ -14,6 +15,7 @@ WORKDIR /code
 # Install dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     default-libmysqlclient-dev \
+    pkg-config \
     build-essential \
     gfortran \
     libopenblas-dev \
@@ -23,7 +25,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.txt /code/
 COPY requirements-dev.txt /code/
 RUN python -m pip install --upgrade pip "setuptools<81" wheel
-RUN python -m pip install --no-build-isolation -r requirements-dev.txt
+# Use a NumPy wheel compatible with this Python version instead of building it.
+RUN python -m pip install --no-build-isolation --only-binary=numpy -r requirements-dev.txt
 
 # Copy project
 COPY . /code/
@@ -34,12 +37,21 @@ RUN chown -R myuser:myuser /code
 # Collect static files
 RUN python tark/manage.py collectstatic --noinput
 
+# Load URL configuration during the build to catch incompatible API integrations.
+RUN python tark/manage.py check --tag urls
+
+# Gunicorn stores its control socket under the runtime user's home directory.
+RUN install -d -m 0700 -o myuser -g myuser /home/myuser
+
 # Use the created user to run the application
 USER myuser
+ENV HOME=/home/myuser
 
 # Expose port
 EXPOSE 8000
 
 # Run gunicorn as the container's main process
 CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout", "120", "tark.wsgi:application"]
-# CMD ["python", "tark/manage.py", "runserver", "0.0.0.0:8000"]
+# for local testing
+# CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout", "120", "tark.wsgi:application", "--workers", "6"]
+# CMD ["python", "tark/manage.py", "runserver", "0.0.0.0:8000"]

README.md

@@ -10,7 +10,7 @@ An archive of all transcripts and its sequences from Ensembl, RefSeq and other s
 
 
 # Requirements
-- Python 3.6
+- Python 3.12 (matching the Docker image and dependency checks)
 - MySQL 5.6+
 
 
@@ -36,12 +36,30 @@ pip install -r requirements.txt
 
 ```
 
-Provide the right credentials to connect to the tark database in secrets.py (created from secrets_template.py)
-
-```
-cp tark/tark/settings/secrets_template.py tark/tark/settings/secrets.py
+The supplied `tark/tark/settings/secrets.py` reads database credentials from
+environment variables. Export these in the same terminal used to start Django,
+replacing the example values with your MySQL connection details:
+
+```sh
+export DATABASE_HOST='127.0.0.1'
+export DATABASE_PORT='3306'
+export DATABASE_NAME='your_tark_database'
+export DATABASE_USER='your_mysql_user'
+export DATABASE_PASSWORD='your_mysql_password'
+export DEBUG='1'
+export DJANGO_ALLOWED_HOSTS='localhost 127.0.0.1'
 ```
 
+`DATABASE_NAME` selects the transcript archive database; Django also connects to
+`tark_django_manager` on the same server using the same credentials. Both databases
+must exist and be accessible to that user. Use your server's hostname and port if
+MySQL is remote. An empty `DATABASE_HOST` uses the MySQL client's default local
+connection. A `.env` file is not loaded automatically.
+
+Alternatively, create `tark/tark/settings/secrets.py` from
+`tark/tark/settings/secrets_template.py` and fill in its values directly. Keep
+credentials out of version control.
+
 
 Run the migrate step with --fake-initial (No need to run the migrations as the database is already there and it is not managed by Django)
 ```
@@ -50,7 +68,8 @@ cd tark/
 ```
 
 Start the development server
-cd tark/tark
+
+From the directory containing `manage.py`:
 ```
  ./manage.py runserver localhost:9000
 ```
@@ -65,11 +84,59 @@ Run all the tests in the package:
 ./manage.py test --settings=tark.settings.test
 ```
 
+### MySQL authentication errors on macOS
+
+If startup reports that `mysql_native_password` cannot be loaded from a MySQL 9
+directory, the Python driver is linked to a client library that does not provide
+the authentication method requested by the server. Changing `PATH` alone does not
+change an already compiled driver.
+
+For this project's pinned `mysqlclient==2.2.8`, if Homebrew `mysql@8.0` and `pkg-config` are already
+installed on Apple Silicon, rebuild the driver from the repository root:
+
+```sh
+PKG_CONFIG_PATH="/opt/homebrew/opt/mysql@8.0/lib/pkgconfig" ARCHFLAGS='-arch arm64' \
+  appenv/bin/python -m pip install --force-reinstall --no-deps \
+  --no-binary=mysqlclient --no-cache-dir mysqlclient==2.2.8
+appenv/bin/python -c 'import MySQLdb; print(MySQLdb.get_client_info())'
+```
+
+The last command should report `8.0.x`. Restart Django after rebuilding.
+
 ### Using Docker
 
-To run the app using docker, build and then run the image
+From the repository root, use `compose.yaml` to build and run the application
+with the database variables exported in your current shell:
+
+```sh
+docker compose up --build -d
+docker compose logs -f web
 ```
-docker build -t tark_app .
 
-docker run -p 8000:8000 tark_app
-```
+Open <http://localhost:8000/>. Set `APP_PORT` before starting Compose to use a
+different local port. Docker must be running.
+
+API documentation is available at `/api/`, with the OpenAPI schema at
+`/api/schema/`. It uses drf-spectacular; the legacy django-rest-swagger integration
+is incompatible with the current Django REST Framework version. Existing custom
+filter descriptions are included through an OpenAPI extension.
+
+Compose requires `DATABASE_HOST`, `DATABASE_NAME`, `DATABASE_USER`, and
+`DATABASE_PASSWORD`; the password may be explicitly exported as an empty string.
+`DATABASE_PORT` defaults to `3306`. It also passes through `SECRET_KEY`, `DEBUG`,
+`DJANGO_ALLOWED_HOSTS`, and `DJANGO_ENVIRONMENT`, with local development defaults
+defined in `compose.yaml`. Export a `SECRET_KEY` for any shared deployment.
+Shell values take precedence over any Compose `.env` file.
+
+This connects to your existing MySQL server. For MySQL running on your Mac, use
+`export DATABASE_HOST='host.docker.internal'`; `localhost` and `127.0.0.1` inside
+the container refer to the container itself. For a remote database, keep its
+reachable hostname and port. Both the archive database and `tark_django_manager`
+must be accessible using the exported credentials.
+
+After changing exported variables, run `docker compose up -d` again to recreate
+the container with the updated environment. To stop and remove it:
+
+```sh
+docker compose down
+```

compose.yaml

@@ -0,0 +1,20 @@
+services:
+  web:
+    build:
+      context: .
+    image: tark_app:local
+    # Gunicorn must run from the directory containing the tark Python package.
+    working_dir: /code/tark
+    ports:
+      - "127.0.0.1:${APP_PORT:-8000}:8000"
+    environment:
+      DATABASE_HOST: "${DATABASE_HOST:?Export DATABASE_HOST before starting Compose}"
+      DATABASE_PORT: "${DATABASE_PORT:-3306}"
+      DATABASE_NAME: "${DATABASE_NAME:?Export DATABASE_NAME before starting Compose}"
+      DATABASE_USER: "${DATABASE_USER:?Export DATABASE_USER before starting Compose}"
+      # An explicitly exported empty password is supported.
+      DATABASE_PASSWORD: "${DATABASE_PASSWORD?Export DATABASE_PASSWORD before starting Compose}"
+      SECRET_KEY: "${SECRET_KEY:-local-development-only-change-me}"
+      DEBUG: "${DEBUG:-0}"
+      DJANGO_ALLOWED_HOSTS: "${DJANGO_ALLOWED_HOSTS:-localhost 127.0.0.1}"
+      DJANGO_ENVIRONMENT: "${DJANGO_ENVIRONMENT:-development}"

requirements.txt

@@ -1,14 +1,14 @@
-Django==3.2
-djangorestframework==3.11.2
+Django==6.1.3
+djangorestframework==3.18.0
 coreapi==2.3.3
 markdown==3.3.7
 django-filter==2.4.0
 django-crispy-forms==1.7.0
-django-rest-swagger==2.1.2
-mysqlclient==2.1.1
-numpy==1.23.5
-biopython==1.79
-bcbio-gff==0.6.4
+drf-spectacular==0.30.0
+mysqlclient==2.2.8
+numpy==1.26.4
+biopython==1.83
+bcbio-gff==0.7.1
 wget==3.2
 pyfaidx==0.6.2
 mysql-connector-python==8.0.31
@@ -28,7 +28,7 @@ pytz==2022.6
 requests==2.27.1
 simplejson==3.18.0
 six==1.16.0
-sqlparse==0.4.3
+sqlparse==0.5.3
 uritemplate==4.1.1
 urllib3==1.26.8
 gunicorn==23.0.0

tark/gene/drf/serializers.py

@@ -16,6 +16,7 @@
 """
 
 from rest_framework import serializers
+from drf_spectacular.utils import extend_schema_field
 from tark_drf.utils.drf_mixin import SerializerMixin
 from gene.models import Gene
 from assembly.drf.serializers import AssemblySerializer
@@ -31,6 +32,7 @@ class Meta:
         fields = '__all__'
 
 
+@extend_schema_field(str)
 class HgncNameField(serializers.RelatedField):
     def to_representation(self, value):
         if value is not None:

tark/tark/settings/base.py

@@ -37,7 +37,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
-    'rest_framework_swagger',
+    'drf_spectacular',
     'session',
     'genome',
     'assembly',
@@ -176,20 +176,13 @@
         'rest_framework.permissions.IsAuthenticatedOrReadOnly',
     ),
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
-    # The coreAPI schema generation is being deprecated, to continue using coreAPI we need to include
-    # DEFAULT_SCHEMA_CLASS explicitly as mentioned described here:
-    # https://www.django-rest-framework.org/community/3.10-announcement/#continuing-to-use-coreapi
-    'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema'
+    'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema'
 }
 
-SWAGGER_SETTINGS = {
-    "exclude_namespaces": [],  # List URL namespaces to ignore
-    "api_version": '0.1',  # Specify your API's version
-    "api_path": "/",  # Specify the path to your API not a root level
-    "api_key": '',  # An API key
-    "is_authenticated": False,  # Set to True to enforce user authentication,
-    "is_superuser": False,  # Set to True to enforce admin only access
-    'VALIDATOR_URL': None,
+SPECTACULAR_SETTINGS = {
+    'TITLE': 'Tark REST API Endpoints',
+    'VERSION': SOFTWARE_RELEASE_TAG,
+    'SERVE_INCLUDE_SCHEMA': False,
 }
 
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
@@ -223,4 +216,4 @@
             'propagate': True,
         },
     },
-}
+}

tark/tark/settings/secrets.py

@@ -21,11 +21,12 @@
 SECRET_KEY = os.getenv('SECRET_KEY', "BACKUP_SECRET")
 
 
-DATABASE_NAME = os.getenv('DATABASE_NAME')
-DATABASE_PORT = os.getenv('DATABASE_PORT')
-DATABASE_USER = os.getenv('DATABASE_USER')
-DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD')
-DATABASE_HOST = os.getenv('DATABASE_HOST')
+# Django's MySQL backend expects strings, including for unset connection options.
+DATABASE_NAME = os.getenv('DATABASE_NAME', '')
+DATABASE_PORT = os.getenv('DATABASE_PORT', '')
+DATABASE_USER = os.getenv('DATABASE_USER', '')
+DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
+DATABASE_HOST = os.getenv('DATABASE_HOST', '')
 debug_value = os.getenv('DEBUG', "0")
 DEBUG = True if debug_value == "1" else False
 ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', "").split(" ")

tark/tark/templates/docs_faqs/faq8.html

@@ -3,7 +3,7 @@ <h4 id="ar_tech_stack">What is the technology stack of Tark?</h4>
     <a href="https://www.python.org/">Python3</a>,
     <a href="https://www.djangoproject.com/">Django</a>,
     <a href="http://www.django-rest-framework.org/">Django REST Framework</a> and
-    <a href="https://django-rest-swagger.readthedocs.io/en/latest/">Django REST Swagger</a>.
+    <a href="https://drf-spectacular.readthedocs.io/">drf-spectacular</a>.
     <br>
     Backend support is provided by <a href="https://www.mysql.com/">MySQL</a>.
 </p>

tark/tark/tests/test_browser_startup.py

@@ -0,0 +1,49 @@
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+from django.urls import resolve, reverse
+
+from release.utils.release_utils import ReleaseUtils
+
+
+class BrowserStartupTests(SimpleTestCase):
+    def setUp(self):
+        # These tests exercise request handling without requiring an archive database.
+        for method, value in (
+            ('get_latest_release', '116'),
+            ('get_latest_assembly', 'GRCh38'),
+            ('get_default_source', 'Ensembl'),
+        ):
+            patcher = patch.object(ReleaseUtils, method, return_value=value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+    def test_home_page_renders(self):
+        response = self.client.get('/')
+        self.assertEqual(response.status_code, 200)
+        self.assertTemplateUsed(response, 'index.html')
+
+    def test_api_documentation_renders(self):
+        response = self.client.get('/api/')
+        self.assertContains(response, 'SwaggerUIBundle')
+        self.assertContains(response, reverse('api_schema'))
+
+    def test_existing_api_route_resolves(self):
+        self.assertEqual(resolve('/api/assembly/').url_name, 'assembly_list')
+
+    def test_openapi_preserves_filter_parameters(self):
+        response = self.client.get(reverse('api_schema'), HTTP_ACCEPT='application/json')
+        self.assertEqual(response.status_code, 200)
+        schema = response.json()
+        self.assertTrue(schema['openapi'].startswith('3.'))
+        parameters = {
+            parameter['name']: parameter
+            for parameter in schema['paths']['/api/assembly/']['get']['parameters']
+        }
+        self.assertIn('assembly_name', parameters)
+        self.assertIn('expand', parameters)
+        self.assertEqual(parameters['expand_all']['schema']['type'], 'boolean')
+        self.assertIn('loc_start', {
+            parameter['name']
+            for parameter in schema['paths']['/api/transcript/']['get']['parameters']
+        })