Flask: refactor for clarity, then regularly poll slurm for job status

Dockerfile

@@ -15,8 +15,9 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 COPY . .
 ENV FLASK_ENV production
+ENV TZ America/Toronto
 ENV PROMETHEUS_MULTIPROC_DIR /tmp
 EXPOSE 5000 8080
 # Prevent accidentally using this image for development by adding the prod server arguments in the entrypoint
 # Automatically run migrations on startup
-ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"]
+ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000"]

docker-compose.ccm.yaml

@@ -8,7 +8,7 @@ x-common: &common
 x-app: &app
   image: "ghcr.io/ccmbioinfo/stager:${ST_VERSION}"
   user: www-data
-  command: --preload --workers ${GUNICORN_WORKERS:-1}
+  command: --workers ${GUNICORN_WORKERS:-1}
   tmpfs:
     - /tmp
   healthcheck:
@@ -130,6 +130,8 @@ services:
   #     - app-hawkins
   proxy:
     image: traefik:2.6
+    environment:
+      TZ: America/Toronto
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - "${PROJECT_ROOT:-.}/traefik:/etc/traefik"

docker-compose.cheo.yaml

@@ -26,7 +26,7 @@ services:
     ports:
       - "5000:5000"
       - "9121:8080"
-    command: --preload --workers ${GUNICORN_WORKERS:-1}
+    command: --workers ${GUNICORN_WORKERS:-1}
     tmpfs:
       - /tmp
     volumes:

docker-compose.test.yaml

@@ -4,13 +4,15 @@ services:
   mysql:
     image: mysql:8.0
     environment:
+      TZ: America/Toronto
       MYSQL_DATABASE: "${TEST_MYSQL_DATABASE}"
       MYSQL_USER: "${TEST_MYSQL_USER}"
       MYSQL_PASSWORD: "${TEST_MYSQL_PASSWORD}"
       MYSQL_ROOT_PASSWORD: "${TEST_MYSQL_ROOT_PASSWORD}"
   minio:
     image: minio/minio:RELEASE.2021-11-03T03-36-36Z
     environment:
+      TZ: America/Toronto
       MINIO_ROOT_USER: "${TEST_MINIO_ACCESS_KEY}"
       MINIO_ROOT_PASSWORD: "${TEST_MINIO_SECRET_KEY}"
       MINIO_REGION_NAME:

docker-compose.yaml

@@ -17,6 +17,7 @@ services:
     image: mysql:8.0
     restart: on-failure
     environment:
+      TZ: America/Toronto
       MYSQL_DATABASE:
       MYSQL_USER:
       MYSQL_PASSWORD:
@@ -29,6 +30,7 @@ services:
     image: minio/minio:RELEASE.2021-11-03T03-36-36Z
     restart: on-failure
     environment:
+      TZ: America/Toronto
       MINIO_ROOT_USER: "${MINIO_ACCESS_KEY}"
       MINIO_ROOT_PASSWORD: "${MINIO_SECRET_KEY}"
       MINIO_REGION_NAME:

flask/Dockerfile

@@ -11,6 +11,7 @@ COPY requirements-dev.txt .
 RUN pip3 install --no-cache-dir -r requirements-dev.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 ENV FLASK_ENV development
+ENV TZ America/Toronto
 # Prevent accidentally running this image in production.
 # Must mount the source code in the working directory to run this image.
 EXPOSE 5000

flask/app/__init__.py

@@ -1,119 +1,27 @@
-import atexit
 import logging
 import os
 from stat import S_ISFIFO
 
-from apscheduler.schedulers.background import BackgroundScheduler
-from flask import Flask, logging as flask_logging
-from slurm_rest import Configuration
-
-from app import (
-    analyses,
-    buckets,
-    datasets,
-    error_handler,
-    families,
-    genes,
-    groups,
-    manage,
-    participants,
-    routes,
-    tissue_samples,
-    users,
-    variants,
-)
-from .extensions import db, login, ma, metrics, migrate, oauth
-from .tasks import send_email_notification
-from .utils import DateTimeEncoder
+from .blueprints import register_blueprints
+from .manage import register_commands
+from .models import db
+from .stager import Stager
 
 
 def create_app(config):
     """
     The application factory. Returns an instance of the app.
     """
     # Create the application object
-    app = Flask(__name__)
-    app.config.from_object(config)
-    app.json_encoder = DateTimeEncoder
-
+    app = Stager(config, db, __name__)
     config_logger(app)
-
-    if app.config["SLURM_ENDPOINT"]:
-        app.logger.info(
-            "Configuring with Slurm REST API %s", app.config["SLURM_ENDPOINT"]
-        )
-        # Could instead use one environment variable and urllib.parse.urlsplit for this
-        app.config["slurm"] = Configuration(
-            host=app.config["SLURM_ENDPOINT"],
-            api_key={
-                "user": app.config["SLURM_USER"],
-                "token": app.config["SLURM_JWT"],
-            },
-        )
-    else:
-        app.config["slurm"] = None
-
-    register_extensions(app)
-    manage.register_commands(app)
+    register_commands(app)
     register_blueprints(app)
-    if os.getenv("SENDGRID_API_KEY"):
-        register_schedulers(app)
-
     return app
 
 
-def register_schedulers(app):
-    scheduler = BackgroundScheduler(timezone="America/Toronto")
-    scheduler.add_job(
-        send_email_notification, "cron", [app], day_of_week="mon-fri", hour="9"
-    )
-
-    scheduler.start()
-
-    # Shut down the scheduler when exiting the app
-    atexit.register(scheduler.shutdown)
-
-
-def register_blueprints(app):
-
-    app.register_blueprint(routes.routes)
-
-    app.register_blueprint(families.family_blueprint)
-    app.register_blueprint(datasets.datasets_blueprint)
-    app.register_blueprint(participants.participants_blueprint)
-    app.register_blueprint(tissue_samples.tissue_blueprint)
-    app.register_blueprint(analyses.analyses_blueprint)
-    app.register_blueprint(genes.genes_blueprint)
-    app.register_blueprint(variants.variants_blueprint)
-
-    app.register_blueprint(buckets.bucket_blueprint)
-    app.register_blueprint(groups.groups_blueprint)
-    app.register_blueprint(users.users_blueprint)
-
-    app.register_blueprint(error_handler.error_blueprint)
-
-
-def register_extensions(app):
-    db.init_app(app)
-    ma.init_app(app)
-    migrate.init_app(app, db)
-    login.init_app(app)
-    oauth.init_app(app)
-    oauth.register(
-        name=app.config["OIDC_PROVIDER"],
-        client_id=app.config["OIDC_CLIENT_ID"],
-        client_secret=app.config["OIDC_CLIENT_SECRET"],
-        server_metadata_url=app.config["OIDC_WELL_KNOWN"],
-        client_kwargs={"scope": "openid"},
-    )
-    metrics.init_app(app)
-    metrics.info("stager", "Stager process info", revision=app.config.get("GIT_SHA"))
-
-
 def config_logger(app):
     """
-    Configure the main loggers: Flask application, SQLAlchemy, and werkzeug
-
     SQLAlchemy logs can be very noisy and hard to filter out with grep because
     the logged queries can span multiple lines. However, they are still useful
     for auditing query efficiency and performance. Therefore, instead of using
@@ -149,9 +57,3 @@ def config_logger(app):
         logging.getLogger("sqlalchemy").addHandler(handler)
         logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
         logging.getLogger("sqlalchemy.pool").setLevel(logging.INFO)
-    # This configures Flask's logger and then we can customize it after
-    flask_logging.create_logger(app)
-    # %(asctime)s may be useful in development but redundant in production with journald
-    flask_logging.default_handler.setFormatter(
-        logging.Formatter("%(levelname)s [%(funcName)s, line %(lineno)s]: %(message)s")
-    )

flask/app/blueprints/__init__.py

@@ -0,0 +1,32 @@
+from flask import Flask
+
+from .misc import routes
+from .families import family_blueprint
+from .datasets import datasets_blueprint
+from .participants import participants_blueprint
+from .tissue_samples import tissue_blueprint
+from .analyses import analyses_blueprint
+from .genes import genes_blueprint
+from .variants import variants_blueprint
+from .unlinked import bucket_blueprint
+from .groups import groups_blueprint
+from .users import users_blueprint
+from .error_handler import error_blueprint
+
+
+def register_blueprints(app: Flask) -> None:
+    app.register_blueprint(routes)
+
+    app.register_blueprint(family_blueprint)
+    app.register_blueprint(datasets_blueprint)
+    app.register_blueprint(participants_blueprint)
+    app.register_blueprint(tissue_blueprint)
+    app.register_blueprint(analyses_blueprint)
+    app.register_blueprint(genes_blueprint)
+    app.register_blueprint(variants_blueprint)
+
+    app.register_blueprint(bucket_blueprint)
+    app.register_blueprint(groups_blueprint)
+    app.register_blueprint(users_blueprint)
+
+    app.register_blueprint(error_blueprint)

flask/app/analyses.py → flask/app/blueprints/analyses.py

@@ -8,11 +8,11 @@
 from sqlalchemy.orm import aliased, joinedload, selectinload
 from sqlalchemy.sql.expression import cast
 
-from . import models
-from .extensions import db
-from .schemas import AnalysisSchema
-from .slurm import run_crg2_on_family
-from .utils import (
+from .. import models
+from ..models import db
+from ..schemas import AnalysisSchema
+from ..slurm import run_crg2_on_family
+from ..utils import (
     check_admin,
     clone_entity,
     csv_response,

flask/app/datasets.py → flask/app/blueprints/datasets.py

@@ -13,10 +13,10 @@
 from sqlalchemy import distinct, func, select
 from sqlalchemy.orm import contains_eager, joinedload, selectinload
 
-from . import models
-from .extensions import db
-from .schemas import RNASeqDatasetSchema
-from .utils import (
+from .. import models
+from ..models import db
+from ..schemas import RNASeqDatasetSchema
+from ..utils import (
     check_admin,
     csv_response,
     expects_csv,

flask/app/error_handler.py → flask/app/blueprints/error_handler.py

@@ -1,6 +1,7 @@
-import requests
 import traceback
+
 from flask import Blueprint, json, jsonify, request, current_app as app
+import requests
 from werkzeug.exceptions import HTTPException
 
 

flask/app/families.py → flask/app/blueprints/families.py

@@ -3,10 +3,11 @@
 from flask import abort, jsonify, request, Blueprint, current_app as app
 from flask_login import current_user, login_required
 from sqlalchemy.orm import joinedload
-from .extensions import db
-from . import models
-from .schemas import FamilySchema
-from .utils import (
+
+from .. import models
+from ..models import db
+from ..schemas import FamilySchema
+from ..utils import (
     check_admin,
     filter_datasets_by_user_groups,
     get_current_user,

flask/app/genes.py → flask/app/blueprints/genes.py

@@ -1,11 +1,13 @@
 from dataclasses import asdict
 from typing import Any, Dict
+
 from flask import abort, jsonify, request, Blueprint
 from flask_login import login_required
 from sqlalchemy import func
 from sqlalchemy.orm import contains_eager, joinedload
-from .models import Gene, GeneAlias
-from .utils import csv_response, expects_csv, expects_json, paged, paginated_response
+
+from ..models import Gene, GeneAlias
+from ..utils import csv_response, expects_csv, expects_json, paged, paginated_response
 
 genes_blueprint = Blueprint(
     "genes",

flask/app/groups.py → flask/app/blueprints/groups.py

@@ -2,13 +2,12 @@
 
 from flask import abort, jsonify, request, Response, Blueprint, current_app as app
 from flask_login import login_required
-from minio import Minio
 
-from . import models
-from .extensions import db
-from .madmin import stager_buckets_policy
-from .schemas import GroupSchema
-from .utils import (
+from .. import models
+from ..models import db
+from ..madmin import stager_buckets_policy
+from ..schemas import GroupSchema
+from ..utils import (
     check_admin,
     get_current_user,
     get_minio_admin,

flask/app/routes.py → flask/app/blueprints/misc.py

@@ -10,9 +10,9 @@
 import numpy as np
 import pandas as pd
 from sqlalchemy.orm import joinedload
-from . import models, schemas
-from .extensions import db, oauth
-from .utils import (
+from .. import models, schemas
+from ..models import db
+from ..utils import (
     get_current_user,
     transaction_or_abort,
     validate_json,
@@ -92,7 +92,7 @@ def oidc_login():
         abort(405)
     provider = app.config.get("OIDC_PROVIDER")
     app.logger.debug(f"Creating client with {provider}...")
-    client = oauth.create_client(provider)
+    client = app.oauth.create_client(provider)
     app.logger.debug("Building redirect url...")
     # It's safe to take redirect_uris from the client since
     # the OAuth provider maintains a list of valid redirect_uris
@@ -114,7 +114,7 @@ def authorize():
     """
     if not app.config.get("ENABLE_OIDC"):
         abort(404)
-    client = oauth.create_client(app.config.get("OIDC_PROVIDER"))
+    client = app.oauth.create_client(app.config.get("OIDC_PROVIDER"))
     # Exchange authorization code for token
     token = client.authorize_access_token()
     userinfo = client.parse_id_token(token)
@@ -161,7 +161,7 @@ def logout():
     if app.config.get("ENABLE_OIDC"):
         # Log out of OAuth session as well as Stager session
         provider = app.config.get("OIDC_PROVIDER")
-        client = oauth.create_client(provider)
+        client = app.oauth.create_client(provider)
         client_id = app.config.get("OIDC_CLIENT_ID")
         metadata = client.load_server_metadata()  # .well-known/openid-configuration
         if username:

flask/app/participants.py → flask/app/blueprints/participants.py

@@ -5,12 +5,11 @@
 from flask_login import current_user, login_required
 from sqlalchemy import distinct, func, select
 from sqlalchemy.orm import contains_eager, joinedload
-from sqlalchemy.orm.exc import NoResultFound
 
-from . import models
-from .extensions import db
-from .schemas import ParticipantSchema
-from .utils import (
+from .. import models
+from ..models import db
+from ..schemas import ParticipantSchema
+from ..utils import (
     check_admin,
     csv_response,
     expects_csv,