diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 401b02367d..21dce0955e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,10 +67,10 @@ jobs: cd server poetry install --with dev - # - name: Run Ruff check - # run: | - # cd server - # poetry run ruff check --output-format=github . + - name: Run Ruff check + run: | + cd server + poetry run ruff check --output-format=github . - name: Run Ruff format check run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7128db8a57..c2dc7032bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,8 +2,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.5 hooks: - # - id: ruff-check - # args: [--fix] - # files: ^server/ + - id: ruff-check + args: [--fix] + files: ^server/ - id: ruff-format files: ^server/ diff --git a/README.md b/README.md index c39e83f2a6..42ee655391 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ pre-commit run --all-files python3 manage.py migrate python3 manage.py collectstatic --noinput python3 manage.py createsuperuser # Unless you will only login with your TACC account - python3 manage.py import-apps # Add set of example apps used in Frontera portal (optional) + python3 manage.py import_apps # Add set of example apps used in Frontera portal (optional) #### Initialize the CMS in the `core_portal_cms` container: diff --git a/server/conftest.py b/server/conftest.py index 0e16506e89..c37247ada4 100644 --- a/server/conftest.py +++ b/server/conftest.py @@ -1,12 +1,14 @@ -import pytest -from mock import MagicMock import json import os import tempfile -from portal.apps.auth.models import TapisOAuthToken -from portal.apps.accounts.models import PortalProfile +from unittest.mock import MagicMock + +import pytest from django.conf import settings +from portal.apps.accounts.models import PortalProfile +from portal.apps.auth.models import TapisOAuthToken + @pytest.fixture def mock_tapis_client(mocker): diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 172366a0fe..db6f9cc716 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -1,10 +1,11 @@ -from typing import Optional, Literal -from pydantic import ConfigDict, NonNegativeInt, NonNegativeFloat +from typing import Literal + +from pydantic import ConfigDict, NonNegativeFloat, NonNegativeInt from portal.apps.projects.schema_models.base_metadata import ( + BaseFileMetadata, BaseMetadataModel, BaseProjectMetadata, - BaseFileMetadata, FileObj, ) @@ -20,8 +21,8 @@ class DrpFileMetadata(BaseFileMetadata): extra="forbid", ) - is_advanced_image_file: Optional[bool] = False - image_type: Optional[ + is_advanced_image_file: bool | None = False + image_type: ( Literal[ "8_bit", "16_bit_signed", @@ -38,18 +39,19 @@ class DrpFileMetadata(BaseFileMetadata): "32_bit_abgr", "1_bit_bitmap", ] - ] = None - height: Optional[NonNegativeInt] = None - width: Optional[NonNegativeInt] = None - number_of_images: Optional[NonNegativeInt] = None - offset_to_first_image: Optional[int] = None - gap_between_images: Optional[int] = None - byte_order: Optional[Literal["big_endian", "little_endian"]] = None - use_binary_correction: Optional[bool] = None + | None + ) = None + height: NonNegativeInt | None = None + width: NonNegativeInt | None = None + number_of_images: NonNegativeInt | None = None + offset_to_first_image: int | None = None + gap_between_images: int | None = None + byte_order: Literal["big_endian", "little_endian"] | None = None + use_binary_correction: bool | None = None class DrpProjectMetadata(BaseProjectMetadata): - license: Optional[str] = "ODC-BY 1.0" + license: str | None = "ODC-BY 1.0" class DrpDatasetMetadata(BaseMetadataModel): @@ -60,8 +62,8 @@ class DrpDatasetMetadata(BaseMetadataModel): ) name: str - description: Optional[str] = None - uuid: Optional[str] = None + description: str | None = None + uuid: str | None = None data_type: Literal["sample", "origin_data", "digital_dataset", "analysis_data", "file"] file_objs: list[FileObj] = [] @@ -70,10 +72,18 @@ class DrpSampleMetadata(DrpDatasetMetadata): """Model for DRP Sample Metadata""" porous_media_type: Literal[ - "sandstone", "soil", "carbonate", "granite", "beads", "fibrous_media", "coal", "energy_storage", "other" + "sandstone", + "soil", + "carbonate", + "granite", + "beads", + "fibrous_media", + "coal", + "energy_storage", + "other", ] - porous_media_other_description: Optional[str] = None + porous_media_other_description: str | None = None source: Literal[ "natural", @@ -81,25 +91,25 @@ class DrpSampleMetadata(DrpDatasetMetadata): "artificial", "computer_generated", ] - collection_method: Optional[str] = None - onshore_offshore: Optional[Literal["onshore", "offshore"]] = None - depth: Optional[str] = None - total_vertical_depth: Optional[str] = None - water_depth: Optional[str] = None - geographic_origin: Optional[str] = None - procedure: Optional[str] = None - equipment: Optional[str] = None - algorithm_description: Optional[str] = None - grain_size_min: Optional[NonNegativeFloat] = None - grain_size_max: Optional[NonNegativeFloat] = None - grain_size_avg: Optional[NonNegativeFloat] = None - grain_size_units: Optional[Literal["nanometer", "micrometer", "millimeter", "other"]] = None - porosity: Optional[float] = None - geographical_location: Optional[str] = None - date_of_collection: Optional[str] = None - date_of_creation: Optional[str] = None - identifier: Optional[str] = None - location: Optional[str] = None # TODO_DRP: Remove in new model + collection_method: str | None = None + onshore_offshore: Literal["onshore", "offshore"] | None = None + depth: str | None = None + total_vertical_depth: str | None = None + water_depth: str | None = None + geographic_origin: str | None = None + procedure: str | None = None + equipment: str | None = None + algorithm_description: str | None = None + grain_size_min: NonNegativeFloat | None = None + grain_size_max: NonNegativeFloat | None = None + grain_size_avg: NonNegativeFloat | None = None + grain_size_units: Literal["nanometer", "micrometer", "millimeter", "other"] | None = None + porosity: float | None = None + geographical_location: str | None = None + date_of_collection: str | None = None + date_of_creation: str | None = None + identifier: str | None = None + location: str | None = None # TODO_DRP: Remove in new model class DrpOriginDatasetMetadata(DrpDatasetMetadata): @@ -107,18 +117,18 @@ class DrpOriginDatasetMetadata(DrpDatasetMetadata): is_segmented: Literal["yes", "no"] sample: str - imaging_center: Optional[str] = None - imaging_equipment_and_model: Optional[str] = None - image_format: Optional[str] = None - image_dimensions: Optional[str] = None - image_byte_order: Optional[str] = None - voxel_x: Optional[NonNegativeFloat] = None - voxel_y: Optional[NonNegativeFloat] = None - voxel_z: Optional[NonNegativeFloat] = None - voxel_units: Optional[Literal["nanometer", "micrometer", "millimeter", "other"]] = None - dimensionality: Optional[str] = None - digital_dataset: Optional[str] = None - external_uri: Optional[str] = None # TODO_DRP: Remove in new model + imaging_center: str | None = None + imaging_equipment_and_model: str | None = None + image_format: str | None = None + image_dimensions: str | None = None + image_byte_order: str | None = None + voxel_x: NonNegativeFloat | None = None + voxel_y: NonNegativeFloat | None = None + voxel_z: NonNegativeFloat | None = None + voxel_units: Literal["nanometer", "micrometer", "millimeter", "other"] | None = None + dimensionality: str | None = None + digital_dataset: str | None = None + external_uri: str | None = None # TODO_DRP: Remove in new model class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): @@ -126,10 +136,15 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): is_segmented: Literal["yes", "no"] dataset_type: Literal[ - "machine_learning", "simulation", "geometric_analysis", "experimental", "characterization", "other" + "machine_learning", + "simulation", + "geometric_analysis", + "experimental", + "characterization", + "other", ] - external_uri: Optional[str] = None + external_uri: str | None = None sample: str # base_origin_data: Optional[str] = None - digital_dataset: Optional[str] = None - digital_dataset_other_information: Optional[str] = None + digital_dataset: str | None = None + digital_dataset_other_information: str | None = None diff --git a/server/portal/apps/_custom/drp/schema.py b/server/portal/apps/_custom/drp/schema.py index 9c35377c8a..b4980e3944 100644 --- a/server/portal/apps/_custom/drp/schema.py +++ b/server/portal/apps/_custom/drp/schema.py @@ -1,13 +1,13 @@ """DRP schema extension.""" -from portal.apps.projects.schema_models import constants from portal.apps._custom.drp.models import ( - DrpProjectMetadata, - DrpSampleMetadata, - DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata, DrpFileMetadata, + DrpOriginDatasetMetadata, + DrpProjectMetadata, + DrpSampleMetadata, ) +from portal.apps.projects.schema_models import constants SCHEMA_MAPPING = { constants.PROJECT: DrpProjectMetadata, diff --git a/server/portal/apps/_custom/drp/urls.py b/server/portal/apps/_custom/drp/urls.py index 12a5d66559..d874a0182b 100644 --- a/server/portal/apps/_custom/drp/urls.py +++ b/server/portal/apps/_custom/drp/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps._custom.drp.views import DigitalRocksSampleView, GenerateImagesView app_name = "custom" diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index c932c11186..4f09092f90 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -1,15 +1,17 @@ import json -from portal.views.base import BaseApiView +import logging + +import networkx as nx from django.conf import settings from django.http import HttpRequest, JsonResponse -from portal.exceptions.api import ApiException + from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps.projects.schema_models import constants -import networkx as nx -from portal.apps.projects.workspace_operations.project_meta_operations import patch_file_obj_entity from portal.apps.projects.tasks import process_file from portal.apps.projects.views import get_project_client -import logging +from portal.apps.projects.workspace_operations.project_meta_operations import patch_file_obj_entity +from portal.exceptions.api import ApiException +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) diff --git a/server/portal/apps/accounts/api/urls.py b/server/portal/apps/accounts/api/urls.py index 7ca265b61c..40964edc1b 100644 --- a/server/portal/apps/accounts/api/urls.py +++ b/server/portal/apps/accounts/api/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.accounts.api.views.systems import SystemKeysView app_name = "portal_accounts_api" diff --git a/server/portal/apps/accounts/api/views/systems.py b/server/portal/apps/accounts/api/views/systems.py index b50f4c8bf8..9d2657dad3 100644 --- a/server/portal/apps/accounts/api/views/systems.py +++ b/server/portal/apps/accounts/api/views/systems.py @@ -3,21 +3,23 @@ :synopsis: Account's systems views """ -import logging import json -from django.http import JsonResponse +import logging + from django.contrib.auth.decorators import login_required +from django.http import JsonResponse from django.utils.decorators import method_decorator -from portal.views.base import BaseApiView -from portal.apps.accounts.managers import accounts as AccountsManager from tapipy.errors import BaseTapyException + +from portal.apps.accounts.managers import accounts as AccountsManager +from portal.apps.datafiles.utils import evaluate_datafiles_storage_system from portal.apps.onboarding.steps.system_access_v3 import ( create_system_credentials_with_keys, - create_system_credentials_with_tms, create_system_credentials_with_password, + create_system_credentials_with_tms, ) from portal.utils.encryption import createKeyPair -from portal.apps.datafiles.utils import evaluate_datafiles_storage_system +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) diff --git a/server/portal/apps/accounts/integrations.py b/server/portal/apps/accounts/integrations.py index 89a63efdd9..e5e05fde0e 100644 --- a/server/portal/apps/accounts/integrations.py +++ b/server/portal/apps/accounts/integrations.py @@ -1,7 +1,7 @@ -from importlib import import_module -from django.conf import settings import logging +from importlib import import_module +from django.conf import settings logger = logging.getLogger(__name__) INTEGRATION_APPS = [s["integration"] for s in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if "integration" in s] @@ -12,14 +12,10 @@ def get_integrations(request): for app in INTEGRATION_APPS: try: - mod = import_module("{}.integrations".format(app)) + mod = import_module(f"{app}.integrations") app_integrations += mod.provide_integrations(request) except Exception as exc: - logger.warning( - "Call to module.provide_integrations fail for module: {app_name}. {exc}".format( - app_name=app, exc=str(exc) - ) - ) + logger.warning(f"Call to module.provide_integrations fail for module: {app}. {str(exc)}") return app_integrations diff --git a/server/portal/apps/accounts/managers/abstract.py b/server/portal/apps/accounts/managers/abstract.py index afb9326b4c..c3aac6c96b 100644 --- a/server/portal/apps/accounts/managers/abstract.py +++ b/server/portal/apps/accounts/managers/abstract.py @@ -4,6 +4,7 @@ """ from abc import ABCMeta, abstractmethod + from six import add_metaclass diff --git a/server/portal/apps/accounts/managers/accounts.py b/server/portal/apps/accounts/managers/accounts.py index c1f39af6db..7e68a632e1 100644 --- a/server/portal/apps/accounts/managers/accounts.py +++ b/server/portal/apps/accounts/managers/accounts.py @@ -5,8 +5,10 @@ import logging from importlib import import_module + from django.conf import settings from paramiko.ssh_exception import AuthenticationException, ChannelException, SSHException + from portal.apps.accounts.managers.ssh_keys import KeyCannotBeAdded logger = logging.getLogger(__name__) diff --git a/server/portal/apps/accounts/managers/ssh_keys.py b/server/portal/apps/accounts/managers/ssh_keys.py index e2d11d373d..09ed103ed2 100644 --- a/server/portal/apps/accounts/managers/ssh_keys.py +++ b/server/portal/apps/accounts/managers/ssh_keys.py @@ -4,7 +4,9 @@ """ import logging + import paramiko + from portal.apps.accounts.managers.abstract import AbstractKeysManager # pylint: disable=invalid-name @@ -20,13 +22,13 @@ class KeyCannotBeAdded(Exception): """ def __init__(self, msg, output, error_output, *args, **kwargs): - super(KeyCannotBeAdded, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.msg = msg self.output = output self.error_output = error_output def __str__(self): - return "{msg}: {output} \n {error}".format(msg=self.msg, output=self.output, error=self.error_output) + return f"{self.msg}: {self.output} \n {self.error_output}" class KeysManager(AbstractKeysManager): @@ -84,7 +86,7 @@ def _get_pub_key_comment(self, system_id): :return str: comment """ - comment = "{username}@{system_id}".format(username=self.username, system_id=system_id) + comment = f"{self.username}@{system_id}" return comment def _get_add_pub_key_command(self, system_id, public_key): @@ -101,9 +103,9 @@ def _get_add_pub_key_command(self, system_id, public_key): 'if [ ! -f "~/.ssh/authorized_keys" ]; then ' "mkdir -p ~/.ssh/ && touch ~/.ssh/authorized_keys " "&& chmod 0600 ~/.ssh/authorized_keys; fi && " - 'grep -q -F "{string}" ~/.ssh/authorized_keys || ' - 'echo "{string}" >> ~/.ssh/authorized_keys' - ).format(string=string) + f'grep -q -F "{string}" ~/.ssh/authorized_keys || ' + f'echo "{string}" >> ~/.ssh/authorized_keys' + ) return command def add_public_key(self, system_id, hostname, public_key, port=22, transport=None): # pylint: disable=too-many-arguments, arguments-differ @@ -136,7 +138,7 @@ def add_public_key(self, system_id, hostname, public_key, port=22, transport=Non if status == -1: logger.info("No response from the server") elif status == 0: - logger.info("Public key added successfully to {}".format(hostname)) + logger.info(f"Public key added successfully to {hostname}") elif status > 0: error_lines = "" for line in stderr.readlines(): diff --git a/server/portal/apps/accounts/managers/unit_test.py b/server/portal/apps/accounts/managers/unit_test.py index 86c4e58b72..dcd7386b91 100644 --- a/server/portal/apps/accounts/managers/unit_test.py +++ b/server/portal/apps/accounts/managers/unit_test.py @@ -1,9 +1,10 @@ -from mock import MagicMock -from portal.apps.accounts.managers.accounts import add_pub_key_to_resource -from portal.apps.accounts.managers.ssh_keys import KeysManager -from portal.apps.accounts.managers.ssh_keys import KeyCannotBeAdded -from paramiko.ssh_exception import AuthenticationException, ChannelException, SSHException +from unittest.mock import MagicMock + import pytest +from paramiko.ssh_exception import AuthenticationException, ChannelException, SSHException + +from portal.apps.accounts.managers.accounts import add_pub_key_to_resource +from portal.apps.accounts.managers.ssh_keys import KeyCannotBeAdded, KeysManager @pytest.fixture diff --git a/server/portal/apps/accounts/migrations/0001_initial.py b/server/portal/apps/accounts/migrations/0001_initial.py index 4b0674da93..7ebc27a943 100644 --- a/server/portal/apps/accounts/migrations/0001_initial.py +++ b/server/portal/apps/accounts/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.5 on 2019-10-05 00:11 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py b/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py index 4ae01d3240..9ab6874091 100644 --- a/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py +++ b/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.7 on 2019-11-05 19:50 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/accounts/models.py b/server/portal/apps/accounts/models.py index 50a6305265..1a5d76e0e4 100644 --- a/server/portal/apps/accounts/models.py +++ b/server/portal/apps/accounts/models.py @@ -4,13 +4,14 @@ """ import logging + from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail from django.db import models from django.utils.translation import gettext_lazy as _ -from portal.utils import encryption as EncryptionUtil +from portal.utils import encryption as EncryptionUtil # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -93,12 +94,12 @@ def save_keys( try: Keys.objects.get(ssh_keys__user=user, system=system_id) except ObjectDoesNotExist: - ssh_keys = super(SSHKeysManager, self).create(user=user) + ssh_keys = super().create(user=user) Keys.objects.create(ssh_keys=ssh_keys, system=system_id, private=priv_key, public=pub_key) return ssh_keys raise ValueError( - """A set of keys for system: '{system}' and username: '{username}' - already exists""".format(system=system_id, username=user.username) + f"""A set of keys for system: '{system_id}' and username: '{user.username}' + already exists""" ) def update_keys(self, user, system_id, priv_key, pub_key): @@ -123,15 +124,15 @@ def update_keys(self, user, system_id, priv_key, pub_key): keys = Keys.objects.get(ssh_keys__user=user, system=system_id) except ObjectDoesNotExist: try: - ssh_keys = super(SSHKeysManager, self).get_queryset().get(user=user) + ssh_keys = super().get_queryset().get(user=user) except ObjectDoesNotExist: - ssh_keys = super(SSHKeysManager, self).create(user=user) + ssh_keys = super().create(user=user) keys = Keys.objects.create(ssh_keys=ssh_keys, system=system_id) keys.public = pub_key keys.private = priv_key keys.save() - return super(SSHKeysManager, self).get_queryset().get(user=user) + return super().get_queryset().get(user=user) def save_hostname_keys( self, @@ -159,12 +160,12 @@ def save_hostname_keys( try: HostKeys.objects.get(ssh_keys__user=user, hostname=hostname) except ObjectDoesNotExist: - ssh_keys = super(SSHKeysManager, self).create(user=user) + ssh_keys = super().create(user=user) HostKeys.objects.create(ssh_keys=ssh_keys, hostname=hostname, private=priv_key, public=pub_key) return ssh_keys raise ValueError( - """A set of keys for hostname: '{hostname}' and username: '{username}' - already exists""".format(hostname=hostname, username=user.username) + f"""A set of keys for hostname: '{hostname}' and username: '{user.username}' + already exists""" ) def update_hostname_keys(self, user, hostname, priv_key, pub_key): @@ -189,15 +190,15 @@ def update_hostname_keys(self, user, hostname, priv_key, pub_key): keys = HostKeys.objects.get(ssh_keys__user=user, hostname=hostname) except ObjectDoesNotExist: try: - ssh_keys = super(SSHKeysManager, self).get_queryset().get(user=user) + ssh_keys = super().get_queryset().get(user=user) except ObjectDoesNotExist: - ssh_keys = super(SSHKeysManager, self).create(user=user) + ssh_keys = super().create(user=user) keys = HostKeys.objects.create(ssh_keys=ssh_keys, hostname=hostname) keys.public = pub_key keys.private = priv_key keys.save() - return super(SSHKeysManager, self).get_queryset().get(user=user) + return super().get_queryset().get(user=user) class SSHKeys(models.Model): @@ -290,7 +291,7 @@ class Keys(models.Model): public = models.TextField() def __init__(self, *args, **kwargs): - super(Keys, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._private = self.private def private_key(self): @@ -306,11 +307,11 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ """ if self.private != self._private or self.pk is None: self.private = EncryptionUtil.encrypt(self.private) - super(Keys, self).save(*args, **kwargs) + super().save(*args, **kwargs) self._private = self.private def __str__(self): - return "{username}: {system}".format(username=self.ssh_keys.user.username, system=self.system) + return f"{self.ssh_keys.user.username}: {self.system}" class HostKeys(models.Model): @@ -330,7 +331,7 @@ class Meta: unique_together = (("hostname", "ssh_keys"),) def __init__(self, *args, **kwargs): - super(HostKeys, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._private = self.private def private_key(self): @@ -346,8 +347,8 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ """ if self.private != self._private or self.pk is None: self.private = EncryptionUtil.encrypt(self.private) - super(HostKeys, self).save(*args, **kwargs) + super().save(*args, **kwargs) self._private = self.private def __str__(self): - return "{username}: {host}".format(username=self.ssh_keys.user.username, host=self.hostname) + return f"{self.ssh_keys.user.username}: {self.hostname}" diff --git a/server/portal/apps/accounts/unit_test.py b/server/portal/apps/accounts/unit_test.py index ca5a8e186d..05f36f5637 100644 --- a/server/portal/apps/accounts/unit_test.py +++ b/server/portal/apps/accounts/unit_test.py @@ -1,7 +1,7 @@ import pytest from django.conf import settings -from django.http import HttpResponseRedirect from django.contrib.auth import get_user +from django.http import HttpResponseRedirect def test_account_redirect(client): diff --git a/server/portal/apps/accounts/urls.py b/server/portal/apps/accounts/urls.py index 43648eaf4c..e397a6b557 100644 --- a/server/portal/apps/accounts/urls.py +++ b/server/portal/apps/accounts/urls.py @@ -4,10 +4,9 @@ """ from django.urls import re_path -from portal.apps.accounts.views import LogoutView -from portal.apps.accounts.views import accounts -from portal.apps.accounts import views +from portal.apps.accounts import views +from portal.apps.accounts.views import LogoutView, accounts app_name = "portal_accounts" urlpatterns = [ diff --git a/server/portal/apps/accounts/views.py b/server/portal/apps/accounts/views.py index 374bed0859..9f65600124 100644 --- a/server/portal/apps/accounts/views.py +++ b/server/portal/apps/accounts/views.py @@ -3,17 +3,17 @@ """ import logging -import requests -from django.forms.models import model_to_dict +import requests from django.conf import settings -from django.http import JsonResponse, HttpResponseRedirect +from django.contrib.auth import logout from django.contrib.auth.decorators import login_required from django.contrib.auth.views import LogoutView as DjangoLogoutView -from django.contrib.auth import logout from django.core.exceptions import ObjectDoesNotExist -from django.template.loader import render_to_string +from django.forms.models import model_to_dict +from django.http import HttpResponseRedirect, JsonResponse from django.shortcuts import redirect +from django.template.loader import render_to_string from pytas.http import TASClient from portal.apps.accounts import integrations @@ -49,7 +49,7 @@ def get_user_history(username): Get user history from tas """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/users/{1}/history".format(settings.TAS_URL, username), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/users/{username}/history", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -76,7 +76,7 @@ def get_profile_data(request): demographics = model_to_dict(django_user.profile) except ObjectDoesNotExist as e: demographics = {} - logger.info("exception e:{} {}".format(type(e), e)) + logger.info(f"exception e:{type(e)} {e}") demographics.update(user_profile) context = { "demographics": demographics, diff --git a/server/portal/apps/auth/api/urls.py b/server/portal/apps/auth/api/urls.py index 8d534897f8..adc5ec8fa1 100644 --- a/server/portal/apps/auth/api/urls.py +++ b/server/portal/apps/auth/api/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from portal.apps.auth.api.views import TapisToken +from portal.apps.auth.api.views import TapisToken app_name = "auth_api" urlpatterns = [ diff --git a/server/portal/apps/auth/api/views.py b/server/portal/apps/auth/api/views.py index 9e949e85da..aee5456a39 100644 --- a/server/portal/apps/auth/api/views.py +++ b/server/portal/apps/auth/api/views.py @@ -2,9 +2,11 @@ import logging from hashlib import sha256 -from django.http import JsonResponse + from django.contrib.auth.decorators import login_required +from django.http import JsonResponse from django.utils.decorators import method_decorator + from portal.views.base import BaseApiView logger = logging.getLogger(__name__) diff --git a/server/portal/apps/auth/api/views_unit_test.py b/server/portal/apps/auth/api/views_unit_test.py index c38673ed6d..21170e95c9 100644 --- a/server/portal/apps/auth/api/views_unit_test.py +++ b/server/portal/apps/auth/api/views_unit_test.py @@ -1,9 +1,11 @@ -import pytest import time from hashlib import sha256 -from portal.apps.auth.models import TapisOAuthToken + +import pytest from tapipy.tapis import Tapis +from portal.apps.auth.models import TapisOAuthToken + @pytest.fixture def authenticated_user_and_setup_complete(authenticated_user, mocker): diff --git a/server/portal/apps/auth/backends.py b/server/portal/apps/auth/backends.py index c6a7c59d6e..488faff935 100644 --- a/server/portal/apps/auth/backends.py +++ b/server/portal/apps/auth/backends.py @@ -1,14 +1,15 @@ """Auth backends""" import logging + import requests from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.backends import ModelBackend + from portal.apps.accounts.models import PortalProfile from portal.apps.users.utils import get_user_data - logger = logging.getLogger(__name__) @@ -19,7 +20,7 @@ def authenticate(self, *args, **kwargs): if "backend" in kwargs and kwargs["backend"] == "tapis": token = kwargs["token"] - logger.info('Attempting login via Tapis with token "%s"' % token[:8].ljust(len(token), "-")) + logger.info('Attempting login via Tapis with token "{}"'.format(token[:8].ljust(len(token), "-"))) response = requests.get( f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/userinfo", headers={"X-Tapis-Token": token} @@ -50,10 +51,10 @@ def authenticate(self, *args, **kwargs): user, created = UserModel.objects.update_or_create(username=username, defaults=defaults) if created: - logger.info('Created local user record for "%s" from TAS Profile' % username) + logger.info(f'Created local user record for "{username}" from TAS Profile') PortalProfile.objects.update_or_create(user=user, defaults=profile_defaults) - logger.info('Login successful for user "%s"' % username) + logger.info(f'Login successful for user "{username}"') else: - logger.info("Tapis Authentication failed: %s" % json_result) + logger.info(f"Tapis Authentication failed: {json_result}") return user diff --git a/server/portal/apps/auth/middleware.py b/server/portal/apps/auth/middleware.py index 50dfee561f..d5a8c479e6 100644 --- a/server/portal/apps/auth/middleware.py +++ b/server/portal/apps/auth/middleware.py @@ -3,12 +3,14 @@ """ import logging + from django.contrib.auth import logout from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.http import HttpResponseRedirect from django.urls import reverse from tapipy.errors import BaseTapyException + from portal.apps.auth.models import TapisOAuthToken logger = logging.getLogger(__name__) diff --git a/server/portal/apps/auth/migrations/0001_initial.py b/server/portal/apps/auth/migrations/0001_initial.py index cbcbd8c113..3bb66d9cc9 100644 --- a/server/portal/apps/auth/migrations/0001_initial.py +++ b/server/portal/apps/auth/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.5 on 2019-09-12 07:04 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py b/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py index 069a5a5876..5414918d51 100644 --- a/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py +++ b/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.28 on 2022-09-20 22:39 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/auth/models.py b/server/portal/apps/auth/models.py index 7315bba90b..87d152651b 100644 --- a/server/portal/apps/auth/models.py +++ b/server/portal/apps/auth/models.py @@ -3,8 +3,9 @@ import logging import time from urllib.parse import urlparse -from django.db import models + from django.conf import settings +from django.db import models from tapipy.tapis import Tapis logger = logging.getLogger(__name__) @@ -108,4 +109,7 @@ def refresh_tokens(self): def __str__(self): access_token_masked = self.access_token[-5:] refresh_token_masked = self.refresh_token[-5:] - return f"access_token:{access_token_masked} refresh_token:{refresh_token_masked} expires_in:{self.expires_in} created:{self.created}" + return ( + f"access_token:{access_token_masked} refresh_token:{refresh_token_masked} " + f"expires_in:{self.expires_in} created:{self.created}" + ) diff --git a/server/portal/apps/auth/models_unit_test.py b/server/portal/apps/auth/models_unit_test.py index 6d3b81f2fd..2fe0eed6c2 100644 --- a/server/portal/apps/auth/models_unit_test.py +++ b/server/portal/apps/auth/models_unit_test.py @@ -1,6 +1,8 @@ -import pytest import time + +import pytest from django.conf import settings + from portal.apps.auth.models import TapisOAuthToken pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/auth/unit_test.py b/server/portal/apps/auth/unit_test.py index 8b48f0483c..5b6a6e492c 100644 --- a/server/portal/apps/auth/unit_test.py +++ b/server/portal/apps/auth/unit_test.py @@ -1,10 +1,12 @@ -from django.test import TransactionTestCase, override_settings +from unittest.mock import MagicMock, patch + +import pytest from django.contrib.auth import get_user_model -from mock import patch, MagicMock -from portal.apps.auth.backends import TapisOAuthBackend +from django.test import TransactionTestCase, override_settings from requests import Response + +from portal.apps.auth.backends import TapisOAuthBackend from portal.apps.auth.views import launch_setup_checks -import pytest pytestmark = pytest.mark.django_db @@ -20,7 +22,7 @@ def test_launch_setup_checks(mocker, regular_user, settings): class TestTapisOAuthBackend(TransactionTestCase): def setUp(self): - super(TestTapisOAuthBackend, self).setUp() + super().setUp() self.backend = TapisOAuthBackend() self.mock_response = MagicMock(autospec=Response) self.mock_requests_patcher = patch("portal.apps.auth.backends.requests.get", return_value=self.mock_response) @@ -33,7 +35,7 @@ def setUp(self): self.mock_user_data = self.mock_user_data_patcher.start() def tearDown(self): - super(TestTapisOAuthBackend, self).tearDown() + super().tearDown() self.mock_requests_patcher.stop() self.mock_user_data_patcher.stop() diff --git a/server/portal/apps/auth/urls.py b/server/portal/apps/auth/urls.py index 9583a79b64..76b7232329 100644 --- a/server/portal/apps/auth/urls.py +++ b/server/portal/apps/auth/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.auth import views app_name = "portal_auth" diff --git a/server/portal/apps/auth/views.py b/server/portal/apps/auth/views.py index aa02ca4e12..da3f65600d 100644 --- a/server/portal/apps/auth/views.py +++ b/server/portal/apps/auth/views.py @@ -3,24 +3,27 @@ """ import logging +import math +import secrets import time + import requests -import secrets -import math from django.conf import settings from django.contrib import messages from django.contrib.auth import authenticate, login -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponseBadRequest, JsonResponse +from django.contrib.sessions.models import Session +from django.http import HttpResponseBadRequest, HttpResponseRedirect, JsonResponse from django.shortcuts import render +from django.urls import reverse from django.utils import timezone -from django.contrib.sessions.models import Session -from .models import TapisOAuthToken + from portal.apps.onboarding.execute import execute_setup_steps, new_user_setup_check from portal.apps.users.tasks import index_allocations from portal.apps.users.utils import check_user_groups from portal.utils import get_client_ip +from .models import TapisOAuthToken + logger = logging.getLogger(__name__) METRICS = logging.getLogger(f"metrics.{__name__}") @@ -134,7 +137,7 @@ def tapis_oauth_callback(request): state = request.GET.get("state") if request.session["auth_state"] != state: - msg = "OAuth Authorization State mismatch!? auth_state=%s does not match returned state=%s" % ( + msg = "OAuth Authorization State mismatch!? auth_state={} does not match returned state={}".format( request.session["auth_state"], state, ) @@ -196,7 +199,7 @@ def tapis_oauth_callback(request): else: if "error" in request.GET: error = request.GET["error"] - logger.warning("Authorization failed: %s" % error) + logger.warning(f"Authorization failed: {error}") return HttpResponseRedirect(reverse("portal_accounts:logout")) diff --git a/server/portal/apps/datafiles/handlers/googledrive_handlers.py b/server/portal/apps/datafiles/handlers/googledrive_handlers.py index bb4c3858ca..aa72420920 100644 --- a/server/portal/apps/datafiles/handlers/googledrive_handlers.py +++ b/server/portal/apps/datafiles/handlers/googledrive_handlers.py @@ -1,7 +1,8 @@ -from portal.libs.googledrive import operations -from django.core.exceptions import PermissionDenied import logging +from django.core.exceptions import PermissionDenied + +from portal.libs.googledrive import operations logger = logging.getLogger(__name__) diff --git a/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py b/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py index 8016bd7ea0..cede194920 100644 --- a/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py +++ b/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py @@ -1,5 +1,6 @@ import pytest from django.core.exceptions import PermissionDenied + from portal.apps.datafiles.handlers.googledrive_handlers import googledrive_get_handler, googledrive_put_handler diff --git a/server/portal/apps/datafiles/handlers/tapis_handlers.py b/server/portal/apps/datafiles/handlers/tapis_handlers.py index 4d2627cce5..5b8bb427ec 100644 --- a/server/portal/apps/datafiles/handlers/tapis_handlers.py +++ b/server/portal/apps/datafiles/handlers/tapis_handlers.py @@ -1,7 +1,8 @@ -from portal.libs.agave import operations -from django.core.exceptions import PermissionDenied import logging +from django.core.exceptions import PermissionDenied + +from portal.libs.agave import operations logger = logging.getLogger(__name__) diff --git a/server/portal/apps/datafiles/models_unit_test.py b/server/portal/apps/datafiles/models_unit_test.py index 9946cd56c2..5b4a164808 100644 --- a/server/portal/apps/datafiles/models_unit_test.py +++ b/server/portal/apps/datafiles/models_unit_test.py @@ -1,6 +1,7 @@ -from portal.apps.datafiles.models import Link import pytest +from portal.apps.datafiles.models import Link + @pytest.mark.django_db def test_link_uuid(): diff --git a/server/portal/apps/datafiles/urls.py b/server/portal/apps/datafiles/urls.py index 68b3c844c1..330a790cd7 100644 --- a/server/portal/apps/datafiles/urls.py +++ b/server/portal/apps/datafiles/urls.py @@ -1,14 +1,14 @@ from django.urls import path + from portal.apps.datafiles.views import ( - TapisFilesView, GoogleDriveFilesView, - TransferFilesView, LinkView, - SystemListingView, SystemDefinitionView, + SystemListingView, + TapisFilesView, + TransferFilesView, ) - app_name = "users" urlpatterns = [ path("systems/list/", SystemListingView.as_view()), diff --git a/server/portal/apps/datafiles/utils.py b/server/portal/apps/datafiles/utils.py index e29dec10cf..678eaf26ea 100644 --- a/server/portal/apps/datafiles/utils.py +++ b/server/portal/apps/datafiles/utils.py @@ -1,11 +1,13 @@ import logging +from typing import NotRequired, TypedDict + from django.conf import settings -from typing import TypedDict, NotRequired, Optional -from tapipy.errors import InternalServerError, BaseTapyException +from tapipy.errors import BaseTapyException, InternalServerError from tapipy.tapis import TapisResult + +from portal.apps.auth.models import TapisOAuthToken from portal.apps.notifications.models import Notification from portal.apps.users.utils import get_user_data -from portal.apps.auth.models import TapisOAuthToken logger = logging.getLogger(__name__) @@ -19,7 +21,7 @@ class PortalDataFilesSystem(TypedDict): api: str homeDir: str hostEval: NotRequired[str] - icon: NotRequired[Optional[str]] + icon: NotRequired[str | None] siteSearchPriority: NotRequired[int] resourceProvider: NotRequired[str] readOnly: NotRequired[bool] @@ -47,7 +49,8 @@ def evaluate_datafiles_storage_system( Args: tapis (TapisOAuthToken): Tapis OAuth token object system (PortalDataFilesSystem): Storage system definition - default_host_eval (str, optional): Default environment variable name to evaluate for homeDir if hostEval is not provided. + default_host_eval (str, optional): Default environment variable name to + evaluate for homeDir if hostEval is not provided. Returns: PortalDataFilesSystem: Evaluated storage system definition @@ -101,7 +104,8 @@ def evaluate_datafiles_storage_systems(tapis: TapisOAuthToken, systems: list, de Args: tapis (TapisOAuthToken): Tapis OAuth token object systems (list): List of storage system definitions - default_host_eval (str, optional): Default environment variable name to evaluate for homeDir if hostEval is not provided. + default_host_eval (str, optional): Default environment variable name to + evaluate for homeDir if hostEval is not provided. Returns: list: List of evaluated storage system definitions diff --git a/server/portal/apps/datafiles/views.py b/server/portal/apps/datafiles/views.py index 76109f16ed..184b83294b 100644 --- a/server/portal/apps/datafiles/views.py +++ b/server/portal/apps/datafiles/views.py @@ -1,28 +1,31 @@ import json import logging from hashlib import sha256 -from portal.apps.users.utils import get_allocations + +import dateutil.parser from django.conf import settings -from django.http import JsonResponse, HttpResponseForbidden +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied +from django.http import HttpResponseForbidden, JsonResponse +from django.utils.decorators import method_decorator from requests.exceptions import HTTPError from tapipy.errors import InternalServerError, UnauthorizedError -from portal.views.base import BaseApiView -from portal.utils import check_group_membership, get_client_ip -from portal.libs.agave.utils import service_account -from portal.apps.datafiles.handlers.tapis_handlers import tapis_get_handler, tapis_put_handler, tapis_post_handler + from portal.apps.datafiles.handlers.googledrive_handlers import googledrive_get_handler, googledrive_put_handler -from portal.libs.transfer.operations import transfer, transfer_folder -from portal.libs.agave.serializers import BaseTapisResultSerializer -from portal.exceptions.api import ApiException +from portal.apps.datafiles.handlers.tapis_handlers import tapis_get_handler, tapis_post_handler, tapis_put_handler from portal.apps.datafiles.models import Link -from django.contrib.auth.decorators import login_required -from django.core.exceptions import PermissionDenied -from django.utils.decorators import method_decorator +from portal.apps.datafiles.utils import evaluate_datafiles_storage_systems, get_user_storage_systems +from portal.apps.users.utils import get_allocations from portal.apps.workspace.api.utils import push_keys_required_if_not_credentials_ensured -from .utils import notify, NOTIFY_ACTIONS -import dateutil.parser +from portal.exceptions.api import ApiException +from portal.libs.agave.serializers import BaseTapisResultSerializer +from portal.libs.agave.utils import service_account +from portal.libs.transfer.operations import transfer, transfer_folder +from portal.utils import check_group_membership, get_client_ip from portal.utils.decorators import retry -from portal.apps.datafiles.utils import evaluate_datafiles_storage_systems, get_user_storage_systems +from portal.views.base import BaseApiView + +from .utils import NOTIFY_ACTIONS, notify logger = logging.getLogger(__name__) METRICS = logging.getLogger(f"metrics.{__name__}") diff --git a/server/portal/apps/datafiles/views_unit_test.py b/server/portal/apps/datafiles/views_unit_test.py index f2603c414a..de2c65d5ca 100644 --- a/server/portal/apps/datafiles/views_unit_test.py +++ b/server/portal/apps/datafiles/views_unit_test.py @@ -1,11 +1,11 @@ import json import logging import os +from unittest.mock import MagicMock, patch import pytest -from django.contrib.auth.models import Group from django.conf import settings -from mock import MagicMock, patch +from django.contrib.auth.models import Group from tapipy.errors import InternalServerError, UnauthorizedError from tapipy.tapis import TapisResult @@ -365,16 +365,12 @@ def test_tapis_file_view_preview_supported_non_text_files( mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) response = client.put( - "/api/datafiles/tapis/preview/private/frontera.home.username/test_text.{}/".format(EXTENSION), + f"/api/datafiles/tapis/preview/private/frontera.home.username/test_text.{EXTENSION}/", content_type="application/json", data={"href": "https//tapis.example/href"}, ) - href = ( - POSTIT_HREF - if TYPE != "ms-office" - else "https://view.officeapps.live.com/op/view.aspx?src={}".format(POSTIT_HREF) - ) + href = POSTIT_HREF if TYPE != "ms-office" else f"https://view.officeapps.live.com/op/view.aspx?src={POSTIT_HREF}" assert response.status_code == 200 assert response.json() == {"data": {"href": href, "fileType": TYPE, "content": None, "error": None}} diff --git a/server/portal/apps/forms/urls.py b/server/portal/apps/forms/urls.py index fec0a2b950..0caa626d96 100644 --- a/server/portal/apps/forms/urls.py +++ b/server/portal/apps/forms/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.forms.views import FormsView app_name = "workbench" diff --git a/server/portal/apps/forms/views.py b/server/portal/apps/forms/views.py index 5c28fb3e37..9249bc7281 100644 --- a/server/portal/apps/forms/views.py +++ b/server/portal/apps/forms/views.py @@ -1,7 +1,8 @@ -from portal.views.base import BaseApiView from django.conf import settings from django.http import JsonResponse +from portal.views.base import BaseApiView + class FormsView(BaseApiView): def get(self, request): diff --git a/server/portal/apps/googledrive_integration/integrations.py b/server/portal/apps/googledrive_integration/integrations.py index 9e125170ba..db647dd20a 100644 --- a/server/portal/apps/googledrive_integration/integrations.py +++ b/server/portal/apps/googledrive_integration/integrations.py @@ -1,8 +1,9 @@ from django.conf import settings -from portal.apps.googledrive_integration.models import GoogleDriveUserToken from django.core.cache import cache from django.urls import reverse +from portal.apps.googledrive_integration.models import GoogleDriveUserToken + def provide_integrations(request): activated = False @@ -12,14 +13,14 @@ def provide_integrations(request): request.user.googledrive_user_token activated = True except GoogleDriveUserToken.DoesNotExist: - if cache.get("{0}_googledrive_error".format(request.session.session_key), False): - error = cache.get("{0}_googledrive_error".format(request.session.session_key)) + if cache.get(f"{request.session.session_key}_googledrive_error", False): + error = cache.get(f"{request.session.session_key}_googledrive_error") pass integration = ( { "label": "Google Drive", - "description": "Access files from your Google Drive account in {}.".format(settings.PORTAL_NAMESPACE), + "description": f"Access files from your Google Drive account in {settings.PORTAL_NAMESPACE}.", "activated": activated, "error": error, "disconnect": reverse("googledrive_integration:disconnect"), diff --git a/server/portal/apps/googledrive_integration/migrations/0001_initial.py b/server/portal/apps/googledrive_integration/migrations/0001_initial.py index 63817bea10..ef69f4d256 100644 --- a/server/portal/apps/googledrive_integration/migrations/0001_initial.py +++ b/server/portal/apps/googledrive_integration/migrations/0001_initial.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.10.5 on 2019-04-18 20:39 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + import portal.apps.googledrive_integration.models diff --git a/server/portal/apps/googledrive_integration/models.py b/server/portal/apps/googledrive_integration/models.py index dbbe6927d3..590b3dd27a 100644 --- a/server/portal/apps/googledrive_integration/models.py +++ b/server/portal/apps/googledrive_integration/models.py @@ -1,15 +1,14 @@ -from google.auth.transport.requests import Request -from googleapiclient import discovery -from django.conf import settings -from django.db import models -import google.oauth2.credentials -import logging - import base64 +import logging import pickle -from django.utils import encoding +import google.oauth2.credentials import jsonpickle +from django.conf import settings +from django.db import models +from django.utils import encoding +from google.auth.transport.requests import Request +from googleapiclient import discovery logger = logging.getLogger(__name__) @@ -20,7 +19,7 @@ class CredentialsField(models.Field): def __init__(self, *args, **kwargs): if "null" not in kwargs: kwargs["null"] = True - super(CredentialsField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_internal_type(self): return "BinaryField" diff --git a/server/portal/apps/googledrive_integration/tasks.py b/server/portal/apps/googledrive_integration/tasks.py index 2f26d65238..f359401006 100644 --- a/server/portal/apps/googledrive_integration/tasks.py +++ b/server/portal/apps/googledrive_integration/tasks.py @@ -1,6 +1,7 @@ -from django.contrib.auth import get_user_model import logging +from django.contrib.auth import get_user_model + logger = logging.getLogger(__name__) diff --git a/server/portal/apps/googledrive_integration/unit_test.py b/server/portal/apps/googledrive_integration/unit_test.py index ff61dbdca2..38dbf6bd23 100644 --- a/server/portal/apps/googledrive_integration/unit_test.py +++ b/server/portal/apps/googledrive_integration/unit_test.py @@ -5,12 +5,13 @@ """ # from django.core.urlresolvers import reverse -from portal.apps.googledrive_integration.models import GoogleDriveUserToken -from mock import MagicMock -from google.oauth2.credentials import Credentials +import logging +from unittest.mock import MagicMock + import pytest +from google.oauth2.credentials import Credentials -import logging +from portal.apps.googledrive_integration.models import GoogleDriveUserToken logger = logging.getLogger("portal.apps.googledrive_integration.views") diff --git a/server/portal/apps/googledrive_integration/urls.py b/server/portal/apps/googledrive_integration/urls.py index 24c66e0400..2cf386d366 100644 --- a/server/portal/apps/googledrive_integration/urls.py +++ b/server/portal/apps/googledrive_integration/urls.py @@ -1,6 +1,7 @@ -from portal.apps.googledrive_integration import views from django.urls import path +from portal.apps.googledrive_integration import views + app_name = "googledrive_integration" urlpatterns = [ path("", views.IndexView.as_view(), name="privacy"), diff --git a/server/portal/apps/googledrive_integration/views.py b/server/portal/apps/googledrive_integration/views.py index 065edcf218..4b4611b8f4 100644 --- a/server/portal/apps/googledrive_integration/views.py +++ b/server/portal/apps/googledrive_integration/views.py @@ -1,15 +1,16 @@ +import logging + import google_auth_oauthlib.flow import requests from django.conf import settings from django.contrib.auth.decorators import login_required -from django.urls import reverse +from django.core.cache import cache from django.http import HttpResponseRedirect +from django.urls import reverse from django.views.decorators.csrf import csrf_exempt from django.views.generic.base import TemplateView -from portal.apps.googledrive_integration.models import GoogleDriveUserToken -from django.core.cache import cache -import logging +from portal.apps.googledrive_integration.models import GoogleDriveUserToken logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ class IndexView(TemplateView): template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context["setup_complete"] = ( False if self.request.user.is_anonymous else self.request.user.profile.setup_complete ) @@ -30,7 +31,7 @@ def get_context_data(self, **kwargs): return context def dispatch(self, request, *args, **kwargs): - return super(IndexView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_client_config(): @@ -73,12 +74,12 @@ def oauth2_callback(request): googledrive = request.session["googledrive"] else: logger.error("Could not retrieve googledrive from session") - cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) + cache.set(f"{request.session.session_key}_googledrive_error", error, error_timeout) return HttpResponseRedirect("/accounts/profile") if not (state == googledrive["state"]): logger.error("Could not retrieve state from googledrive stored var") - cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) + cache.set(f"{request.session.session_key}_googledrive_error", error, error_timeout) return HttpResponseRedirect("/accounts/profile") try: @@ -90,10 +91,10 @@ def oauth2_callback(request): ], state=state, ) - flow.redirect_uri = "https://{}{}".format(request.get_host(), redirect_uri) + flow.redirect_uri = f"https://{request.get_host()}{redirect_uri}" # Use the authorization server's response to fetch the OAuth 2.0 tokens. - authorization_response = "https://{}{}".format(request.get_host(), request.get_full_path()) + authorization_response = f"https://{request.get_host()}{request.get_full_path()}" flow.fetch_token(authorization_response=authorization_response) @@ -115,8 +116,8 @@ def oauth2_callback(request): GoogleDriveUserToken.objects.update_or_create(user=request.user, defaults={"credentials": credentials}) except Exception as e: - logger.exception("Unable to complete Google Drive integration setup: %s" % e) - cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) + logger.exception(f"Unable to complete Google Drive integration setup: {e}") + cache.set(f"{request.session.session_key}_googledrive_error", error, error_timeout) return HttpResponseRedirect("/accounts/profile") @@ -142,7 +143,7 @@ def disconnect(request): else: logger.error("Disconnect Google Drive; google drive account revoke error.", extra={"user": request.user}) - logger.debug("status code:{}".format(status_code)) + logger.debug(f"status code:{status_code}") return HttpResponseRedirect("/accounts/profile") @@ -151,6 +152,6 @@ def disconnect(request): except Exception as e: logger.error("Disconnect Google Drive; GoogleDriveUserToken delete error.", extra={"user": request.user}) - logger.exception("google drive delete error: {}".format(e)) + logger.exception(f"google drive delete error: {e}") return HttpResponseRedirect("/accounts/profile") diff --git a/server/portal/apps/jupyter_mounts/api/urls.py b/server/portal/apps/jupyter_mounts/api/urls.py index c5a82d7c5d..cf6cbfc0d3 100644 --- a/server/portal/apps/jupyter_mounts/api/urls.py +++ b/server/portal/apps/jupyter_mounts/api/urls.py @@ -1,6 +1,7 @@ """Jupyter Mounts API Urls""" from django.urls import path + from portal.apps.jupyter_mounts.api import views app_name = "jupyter_mounts" diff --git a/server/portal/apps/jupyter_mounts/api/views.py b/server/portal/apps/jupyter_mounts/api/views.py index ca96d56ff5..d6dfd04dee 100644 --- a/server/portal/apps/jupyter_mounts/api/views.py +++ b/server/portal/apps/jupyter_mounts/api/views.py @@ -1,15 +1,15 @@ -from portal.utils.decorators import agave_jwt_login -from django.contrib.auth.decorators import login_required +import logging + from django.conf import settings -from django.utils.decorators import method_decorator +from django.contrib.auth.decorators import login_required from django.http import JsonResponse +from django.utils.decorators import method_decorator + from portal.apps.auth.models import TapisOAuthToken -from portal.views.base import BaseApiView -from portal.apps.projects.workspace_operations.shared_workspace_operations import list_projects, get_workspace_role from portal.apps.datafiles.utils import evaluate_datafiles_storage_systems - -import logging - +from portal.apps.projects.workspace_operations.shared_workspace_operations import get_workspace_role, list_projects +from portal.utils.decorators import agave_jwt_login +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def getDatafilesStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: } ) except Exception: - logger.exception("Could not retrieve system {}".format(system)) + logger.exception(f"Could not retrieve system {system}") return result def getLocalStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: @@ -63,7 +63,7 @@ def getLocalStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: } ) except Exception: - logger.exception("Could not retrieve system {}".format(system)) + logger.exception(f"Could not retrieve system {system}") return result def getProjectSystems(self, tapis_oauth: TapisOAuthToken) -> list: @@ -88,9 +88,7 @@ def getProjectSystems(self, tapis_oauth: TapisOAuthToken) -> list: result.append( { "path": project["path"], - "mountPath": "/{namespace}/My Projects/{name}".format( - namespace=settings.PORTAL_NAMESPACE, name=name - ), + "mountPath": f"/{settings.PORTAL_NAMESPACE}/My Projects/{name}", "pems": permissions, } ) diff --git a/server/portal/apps/jupyter_mounts/api/views_unit_test.py b/server/portal/apps/jupyter_mounts/api/views_unit_test.py index 4f73531c28..ba6d8126d3 100644 --- a/server/portal/apps/jupyter_mounts/api/views_unit_test.py +++ b/server/portal/apps/jupyter_mounts/api/views_unit_test.py @@ -1,6 +1,7 @@ -import pytest import json import os + +import pytest from django.conf import settings diff --git a/server/portal/apps/licenses/admin.py b/server/portal/apps/licenses/admin.py index b59fce4450..82c6495fdb 100644 --- a/server/portal/apps/licenses/admin.py +++ b/server/portal/apps/licenses/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from . import models diff --git a/server/portal/apps/licenses/migrations/0001_initial.py b/server/portal/apps/licenses/migrations/0001_initial.py index 662cc19ffc..c6aaa0ee8c 100644 --- a/server/portal/apps/licenses/migrations/0001_initial.py +++ b/server/portal/apps/licenses/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.7 on 2019-11-05 19:50 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py b/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py index 4f49f09144..ecaf955b21 100644 --- a/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py +++ b/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.10 on 2024-03-19 16:38 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/licenses/models.py b/server/portal/apps/licenses/models.py index 5a2f69fceb..d5c740f718 100644 --- a/server/portal/apps/licenses/models.py +++ b/server/portal/apps/licenses/models.py @@ -1,6 +1,7 @@ +import logging + from django.conf import settings from django.db import models -import logging logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ class Meta: abstract = True def __str__(self): - return "%s: %s" % (self.license_type, self.user.username) + return f"{self.license_type}: {self.user.username}" def license_as_str(self): self.license_file_content = self.license_file_content.replace("\r\n", "\n") diff --git a/server/portal/apps/news/api/views.py b/server/portal/apps/news/api/views.py index 6d61a4fc04..68647043b1 100644 --- a/server/portal/apps/news/api/views.py +++ b/server/portal/apps/news/api/views.py @@ -1,11 +1,11 @@ from html import unescape +import requests from django.conf import settings from django.contrib.auth.decorators import login_required from django.http import JsonResponse from django.utils.decorators import method_decorator from django.utils.html import strip_tags -import requests from portal.exceptions.api import ApiException from portal.views.base import BaseApiView @@ -28,7 +28,7 @@ def get(self, request, *args, **kwargs): def _get_user_news(self): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/announcements".format(settings.TAS_URL), auth=auth) + r = requests.get(f"{settings.TAS_URL}/announcements", auth=auth) resp = r.json() diff --git a/server/portal/apps/news/views.py b/server/portal/apps/news/views.py index c77a78c249..a3630975d5 100644 --- a/server/portal/apps/news/views.py +++ b/server/portal/apps/news/views.py @@ -1,7 +1,7 @@ -from django.views.generic.base import TemplateView from django.conf import settings from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator +from django.views.generic.base import TemplateView @method_decorator(login_required, name="dispatch") @@ -13,7 +13,7 @@ class IndexView(TemplateView): template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context["setup_complete"] = self.request.user.profile.setup_complete context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/notifications/consumers.py b/server/portal/apps/notifications/consumers.py index 8f9362202d..0da7636972 100644 --- a/server/portal/apps/notifications/consumers.py +++ b/server/portal/apps/notifications/consumers.py @@ -1,6 +1,6 @@ import logging -from channels.generic.websocket import AsyncJsonWebsocketConsumer +from channels.generic.websocket import AsyncJsonWebsocketConsumer logger = logging.getLogger(__name__) diff --git a/server/portal/apps/notifications/consumers_unit_test.py b/server/portal/apps/notifications/consumers_unit_test.py index 6b0439802a..0c918a7641 100644 --- a/server/portal/apps/notifications/consumers_unit_test.py +++ b/server/portal/apps/notifications/consumers_unit_test.py @@ -1,6 +1,7 @@ import pytest -from channels.testing import WebsocketCommunicator from channels.layers import get_channel_layer +from channels.testing import WebsocketCommunicator + from .consumers import NotificationsConsumer diff --git a/server/portal/apps/notifications/migrations/0001_initial.py b/server/portal/apps/notifications/migrations/0001_initial.py index 65a02db552..e8ae293594 100644 --- a/server/portal/apps/notifications/migrations/0001_initial.py +++ b/server/portal/apps/notifications/migrations/0001_initial.py @@ -1,6 +1,7 @@ # Generated by Django 2.2.7 on 2019-12-05 22:37 import datetime + from django.db import migrations, models diff --git a/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py b/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py index 5375fe8003..9f4afb3442 100644 --- a/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py +++ b/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py @@ -1,7 +1,7 @@ # Generated by Django 2.2.7 on 2020-02-18 21:15 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/notifications/models.py b/server/portal/apps/notifications/models.py index c324cc3e07..44535df249 100644 --- a/server/portal/apps/notifications/models.py +++ b/server/portal/apps/notifications/models.py @@ -1,9 +1,10 @@ -from django.db import models -from django.core.serializers.json import DjangoJSONEncoder -from django.utils import timezone -import logging import json +import logging + import six +from django.core.serializers.json import DjangoJSONEncoder +from django.db import models +from django.utils import timezone logger = logging.getLogger(__name__) @@ -67,7 +68,7 @@ def save(self, *args, **kwargs): logger.debug("Keys with error: %s . Value: %s", key, self.extra[key]) raise - super(BaseNotify, self).save(*args, **kwargs) + super().save(*args, **kwargs) @property def extra_content(self): @@ -92,7 +93,7 @@ def mark_deleted(self): self.save() def to_dict(self): - event_data = super(Notification, self).to_dict() + event_data = super().to_dict() event_data.update({"user": self.user, "read": self.read, "deleted": self.deleted}) return event_data @@ -101,6 +102,6 @@ class Broadcast(BaseNotify): group = models.CharField(max_length=20) def to_dict(self): - event_data = super(Broadcast, self).to_dict() + event_data = super().to_dict() event_data.update({"group": self.group}) return event_data diff --git a/server/portal/apps/notifications/unit_test.py b/server/portal/apps/notifications/unit_test.py index e6d249f91c..cdc8861a41 100644 --- a/server/portal/apps/notifications/unit_test.py +++ b/server/portal/apps/notifications/unit_test.py @@ -1,14 +1,13 @@ -from django.test import TestCase -from django.test import Client -from django.contrib.auth import get_user_model -from django.urls import reverse -from unittest import skip import json +import logging import os +from unittest import skip -from .models import Notification +from django.contrib.auth import get_user_model +from django.test import Client, TestCase +from django.urls import reverse -import logging +from .models import Notification logger = logging.getLogger(__name__) diff --git a/server/portal/apps/notifications/urls.py b/server/portal/apps/notifications/urls.py index 34d09b56f3..4b06e912e3 100644 --- a/server/portal/apps/notifications/urls.py +++ b/server/portal/apps/notifications/urls.py @@ -1,6 +1,6 @@ from django.urls import re_path -from portal.apps.notifications.views import ManageNotificationsView +from portal.apps.notifications.views import ManageNotificationsView app_name = "notifications" urlpatterns = [ diff --git a/server/portal/apps/notifications/views.py b/server/portal/apps/notifications/views.py index 2b77309b62..ecf0bf0a33 100644 --- a/server/portal/apps/notifications/views.py +++ b/server/portal/apps/notifications/views.py @@ -1,11 +1,11 @@ +import json import logging + from django.http import JsonResponse -from portal.apps.notifications.models import Notification +from portal.apps.notifications.models import Notification from portal.views.base import BaseApiView -import json - logger = logging.getLogger(__name__) diff --git a/server/portal/apps/onboarding/api/urls.py b/server/portal/apps/onboarding/api/urls.py index fa7a7940ff..4f2c21b4c2 100644 --- a/server/portal/apps/onboarding/api/urls.py +++ b/server/portal/apps/onboarding/api/urls.py @@ -4,8 +4,8 @@ """ from django.urls import path -from portal.apps.onboarding.api import views +from portal.apps.onboarding.api import views app_name = "portal_onboarding_api" urlpatterns = [ diff --git a/server/portal/apps/onboarding/api/views.py b/server/portal/apps/onboarding/api/views.py index b00c4638ac..a52948cbce 100644 --- a/server/portal/apps/onboarding/api/views.py +++ b/server/portal/apps/onboarding/api/views.py @@ -1,21 +1,23 @@ +import json import logging -from portal.views.base import BaseApiView + +from django.conf import settings +from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth import get_user_model -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.http import ( Http404, - JsonResponse, HttpResponseBadRequest, + JsonResponse, ) -from django.contrib.auth.decorators import login_required -from django.contrib.admin.views.decorators import staff_member_required from django.utils.decorators import method_decorator -from django.conf import settings + +from portal.apps.onboarding.execute import execute_setup_steps, execute_single_step, load_setup_step, log_setup_state from portal.apps.onboarding.models import SetupEvent, SetupEventEncoder -from portal.apps.onboarding.execute import log_setup_state, load_setup_step, execute_single_step, execute_setup_steps from portal.apps.onboarding.state import SetupState from portal.apps.users.utils import q_to_model_queries -import json +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -155,9 +157,7 @@ def complete(self, request, setup_step): if not request.user.is_staff: raise PermissionDenied setup_step.state = SetupState.COMPLETED - setup_step.log( - "{step} marked complete by {staff}".format(step=setup_step.display_name(), staff=request.user.username) - ) + setup_step.log(f"{setup_step.display_name()} marked complete by {request.user.username}") def reset(self, request, setup_step): """ @@ -165,16 +165,14 @@ def reset(self, request, setup_step): """ if not request.user.is_staff: raise PermissionDenied - setup_step.log("{step} reset by {staff}".format(step=setup_step.display_name(), staff=request.user.username)) + setup_step.log(f"{setup_step.display_name()} reset by {request.user.username}") # Mark the user's setup_complete as False setup_step.user.profile.setup_complete = False setup_step.user.profile.save() log_setup_state( setup_step.user, - "{user} setup marked incomplete, due to reset of {step}".format( - user=setup_step.user.username, step=setup_step.step_name() - ), + f"{setup_step.user.username} setup marked incomplete, due to reset of {setup_step.step_name()}", ) setup_step.prepare() @@ -182,11 +180,7 @@ def client_action(self, request, setup_step, action, data): """ Call client_action on a setup step """ - setup_step.log( - "{action} action on {step} by {username}".format( - action=action, step=setup_step.step_name(), username=request.user.username - ) - ) + setup_step.log(f"{action} action on {setup_step.step_name()} by {request.user.username}") setup_step.client_action(action, data, request) def post(self, request, username): diff --git a/server/portal/apps/onboarding/api/views_unit_test.py b/server/portal/apps/onboarding/api/views_unit_test.py index b3cb0d381b..03a614c7a4 100644 --- a/server/portal/apps/onboarding/api/views_unit_test.py +++ b/server/portal/apps/onboarding/api/views_unit_test.py @@ -1,11 +1,13 @@ -from mock import MagicMock -from django.http import JsonResponse import json +import logging +from unittest.mock import MagicMock + +import pytest +from django.http import JsonResponse + +from portal.apps.onboarding.api.views import SetupStepView, get_user_onboarding from portal.apps.onboarding.models import SetupEvent from portal.apps.onboarding.state import SetupState -from portal.apps.onboarding.api.views import SetupStepView, get_user_onboarding -import pytest -import logging logger = logging.getLogger(__name__) @@ -28,24 +30,24 @@ def mocked_log_setup_state(mocker): def test_get_user(client, authenticated_user): - response = client.get("/api/onboarding/user/{}/".format(authenticated_user.username)) + response = client.get(f"/api/onboarding/user/{authenticated_user.username}/") assert response.status_code == 200 result = json.loads(response.content) assert result["username"] == "username" def test_get_user_unauthenticated_forbidden(client, regular_user): - response = client.get("/api/onboarding/user/{}/".format(regular_user.username)) + response = client.get(f"/api/onboarding/user/{regular_user.username}/") assert response.status_code == 302 def test_get_other_user_forbidden(client, authenticated_user, regular_user2): - response = client.get("/api/onboarding/user/{}/".format(regular_user2.username)) + response = client.get(f"/api/onboarding/user/{regular_user2.username}/") assert response.status_code == 403 def test_get_user_as_staff(client, authenticated_staff, regular_user): - response = client.get("/api/onboarding/user/{}/".format(regular_user.username)) + response = client.get(f"/api/onboarding/user/{regular_user.username}/") assert response.status_code == 200 result = json.loads(response.content) assert result["username"] == regular_user.username @@ -68,7 +70,7 @@ def test_get_non_existent_user_as_staff(client, authenticated_staff): def test_get_user_as_user(client, settings, authenticated_user, mock_steps): # A user should be able to retrieve their own setup event info - response = client.get("/api/onboarding/user/{}".format(authenticated_user.username), follow=True) + response = client.get(f"/api/onboarding/user/{authenticated_user.username}", follow=True) result = response.json() result = json.loads(response.content) @@ -82,7 +84,7 @@ def test_get_user_as_user(client, settings, authenticated_user, mock_steps): def test_retry_step(client, settings, authenticated_user, mock_retry_step, mocker): mock_execute_single_step = mocker.patch("portal.apps.onboarding.api.views.execute_single_step") - response = client.get("/api/onboarding/user/{}".format(authenticated_user.username), follow=True) + response = client.get(f"/api/onboarding/user/{authenticated_user.username}", follow=True) mock_execute_single_step.apply_async.assert_called_with( args=[authenticated_user.username, "portal.apps.onboarding.steps.test_steps.MockStep"], countdown=2 ) @@ -96,14 +98,14 @@ def test_retry_step(client, settings, authenticated_user, mock_retry_step, mocke def test_incomplete_post(client, authenticated_user): # post should return HttpResponseBadRequest (400) if fields are missing response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user), + f"/api/onboarding/user/{authenticated_user}/", content_type="application/json", data=json.dumps({"action": "user_confirm"}), ) assert response.status_code == 400 response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user), + f"/api/onboarding/user/{authenticated_user}/", content_type="application/json", data=json.dumps({"step": "setupstep"}), ) @@ -123,7 +125,7 @@ def test_client_action(regular_user, rf): def test_reset_not_staff(client, authenticated_user): response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user.username), + f"/api/onboarding/user/{authenticated_user.username}/", content_type="application/json", data=json.dumps({"action": "reset", "step": "portal.apps.onboarding.steps.test_steps.MockStep"}), ) @@ -149,13 +151,13 @@ def test_reset(rf, staff_user, regular_user, mocked_log_setup_state): def test_complete_not_staff(client, authenticated_user, regular_user2): - response = client.post("/api/onboarding/user/{}/".format(regular_user2)) + response = client.post(f"/api/onboarding/user/{regular_user2}/") assert response.status_code == 403 def test_complete(client, authenticated_staff, regular_user, mock_steps, mocked_executor): response = client.post( - "/api/onboarding/user/{}/".format(regular_user.username), + f"/api/onboarding/user/{regular_user.username}/", content_type="application/json", data=json.dumps({"action": "complete", "step": "portal.apps.onboarding.steps.test_steps.MockStep"}), ) diff --git a/server/portal/apps/onboarding/conftest.py b/server/portal/apps/onboarding/conftest.py index 51f7d1a9bf..876500a733 100644 --- a/server/portal/apps/onboarding/conftest.py +++ b/server/portal/apps/onboarding/conftest.py @@ -1,4 +1,5 @@ import pytest + from portal.apps.onboarding.models import SetupEvent from portal.apps.onboarding.state import SetupState diff --git a/server/portal/apps/onboarding/execute.py b/server/portal/apps/onboarding/execute.py index eb244111dc..2b30216bbd 100644 --- a/server/portal/apps/onboarding/execute.py +++ b/server/portal/apps/onboarding/execute.py @@ -1,12 +1,14 @@ -from inspect import isclass +import logging from importlib import import_module +from inspect import isclass + +from celery import shared_task from django.conf import settings -from portal.apps.onboarding.state import SetupState + +from portal.apps.accounts.models import PortalProfile from portal.apps.onboarding.models import SetupEvent +from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.steps.abstract import AbstractStep -from celery import shared_task -from portal.apps.accounts.models import PortalProfile -import logging logger = logging.getLogger(__name__) @@ -18,18 +20,18 @@ class StepExecuteException(Exception): """ def __init__(self, message): - super(StepExecuteException, self).__init__(message) + super().__init__(message) def new_user_setup_check(user): extra_steps = getattr(settings, "PORTAL_USER_ACCOUNT_SETUP_STEPS", []) if len(extra_steps) == 0: - logger.info("No extra setup steps for user {username}".format(username=user.username)) + logger.info(f"No extra setup steps for user {user.username}") profile = PortalProfile.objects.get(user=user) profile.setup_complete = True profile.save() else: - logger.info("Preparing onboarding steps for user {username}".format(username=user.username)) + logger.info(f"Preparing onboarding steps for user {user.username}") prepare_setup_steps(user) @@ -50,10 +52,10 @@ def load_setup_step(user, step): module = import_module(module_str) call = getattr(module, callable_str) if not isclass(call): - raise ValueError("Setup step {step} is not a class".format(step=step)) + raise ValueError(f"Setup step {step} is not a class") setup_step = call(user) if not isinstance(setup_step, AbstractStep): - raise ValueError("Setup step {step} is not a subclass of AbstractStep".format(step=step)) + raise ValueError(f"Setup step {step} is not a subclass of AbstractStep") return setup_step @@ -76,7 +78,7 @@ def process_setup_step(setup_step): except Exception as err: logger.exception("Problem processing setup step") setup_step.state = SetupState.ERROR - setup_step.log("Exception: {err}".format(err=str(err))) + setup_step.log(f"Exception: {str(err)}") @shared_task() @@ -103,7 +105,7 @@ def execute_setup_steps(username): # a step failing to reach the COMPLETED state, mark the user as setup_complete user.profile.setup_complete = True user.profile.save() - log_setup_state(user, "{user} setup is now complete".format(user=user.username)) + log_setup_state(user, f"{user.username} setup is now complete") @shared_task() diff --git a/server/portal/apps/onboarding/execute_unit_test.py b/server/portal/apps/onboarding/execute_unit_test.py index 0bb14428f2..9aa1bae971 100644 --- a/server/portal/apps/onboarding/execute_unit_test.py +++ b/server/portal/apps/onboarding/execute_unit_test.py @@ -1,19 +1,20 @@ -from mock import MagicMock -from portal.apps.onboarding.steps.test_steps import MockProcessingCompleteStep +from unittest.mock import MagicMock + +import pytest + from portal.apps.accounts.models import PortalProfile -from portal.apps.onboarding.models import SetupEvent -from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.execute import ( + StepExecuteException, execute_setup_steps, execute_single_step, - prepare_setup_steps, load_setup_step, log_setup_state, new_user_setup_check, - StepExecuteException, + prepare_setup_steps, ) -import pytest - +from portal.apps.onboarding.models import SetupEvent +from portal.apps.onboarding.state import SetupState +from portal.apps.onboarding.steps.test_steps import MockProcessingCompleteStep pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/onboarding/migrations/0001_initial.py b/server/portal/apps/onboarding/migrations/0001_initial.py index b53eb797cc..3e1073374d 100644 --- a/server/portal/apps/onboarding/migrations/0001_initial.py +++ b/server/portal/apps/onboarding/migrations/0001_initial.py @@ -1,9 +1,9 @@ # Generated by Django 2.2.7 on 2019-11-05 19:50 -from django.conf import settings import django.contrib.postgres.fields.jsonb -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py b/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py index c84ed2ae04..ac1c5cbc35 100644 --- a/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py +++ b/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py @@ -1,8 +1,8 @@ # Generated by Django 3.2.19 on 2023-05-04 22:00 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/onboarding/models.py b/server/portal/apps/onboarding/models.py index b0a295594f..7eed644306 100644 --- a/server/portal/apps/onboarding/models.py +++ b/server/portal/apps/onboarding/models.py @@ -3,9 +3,9 @@ :synopsis: Onboarding models """ -from django.db import models from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder +from django.db import models class SetupEvent(models.Model): @@ -33,14 +33,7 @@ class SetupEvent(models.Model): data = models.JSONField(null=True) def __str__(self): - return "{username} {time} {step} ({state}) - {message} ({data})".format( - username=self.user.username, - time=self.time, - step=self.step, - state=self.state, - message=self.message, - data=self.data, - ) + return f"{self.user.username} {self.time} {self.step} ({self.state}) - {self.message} ({self.data})" def to_dict(self): return { @@ -59,4 +52,4 @@ def default(self, obj): # pylint: disable=method-hidden, arguments-differ event = obj return event.to_dict() else: - return super(SetupEventEncoder, self).default(obj) + return super().default(obj) diff --git a/server/portal/apps/onboarding/models_unit_test.py b/server/portal/apps/onboarding/models_unit_test.py index f36849a674..de9258e465 100644 --- a/server/portal/apps/onboarding/models_unit_test.py +++ b/server/portal/apps/onboarding/models_unit_test.py @@ -1,7 +1,7 @@ -from portal.apps.onboarding.state import SetupState -from portal.apps.onboarding.models import SetupEvent import pytest +from portal.apps.onboarding.models import SetupEvent +from portal.apps.onboarding.state import SetupState pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/onboarding/steps/abstract.py b/server/portal/apps/onboarding/steps/abstract.py index 26dbc03ec8..bf0d899cd5 100644 --- a/server/portal/apps/onboarding/steps/abstract.py +++ b/server/portal/apps/onboarding/steps/abstract.py @@ -1,8 +1,10 @@ from abc import ABCMeta, abstractmethod + +from django.conf import settings from six import add_metaclass + from portal.apps.onboarding.models import SetupEvent from portal.apps.onboarding.state import SetupState -from django.conf import settings @add_metaclass(ABCMeta) @@ -70,12 +72,10 @@ def complete(self, message, data=None): self.log(message, data) def __str__(self): - return "<{step} for {username} is {state}>".format( - step=self.step_name(), state=self.state, username=self.user.username - ) + return f"<{self.step_name()} for {self.user.username} is {self.state}>" def step_name(self): - return "{module}.{classname}".format(module=self.__module__, classname=self.__class__.__name__) + return f"{self.__module__}.{self.__class__.__name__}" @abstractmethod def display_name(self): diff --git a/server/portal/apps/onboarding/steps/abstract_unit_test.py b/server/portal/apps/onboarding/steps/abstract_unit_test.py index 31887afe08..d2ba387d14 100644 --- a/server/portal/apps/onboarding/steps/abstract_unit_test.py +++ b/server/portal/apps/onboarding/steps/abstract_unit_test.py @@ -1,8 +1,9 @@ +import pytest +from django.db.models import signals + from portal.apps.onboarding.models import SetupEvent from portal.apps.onboarding.state import SetupState -from django.db.models import signals from portal.apps.onboarding.steps.test_steps import MockStep -import pytest @pytest.fixture(autouse=True) diff --git a/server/portal/apps/onboarding/steps/access.py b/server/portal/apps/onboarding/steps/access.py index 23e6577fef..daab3d01da 100644 --- a/server/portal/apps/onboarding/steps/access.py +++ b/server/portal/apps/onboarding/steps/access.py @@ -7,7 +7,7 @@ def __init__(self, user): """ Call super class constructor """ - super(RequestAccessStep, self).__init__(user) + super().__init__(user) self.user_confirm = "Request Portal Access" self.staff_approve = "Grant Portal Access" self.staff_deny = "Deny Access Request" @@ -20,7 +20,7 @@ def description(self): After sending the request, wait for their approval.""" def prepare(self): - super(RequestAccessStep, self).prepare() + super().prepare() self.state = SetupState.PENDING self.log("Waiting for access check") @@ -43,8 +43,8 @@ def client_action(self, action, data, request): return if action == "staff_approve": - self.complete("Portal access request approved by {user}".format(user=request.user.username)) + self.complete(f"Portal access request approved by {request.user.username}") elif action == "staff_deny": self.deny("Portal access request has not been approved.") else: - self.fail("Invalid client action {action}".format(action=action)) + self.fail(f"Invalid client action {action}") diff --git a/server/portal/apps/onboarding/steps/access_unit_test.py b/server/portal/apps/onboarding/steps/access_unit_test.py index ee9c10a3c2..07999f9375 100644 --- a/server/portal/apps/onboarding/steps/access_unit_test.py +++ b/server/portal/apps/onboarding/steps/access_unit_test.py @@ -1,15 +1,17 @@ -from django.test import TestCase, RequestFactory +from unittest.mock import ANY, patch + +import pytest from django.contrib.auth import get_user_model -from mock import patch, ANY +from django.test import RequestFactory, TestCase + from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.steps.access import RequestAccessStep -import pytest @pytest.mark.django_db(transaction=True) class TestRequestAccessStep(TestCase): def setUp(self): - super(TestRequestAccessStep, self).setUp() + super().setUp() # Create a test user User = get_user_model() @@ -19,7 +21,7 @@ def setUp(self): self.staff.is_staff = True def tearDown(self): - super(TestRequestAccessStep, self).tearDown() + super().tearDown() @patch("portal.apps.onboarding.steps.access.RequestAccessStep.log") def test_prepare(self, mock_log): diff --git a/server/portal/apps/onboarding/steps/allocation.py b/server/portal/apps/onboarding/steps/allocation.py index f3f5d4b379..22d2d8b4be 100644 --- a/server/portal/apps/onboarding/steps/allocation.py +++ b/server/portal/apps/onboarding/steps/allocation.py @@ -1,5 +1,5 @@ -from portal.apps.onboarding.steps.abstract import AbstractStep from portal.apps.onboarding.state import SetupState +from portal.apps.onboarding.steps.abstract import AbstractStep from portal.apps.users.utils import get_allocations @@ -8,13 +8,16 @@ def __init__(self, user): """ Call super class constructor """ - super(AllocationStep, self).__init__(user) + super().__init__(user) def display_name(self): return "Allocations" def description(self): - return """Accessing your allocations. If unsuccessful, verify the PI has added you to the allocations for this project.""" + return ( + "Accessing your allocations. If unsuccessful, verify the PI " + "has added you to the allocations for this project." + ) def prepare(self): self.state = SetupState.PENDING @@ -32,7 +35,7 @@ def process(self): if not allocations.get("active"): self.state = SetupState.FAILED self.log( - """User {0} does not have any allocations""".format(self.user.username), + f"""User {self.user.username} does not have any allocations""", ) else: if "expected_hosts" in self.settings: @@ -44,8 +47,8 @@ def process(self): if missing_hosts: self.state = SetupState.FAILED - self.log("User {0} is missing allocations on: {1}".format(self.user.username, missing_hosts)) + self.log(f"User {self.user.username} is missing allocations on: {missing_hosts}") return - self.log("Expected host allocations found: {0}".format(matched_hosts)) + self.log(f"Expected host allocations found: {matched_hosts}") self.complete("Allocations retrieved", data=allocations) diff --git a/server/portal/apps/onboarding/steps/allocation_unit_test.py b/server/portal/apps/onboarding/steps/allocation_unit_test.py index 80d024764c..497a7c7bdc 100644 --- a/server/portal/apps/onboarding/steps/allocation_unit_test.py +++ b/server/portal/apps/onboarding/steps/allocation_unit_test.py @@ -1,7 +1,9 @@ -from portal.apps.onboarding.steps.allocation import AllocationStep -from django.conf import settings -from mock import ANY +from unittest.mock import ANY + import pytest +from django.conf import settings + +from portal.apps.onboarding.steps.allocation import AllocationStep @pytest.fixture diff --git a/server/portal/apps/onboarding/steps/mfa.py b/server/portal/apps/onboarding/steps/mfa.py index 0800fe89eb..0cef1d701a 100644 --- a/server/portal/apps/onboarding/steps/mfa.py +++ b/server/portal/apps/onboarding/steps/mfa.py @@ -1,13 +1,14 @@ -from portal.apps.onboarding.steps.abstract import AbstractStep -from portal.apps.onboarding.state import SetupState +import requests from django.conf import settings from requests.auth import HTTPBasicAuth -import requests + +from portal.apps.onboarding.state import SetupState +from portal.apps.onboarding.steps.abstract import AbstractStep class MFAStep(AbstractStep): def __init__(self, user): - super(MFAStep, self).__init__(user) + super().__init__(user) self.user_confirm = "Confirm MFA Pairing" def display_name(self): @@ -17,8 +18,9 @@ def description(self): return """First, set up multi-factor authentication on the TACC User Portal using the - TACC Token App, - then confirm the pairing was successful.""" + + TACC Token App + , then confirm the pairing was successful.""" def custom_status(self): if self.state == SetupState.COMPLETED: @@ -52,4 +54,4 @@ def client_action(self, action, data, request): self.prepare() def tas_pairings_url(self): - return "{0}/tup/users/{1}/pairings".format(settings.TAS_URL, self.user.username) + return f"{settings.TAS_URL}/tup/users/{self.user.username}/pairings" diff --git a/server/portal/apps/onboarding/steps/mfa_unit_test.py b/server/portal/apps/onboarding/steps/mfa_unit_test.py index 89f13de521..2f3ed01224 100644 --- a/server/portal/apps/onboarding/steps/mfa_unit_test.py +++ b/server/portal/apps/onboarding/steps/mfa_unit_test.py @@ -1,6 +1,7 @@ -from portal.apps.onboarding.steps.mfa import MFAStep import pytest +from portal.apps.onboarding.steps.mfa import MFAStep + @pytest.fixture def mock_mfa_check(mocker): diff --git a/server/portal/apps/onboarding/steps/project_membership.py b/server/portal/apps/onboarding/steps/project_membership.py index 8aa509476f..5ff57c87dc 100644 --- a/server/portal/apps/onboarding/steps/project_membership.py +++ b/server/portal/apps/onboarding/steps/project_membership.py @@ -1,13 +1,14 @@ import logging + from django.conf import settings -from requests.auth import HTTPBasicAuth from pytas.http import TASClient +from requests.auth import HTTPBasicAuth from rt import Rt -from portal.apps.onboarding.steps.abstract import AbstractStep + from portal.apps.onboarding.state import SetupState +from portal.apps.onboarding.steps.abstract import AbstractStep from portal.apps.users.tasks import index_allocations - logger = logging.getLogger(__name__) @@ -16,7 +17,7 @@ def __init__(self, user): """ Call super class constructor """ - super(ProjectMembershipStep, self).__init__(user) + super().__init__(user) self.project_sql_ids = ( self.settings["project_sql_id"] if isinstance(self.settings["project_sql_id"], list) @@ -72,9 +73,7 @@ def send_project_request(self, request): ticket_text = ticket_text.format( username=self.user.username, project=self.default_project["title"], - onboarding_url=request.build_absolute_uri( - "/workbench/onboarding/setup/{username}".format(username=self.user.username) - ), + onboarding_url=request.build_absolute_uri(f"/workbench/onboarding/setup/{self.user.username}"), ) try: @@ -145,7 +144,7 @@ def deny_project_request(self): ) tracker.edit_ticket(ticket_id, Status="resolved") else: - self.fail("The portal was unable to close RT Ticket {ticket_id}".format(ticket_id=ticket_id)) + self.fail(f"The portal was unable to close RT Ticket {ticket_id}") def close_project_request(self, deny=False): ticket_id = None @@ -168,7 +167,7 @@ def close_project_request(self, deny=False): ) tracker.edit_ticket(ticket_id, Status="resolved") else: - self.fail("The portal was unable to close RT Ticket {ticket_id}".format(ticket_id=ticket_id)) + self.fail(f"The portal was unable to close RT Ticket {ticket_id}") def process(self): if self.is_project_member(): @@ -189,13 +188,13 @@ def client_action(self, action, data, request): try: self.add_to_project() self.close_project_request() - self.complete("Portal access request approved by {user}".format(user=request.user.username)) + self.complete(f"Portal access request approved by {request.user.username}") except Exception as err: - logger.exception(msg="Error during staff_approve on {}".format(self.step_name())) + logger.exception(msg=f"Error during staff_approve on {self.step_name()}") logger.error(err.args) self.fail("An error occurred while trying to add this user to the project") elif action == "staff_deny": self.deny_project_request() self.deny("Portal access request has not been approved.") else: - self.fail("Invalid client action {action}".format(action=action)) + self.fail(f"Invalid client action {action}") diff --git a/server/portal/apps/onboarding/steps/project_membership_unit_test.py b/server/portal/apps/onboarding/steps/project_membership_unit_test.py index 708ecf7b6a..346eb83a10 100644 --- a/server/portal/apps/onboarding/steps/project_membership_unit_test.py +++ b/server/portal/apps/onboarding/steps/project_membership_unit_test.py @@ -1,10 +1,12 @@ -from django.conf import settings -from portal.apps.onboarding.steps.project_membership import ProjectMembershipStep -from portal.apps.onboarding.models import SetupEvent -from mock import MagicMock, ANY -import pytest import json import os +from unittest.mock import ANY, MagicMock + +import pytest +from django.conf import settings + +from portal.apps.onboarding.models import SetupEvent +from portal.apps.onboarding.steps.project_membership import ProjectMembershipStep @pytest.fixture diff --git a/server/portal/apps/onboarding/steps/system_access.py b/server/portal/apps/onboarding/steps/system_access.py index 32ef470d51..8b90b4109d 100644 --- a/server/portal/apps/onboarding/steps/system_access.py +++ b/server/portal/apps/onboarding/steps/system_access.py @@ -1,7 +1,9 @@ -from .project_membership import ProjectMembershipStep +import logging + from portal.apps.onboarding.state import SetupState from portal.apps.users.utils import get_allocations -import logging + +from .project_membership import ProjectMembershipStep logger = logging.getLogger(__name__) @@ -11,7 +13,7 @@ def __init__(self, user): """ Call super class constructor """ - super(SystemAccessStep, self).__init__(user) + super().__init__(user) self.user_confirm = "Request System Access" self.staff_deny = "Deny System Access Request" diff --git a/server/portal/apps/onboarding/steps/system_access_unit_test.py b/server/portal/apps/onboarding/steps/system_access_unit_test.py index a629236b65..0087718e83 100644 --- a/server/portal/apps/onboarding/steps/system_access_unit_test.py +++ b/server/portal/apps/onboarding/steps/system_access_unit_test.py @@ -1,9 +1,11 @@ -from django.conf import settings -from portal.apps.onboarding.steps.system_access import SystemAccessStep -import pytest import json import os +import pytest +from django.conf import settings + +from portal.apps.onboarding.steps.system_access import SystemAccessStep + @pytest.fixture def tas_client(mocker): diff --git a/server/portal/apps/onboarding/steps/system_access_v3.py b/server/portal/apps/onboarding/steps/system_access_v3.py index 6b00fd0097..414a898eb7 100644 --- a/server/portal/apps/onboarding/steps/system_access_v3.py +++ b/server/portal/apps/onboarding/steps/system_access_v3.py @@ -1,8 +1,10 @@ import logging -from portal.apps.onboarding.steps.abstract import AbstractStep + +from tapipy.errors import BaseTapyException + from portal.apps.onboarding.state import SetupState +from portal.apps.onboarding.steps.abstract import AbstractStep from portal.libs.agave.utils import service_account -from tapipy.errors import BaseTapyException from portal.utils.encryption import createKeyPair logger = logging.getLogger(__name__) @@ -86,7 +88,7 @@ def __init__(self, user): """ Call super class constructor """ - super(SystemAccessStepV3, self).__init__(user) + super().__init__(user) def display_name(self): return "System Access" diff --git a/server/portal/apps/onboarding/steps/test_steps.py b/server/portal/apps/onboarding/steps/test_steps.py index ad46c32289..24d3b8ccec 100644 --- a/server/portal/apps/onboarding/steps/test_steps.py +++ b/server/portal/apps/onboarding/steps/test_steps.py @@ -1,4 +1,5 @@ -from mock import MagicMock +from unittest.mock import MagicMock + from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.steps.abstract import AbstractStep @@ -11,7 +12,7 @@ class MockStep(AbstractStep): def __init__(self, user): self.prepare_spy = MagicMock() - super(MockStep, self).__init__(user) + super().__init__(user) def display_name(self): return "Mock Step" @@ -29,7 +30,7 @@ class MockProcessingCompleteStep(AbstractStep): """ def __init__(self, user): - super(MockProcessingCompleteStep, self).__init__(user) + super().__init__(user) self.process_spy = MagicMock() def prepare(self): @@ -53,7 +54,7 @@ class MockProcessingFailStep(AbstractStep): """ def __init__(self, user): - super(MockProcessingFailStep, self).__init__(user) + super().__init__(user) self.process_spy = MagicMock() def prepare(self): @@ -78,7 +79,7 @@ class MockUserStep(AbstractStep): """ def __init__(self, user): - super(MockUserStep, self).__init__(user) + super().__init__(user) self.client_action_spy = MagicMock() def prepare(self): @@ -103,7 +104,7 @@ class MockStaffStep(AbstractStep): """ def __init__(self, user): - super(MockStaffStep, self).__init__(user) + super().__init__(user) self.staff_approve_spy = MagicMock() self.staff_deny_spy = MagicMock() @@ -122,10 +123,10 @@ def client_action(self, action, data, request): return if action == "staff_approve": - self.complete("Approved by {user}".format(user=request.user.username)) + self.complete(f"Approved by {request.user.username}") self.staff_approve_spy(action, data, request) elif action == "staff_deny": - self.fail("Denied by {user}".format(user=request.user.username)) + self.fail(f"Denied by {request.user.username}") self.staff_deny_spy(action, data, request) @@ -135,7 +136,7 @@ class MockErrorStep(AbstractStep): """ def __init__(self, user): - super(MockErrorStep, self).__init__(user) + super().__init__(user) def prepare(self): self.state = SetupState.PENDING diff --git a/server/portal/apps/portal_messages/admin.py b/server/portal/apps/portal_messages/admin.py index 945b443dc9..614c494394 100644 --- a/server/portal/apps/portal_messages/admin.py +++ b/server/portal/apps/portal_messages/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from portal.apps.portal_messages.models import CustomMessageTemplate diff --git a/server/portal/apps/portal_messages/intro_unit_test.py b/server/portal/apps/portal_messages/intro_unit_test.py index d58104aaa4..ec1c861671 100644 --- a/server/portal/apps/portal_messages/intro_unit_test.py +++ b/server/portal/apps/portal_messages/intro_unit_test.py @@ -1,5 +1,6 @@ import pytest -from portal.apps.portal_messages.models import IntroMessages, CustomMessageTemplate, CustomMessages + +from portal.apps.portal_messages.models import CustomMessages, CustomMessageTemplate, IntroMessages @pytest.fixture diff --git a/server/portal/apps/portal_messages/migrations/0001_initial.py b/server/portal/apps/portal_messages/migrations/0001_initial.py index 0124af887f..40ac60822a 100644 --- a/server/portal/apps/portal_messages/migrations/0001_initial.py +++ b/server/portal/apps/portal_messages/migrations/0001_initial.py @@ -1,9 +1,9 @@ # Generated by Django 2.2.17 on 2021-10-28 15:48 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py b/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py index 1d390170c7..5943885a75 100644 --- a/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py +++ b/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.27 on 2022-03-22 15:48 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py b/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py index 2e4e038e98..82ccf9180e 100644 --- a/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py +++ b/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py @@ -1,7 +1,6 @@ from django.conf import settings from django.db import migrations - CPU_DASHBOARD_MESSAGE = "This page allows you to monitor your job status and get help with tickets. " CPU_DATAFILES_MESSAGE = ( diff --git a/server/portal/apps/portal_messages/models.py b/server/portal/apps/portal_messages/models.py index b44496a904..476eb4bd73 100644 --- a/server/portal/apps/portal_messages/models.py +++ b/server/portal/apps/portal_messages/models.py @@ -1,6 +1,7 @@ import logging -from django.db import models + from django.conf import settings +from django.db import models from django.utils import timezone logger = logging.getLogger(__name__) @@ -69,7 +70,7 @@ def to_dict(self): } def __str__(self): - return "%s | %s | %s | %s" % ( + return "{} | {} | {} | {}".format( self.message_type, self.component, ("dismissible" if self.dismissible else "not dismissible"), diff --git a/server/portal/apps/portal_messages/urls.py b/server/portal/apps/portal_messages/urls.py index a9b21bc9e7..a30293c33b 100644 --- a/server/portal/apps/portal_messages/urls.py +++ b/server/portal/apps/portal_messages/urls.py @@ -1,8 +1,8 @@ """Message URLs""" from django.urls import path -from portal.apps.portal_messages import views +from portal.apps.portal_messages import views app_name = "message" urlpatterns = [path("intro/", views.IntroMessagesView.as_view()), path("custom/", views.CustomMessagesView.as_view())] diff --git a/server/portal/apps/portal_messages/views.py b/server/portal/apps/portal_messages/views.py index 16f18ad7b2..349ed44355 100644 --- a/server/portal/apps/portal_messages/views.py +++ b/server/portal/apps/portal_messages/views.py @@ -4,14 +4,15 @@ and generate CustomMessages """ +import json import logging -from portal.views.base import BaseApiView -from django.http import JsonResponse -from portal.apps.portal_messages.models import IntroMessages, CustomMessages, CustomMessageTemplate + from django.contrib.auth.decorators import login_required +from django.http import JsonResponse from django.utils.decorators import method_decorator -import json +from portal.apps.portal_messages.models import CustomMessages, CustomMessageTemplate, IntroMessages +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) diff --git a/server/portal/apps/projects/admin.py b/server/portal/apps/projects/admin.py index 11cfaa16b0..6639e34dcf 100644 --- a/server/portal/apps/projects/admin.py +++ b/server/portal/apps/projects/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from portal.apps.projects.models.base import ProjectId diff --git a/server/portal/apps/projects/conftest.py b/server/portal/apps/projects/conftest.py index 0f12423148..bcefc8237b 100644 --- a/server/portal/apps/projects/conftest.py +++ b/server/portal/apps/projects/conftest.py @@ -1,4 +1,5 @@ import pytest + from portal.apps.projects.models.base import Project diff --git a/server/portal/apps/projects/exceptions.py b/server/portal/apps/projects/exceptions.py index c8f9349d92..258aff1aea 100644 --- a/server/portal/apps/projects/exceptions.py +++ b/server/portal/apps/projects/exceptions.py @@ -5,8 +5,8 @@ """ import logging -from portal.exceptions.api import ApiException +from portal.exceptions.api import ApiException LOGGER = logging.getLogger(__name__) @@ -27,4 +27,4 @@ def __init__(self, message=None, status=None, extra=None, **kwargs): """ msg = "User is not Authorized." sts = 403 - super(NotAuthorizedError, self).__init__(message=message or msg, status=status or sts, extra=extra, **kwargs) + super().__init__(message=message or msg, status=status or sts, extra=extra, **kwargs) diff --git a/server/portal/apps/projects/management/commands/migrate-projects.py b/server/portal/apps/projects/management/commands/migrate-projects.py index 54135d01f2..ace7f7201f 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects.py +++ b/server/portal/apps/projects/management/commands/migrate-projects.py @@ -1,13 +1,14 @@ """Management command.""" +import logging + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from portal.libs.agave.utils import service_account + from portal.apps.projects.models.base import Project from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.search.tasks import index_project -from django.contrib.auth import get_user_model -import logging - +from portal.libs.agave.utils import service_account logger = logging.getLogger(__name__) @@ -40,16 +41,14 @@ def handle(self, *args, **options): ) ) if len(admins) != 1: - raise Exception("Not exactly one admin for {project_id}".format(project_id=meta.project_id)) + raise Exception(f"Not exactly one admin for {meta.project_id}") # Get first role tuple, first item in tuple which is username admin = get_user_model().objects.get(username=admins[0][0]) project.add_pi(admin) - logger.info( - "Set {admin} as PI on {project_id}".format(admin=admin.username, project_id=meta.project_id) - ) + logger.info(f"Set {admin.username} as PI on {meta.project_id}") except Exception as e: - logger.error("Could not migrate {project_id}".format(project_id=meta.project_id)) + logger.error(f"Could not migrate {meta.project_id}") logger.exception(e) index_project.apply_async(args=[meta.project_id]) diff --git a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py index 097dd8bfd8..3aec544cf5 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py +++ b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py @@ -1,9 +1,9 @@ -from portal.apps.projects.models.base import Project -from portal.apps.projects.models.metadata import LegacyProjectMetadata +import pytest from django.core import management from django.db.models import signals -import pytest +from portal.apps.projects.models.base import Project +from portal.apps.projects.models.metadata import LegacyProjectMetadata pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/projects/management/commands/projects_id.py b/server/portal/apps/projects/management/commands/projects_id.py index ae42a8f55f..7c569527f5 100644 --- a/server/portal/apps/projects/management/commands/projects_id.py +++ b/server/portal/apps/projects/management/commands/projects_id.py @@ -1,10 +1,11 @@ """Management command.""" -from django.db import transaction from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand +from django.db import transaction + from portal.apps.projects.models.base import ProjectId -from portal.apps.projects.models.utils import get_latest_project_storage, get_latest_project_directory +from portal.apps.projects.models.utils import get_latest_project_directory, get_latest_project_storage class Command(BaseCommand): @@ -57,8 +58,8 @@ def handle(self, *args, **options): max_project_id = options["max_project_id"] if max_project_id: self.stdout.write( - "NOTE(!!!!): Ignoring project ids >= {} when " - "processing/updating the storage systems and directories".format(max_project_id) + f"NOTE(!!!!): Ignoring project ids >= {max_project_id} when " + "processing/updating the storage systems and directories" ) latest_storage_system_id = get_latest_project_storage(max_project_id=max_project_id) @@ -69,13 +70,13 @@ def handle(self, *args, **options): if latest_project_id == -1: self.stdout.write("There are no project directories.") - self.stdout.write("Latest storage system project id: {}".format(latest_storage_system_id)) - self.stdout.write("Latest directory project id: {}".format(latest_project_id)) + self.stdout.write(f"Latest storage system project id: {latest_storage_system_id}") + self.stdout.write(f"Latest directory project id: {latest_project_id}") try: with transaction.atomic(): model_project_id = ProjectId.objects.select_for_update().latest("last_updated").value - self.stdout.write("Latest project id in ProjectId model: {}".format(model_project_id)) + self.stdout.write(f"Latest project id in ProjectId model: {model_project_id}") except ObjectDoesNotExist: self.stdout.write("Latest project id in ProjectId model: None") @@ -84,5 +85,5 @@ def handle(self, *args, **options): ProjectId.update(options.get("update")) elif options["update_using_max_value_found"]: max_value_found = max(latest_storage_system_id, latest_project_id, 0) - self.stdout.write("Updating to value latest storage system id: {}".format(max_value_found)) + self.stdout.write(f"Updating to value latest storage system id: {max_value_found}") ProjectId.update(max_value_found) diff --git a/server/portal/apps/projects/management/commands/projects_id_unit_test.py b/server/portal/apps/projects/management/commands/projects_id_unit_test.py index 44e9d0da7b..aeae7bea99 100644 --- a/server/portal/apps/projects/management/commands/projects_id_unit_test.py +++ b/server/portal/apps/projects/management/commands/projects_id_unit_test.py @@ -1,8 +1,9 @@ from io import StringIO + import pytest from django.core.management import call_command -from portal.apps.projects.models.utils import get_latest_project_storage, get_latest_project_directory +from portal.apps.projects.models.utils import get_latest_project_directory, get_latest_project_storage pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/projects/managers/base.py b/server/portal/apps/projects/managers/base.py index 092737959c..5c31dbc253 100644 --- a/server/portal/apps/projects/managers/base.py +++ b/server/portal/apps/projects/managers/base.py @@ -5,18 +5,19 @@ """ import logging + from django.conf import settings from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist + +from portal.apps.projects.models import Project, ProjectId +from portal.apps.projects.models.utils import get_latest_project_directory, get_latest_project_storage +from portal.apps.projects.serializers import MetadataJSONSerializer from portal.libs.agave.utils import service_account # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem from portal.libs.elasticsearch.docs.base import IndexedProject -from portal.apps.projects.models import Project, ProjectId -from portal.apps.projects.serializers import MetadataJSONSerializer -from portal.apps.projects.models.utils import get_latest_project_storage, get_latest_project_directory -from django.core.exceptions import ObjectDoesNotExist - # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ # pylint: enable=invalid-name -class ProjectsManager(object): +class ProjectsManager: """Projects Manager.""" meta_serializer_cls = MetadataJSONSerializer @@ -46,7 +47,7 @@ def _add_acls(self, username, project_id, project_root): client = service_account() job = client.jobs.submit( body={ - "name": "{username}-{project_id}-acls".format(username=username, project_id=project_id), + "name": f"{username}-{project_id}-acls", "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, "archive": False, "parameters": { @@ -69,7 +70,7 @@ def _remove_acls(self, username, project_id, project_root): client = service_account() job = client.jobs.submit( body={ - "name": "{username}-{project_id}-acls".format(username=username, project_id=project_id), + "name": f"{username}-{project_id}-acls", "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, "archive": False, "parameters": { @@ -151,29 +152,27 @@ def create(self, title): ProjectId.objects.create(value=max_value_found) prjId = ProjectId.next_id() - project_id = "{prefix}-{prjId}".format(prefix=settings.PORTAL_PROJECTS_ID_PREFIX, prjId=prjId) + project_id = f"{settings.PORTAL_PROJECTS_ID_PREFIX}-{prjId}" try: prj = Project.create(self.user.tapis_oauth.client, title, project_id, self.user) except ValueError: # Tapis StorageSystem or ProjectMetadata with this ProjectID already exists, # try to update to latest project value and recreate - logger.info("Project with id: {} already exists".format(project_id)) + logger.info(f"Project with id: {project_id} already exists") latest_storage_system_id = get_latest_project_storage() latest_project_id = get_latest_project_directory() max_value_found = max(latest_storage_system_id, latest_project_id, 0) - logger.info("Updating ProjectId to latest project dir or storage system id: {}".format(max_value_found)) + logger.info(f"Updating ProjectId to latest project dir or storage system id: {max_value_found}") ProjectId.update(max_value_found) - project_id = "{prefix}-{prj_id}".format( - prefix=settings.PORTAL_PROJECTS_ID_PREFIX, prj_id=ProjectId.next_id() - ) + project_id = f"{settings.PORTAL_PROJECTS_ID_PREFIX}-{ProjectId.next_id()}" prj = Project.create(self.user.tapis_oauth.client, title, project_id, self.user) prj.storage.update_role(self.user.username, "ADMIN") - METRICS.info("user:{} created project: id={}, title:{}".format(self.user.username, project_id, title)) + METRICS.info(f"user:{self.user.username} created project: id={project_id}, title:{title}") return prj diff --git a/server/portal/apps/projects/managers/unit_test.py b/server/portal/apps/projects/managers/unit_test.py index f4abeb3e8c..8b193ff479 100644 --- a/server/portal/apps/projects/managers/unit_test.py +++ b/server/portal/apps/projects/managers/unit_test.py @@ -6,10 +6,12 @@ import logging import os + +import pytest from django.conf import settings + from portal.apps.projects.managers.base import ProjectsManager from portal.apps.projects.models.base import ProjectId -import pytest LOGGER = logging.getLogger(__name__) diff --git a/server/portal/apps/projects/migrations/0001_initial.py b/server/portal/apps/projects/migrations/0001_initial.py index 54172f8d31..205965615f 100644 --- a/server/portal/apps/projects/migrations/0001_initial.py +++ b/server/portal/apps/projects/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.16 on 2020-11-12 20:02 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py b/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py index 6d7c9e212e..265c635d0f 100644 --- a/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py +++ b/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.10 on 2024-03-19 16:38 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py b/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py index 8006a92e2d..1930d89ee9 100644 --- a/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py +++ b/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py @@ -4,9 +4,10 @@ import django.core.validators import django.db.models.deletion import django.utils.timezone -import portal.apps.projects.models.project_metadata from django.conf import settings from django.db import migrations, models + +import portal.apps.projects.models.project_metadata from portal.apps.projects.schema_models import constants diff --git a/server/portal/apps/projects/models/base.py b/server/portal/apps/projects/models/base.py index 754ba957ef..bef36c4e1b 100644 --- a/server/portal/apps/projects/models/base.py +++ b/server/portal/apps/projects/models/base.py @@ -14,21 +14,22 @@ import logging import os -from django.db import models, transaction -from django.core.exceptions import ObjectDoesNotExist + from django.conf import settings from django.contrib.auth import get_user_model -from portal.utils import encryption as EncryptionUtil -from portal.libs.agave.utils import service_account +from django.core.exceptions import ObjectDoesNotExist +from django.db import models, transaction + +from portal.apps.accounts.models import SSHKeys # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem # from portal.libs.agave.serializers import BaseAgaveSystemSerializer from portal.apps.projects import utils as ProjectsUtils -from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.projects.exceptions import NotAuthorizedError -from portal.apps.accounts.models import SSHKeys - +from portal.apps.projects.models.metadata import LegacyProjectMetadata +from portal.libs.agave.utils import service_account +from portal.utils import encryption as EncryptionUtil # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -62,7 +63,7 @@ def set_storage_auth(storage): return storage -class Project(object): +class Project: """Project class.""" metadata_name = settings.PORTAL_PROJECTS_SYSTEM_PREFIX @@ -420,8 +421,8 @@ def change_project_role(self, user, old_role, new_role): old_role = "member" if new_role == "team_member": new_role = "member" - add_new_role = getattr(self, "add_{}".format(new_role)) - remove_old_role = getattr(self, "remove_{}".format(old_role)) + add_new_role = getattr(self, f"add_{new_role}") + remove_old_role = getattr(self, f"remove_{old_role}") remove_old_role(user) add_new_role(user) @@ -478,9 +479,7 @@ def change_storage_system_role(self, user, new_role): def __repr__(self): """Repr.""" - return "Project({project_id}, {metadata}, {storage})".format( - project_id=self.project_id, metadata=self.metadata, storage=self.storage - ) + return f"Project({self.project_id}, {self.metadata}, {self.storage})" def __str__(self): """Str -> self.project_id.""" diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index a7ec0b2ede..a82913b192 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -5,6 +5,7 @@ """ import logging + from django.conf import settings from django.db import models @@ -71,7 +72,7 @@ class AbstractProjectMetadata(models.Model): def __str__(self): """Str -> self.prj_id - self.title.""" - return "{prj_id} - {title}".format(prj_id=self.project_id, title=self.title) + return f"{self.project_id} - {self.title}" class LegacyProjectMetadata(AbstractProjectMetadata): diff --git a/server/portal/apps/projects/models/project_metadata.py b/server/portal/apps/projects/models/project_metadata.py index c3b24851bb..e7c23a4959 100644 --- a/server/portal/apps/projects/models/project_metadata.py +++ b/server/portal/apps/projects/models/project_metadata.py @@ -1,11 +1,13 @@ """Models for representing project metadata""" import uuid -from django.utils import timezone -from django.db import models -from django.core.validators import MinLengthValidator -from django.core.serializers.json import DjangoJSONEncoder + from django.contrib.auth import get_user_model +from django.core.serializers.json import DjangoJSONEncoder +from django.core.validators import MinLengthValidator +from django.db import models +from django.utils import timezone + from portal.apps.projects.schema_models import constants user_model = get_user_model() diff --git a/server/portal/apps/projects/models/unit_test.py b/server/portal/apps/projects/models/unit_test.py index 8555e157ff..9980da91c0 100644 --- a/server/portal/apps/projects/models/unit_test.py +++ b/server/portal/apps/projects/models/unit_test.py @@ -4,14 +4,14 @@ :synopsis: Projects app unit tests. """ -from portal.apps.projects.models.metadata import LegacyProjectMetadata -from portal.apps.projects.models.base import Project -from portal.apps.projects.models.utils import get_latest_project_storage - # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem import pytest +from portal.apps.projects.models.base import Project +from portal.apps.projects.models.metadata import LegacyProjectMetadata +from portal.apps.projects.models.utils import get_latest_project_storage + @pytest.fixture() def agave_client(mocker): diff --git a/server/portal/apps/projects/models/utils.py b/server/portal/apps/projects/models/utils.py index 73866e8c77..cdeac29413 100644 --- a/server/portal/apps/projects/models/utils.py +++ b/server/portal/apps/projects/models/utils.py @@ -1,7 +1,8 @@ from django.conf import settings -from portal.libs.agave.utils import service_account -from portal.libs.agave.operations import iterate_listing + from portal.apps.projects.models.base import Project +from portal.libs.agave.operations import iterate_listing +from portal.libs.agave.utils import service_account def get_latest_project_storage(max_project_id=None): diff --git a/server/portal/apps/projects/schema_models/base_metadata.py b/server/portal/apps/projects/schema_models/base_metadata.py index 92a0a68a09..44de617d16 100644 --- a/server/portal/apps/projects/schema_models/base_metadata.py +++ b/server/portal/apps/projects/schema_models/base_metadata.py @@ -1,7 +1,7 @@ """Portal-agnostic base Pydantic models for the project metadata graph.""" from functools import partial -from typing import Literal, Optional +from typing import Literal from pydantic import BaseModel, ConfigDict from pydantic.alias_generators import to_camel @@ -29,7 +29,7 @@ class BaseFileMetadata(BaseMetadataModel): model_config = ConfigDict(extra="ignore") data_type: Literal["file"] - name: Optional[str] = None + name: str | None = None class FileObj(BaseMetadataModel): @@ -38,12 +38,12 @@ class FileObj(BaseMetadataModel): system: str name: str path: str - legacy_path: Optional[str] = None + legacy_path: str | None = None type: Literal["file", "dir"] - length: Optional[int] = None - last_modified: Optional[str] = None - uuid: Optional[str] = None - value: Optional[dict] = None + length: int | None = None + last_modified: str | None = None + uuid: str | None = None + value: dict | None = None class PartialTrashEntity(BaseMetadataModel): @@ -64,7 +64,7 @@ class ProjectRelatedDatasets(BaseMetadataModel): """A dataset related to a project.""" dataset_title: str - dataset_description: Optional[str] = None + dataset_description: str | None = None dataset_link: str = "" @@ -82,11 +82,11 @@ class ProjectRelatedPublications(BaseMetadataModel): publication_type: Literal["context", "linked_dataset", "cited_by"] publication_title: str publication_link: str - publication_author: Optional[str] = None - publication_doi: Optional[str] = None - publication_date_of_publication: Optional[str] = None - publication_publisher: Optional[str] = None - publication_description: Optional[str] = None + publication_author: str | None = None + publication_doi: str | None = None + publication_date_of_publication: str | None = None + publication_publisher: str | None = None + publication_description: str | None = None class GuestUser(BaseMetadataModel): @@ -108,17 +108,17 @@ class BaseProjectMetadata(BaseMetadataModel): project_id: str title: str description: str = "" - license: Optional[str] = None - doi: Optional[str] = None - institution: Optional[str] = None - keywords: Optional[str | list[str]] = None + license: str | None = None + doi: str | None = None + institution: str | None = None + keywords: str | list[str] | None = None related_datasets: list[ProjectRelatedDatasets] = [] related_software: list[ProjectRelatedSoftware] = [] related_publications: list[ProjectRelatedPublications] = [] - publication_date: Optional[str] = None + publication_date: str | None = None authors: list[dict] = [] file_objs: list[FileObj] = [] - is_review_project: Optional[bool] = None - is_published_project: Optional[bool] = None + is_review_project: bool | None = None + is_published_project: bool | None = None guest_users: list[GuestUser] = [] - cover_image: Optional[str] = None + cover_image: str | None = None diff --git a/server/portal/apps/projects/schema_models/schema.py b/server/portal/apps/projects/schema_models/schema.py index ae93db258b..1a99299a3f 100644 --- a/server/portal/apps/projects/schema_models/schema.py +++ b/server/portal/apps/projects/schema_models/schema.py @@ -7,8 +7,8 @@ from portal.apps.projects.schema_models import constants from portal.apps.projects.schema_models.base_metadata import ( - BaseProjectMetadata, BaseFileMetadata, + BaseProjectMetadata, PartialTrashEntity, ) diff --git a/server/portal/apps/projects/serializers.py b/server/portal/apps/projects/serializers.py index c2fcb8b532..3fc55498f0 100644 --- a/server/portal/apps/projects/serializers.py +++ b/server/portal/apps/projects/serializers.py @@ -4,10 +4,12 @@ :synopsis: Serializer classes for project objects. """ -import logging import datetime import json +import logging + from django.contrib.auth import get_user_model + from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.libs.agave.utils import to_camel_case @@ -61,4 +63,4 @@ def default(self, obj): # pylint: disable=method-hidden, arguments-differ else: ret[attname] = field.value_to_string(obj) return ret - return super(MetadataJSONSerializer, self).default(obj) + return super().default(obj) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index a0e433914a..1dd3b316f7 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -1,20 +1,23 @@ -import os import base64 import logging +import os from pathlib import Path + from celery import shared_task from django.db import transaction -from portal.libs.agave.utils import user_account + +from portal.apps.notifications.models import Notification from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps.projects.schema_models import constants +from portal.apps.projects.schema_models.base_metadata import FileObj +from portal.apps.projects.workspace_operations.graph_operations import get_path_uuid_mapping from portal.apps.projects.workspace_operations.project_meta_operations import ( add_file_associations, create_file_obj, get_file_obj, get_ordered_value, ) -from portal.apps.projects.workspace_operations.graph_operations import get_path_uuid_mapping -from portal.apps.projects.schema_models.base_metadata import FileObj +from portal.libs.agave.utils import user_account from portal.libs.files.file_processing import ( binary_correction, conf_raw, @@ -23,7 +26,6 @@ create_histogram, create_thumbnail, ) -from portal.apps.notifications.models import Notification logger = logging.getLogger(__name__) diff --git a/server/portal/apps/projects/urls.py b/server/portal/apps/projects/urls.py index 6d1c6b6a14..df46cf4a8a 100644 --- a/server/portal/apps/projects/urls.py +++ b/server/portal/apps/projects/urls.py @@ -1,8 +1,9 @@ """Data Depot API Urls""" -from portal.apps.projects import views from django.urls import path +from portal.apps.projects import views + app_name = "projects" urlpatterns = [ path("system//", views.ProjectInstanceApiView.as_view(), name="project_sys"), diff --git a/server/portal/apps/projects/utils.py b/server/portal/apps/projects/utils.py index 2d5132cd77..1f8fa9efe4 100644 --- a/server/portal/apps/projects/utils.py +++ b/server/portal/apps/projects/utils.py @@ -4,9 +4,11 @@ """ import logging + from django.conf import settings + +from portal.libs.agave.operations import delete, mkdir from portal.libs.agave.utils import service_account -from portal.libs.agave.operations import mkdir, delete # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -38,4 +40,4 @@ def project_id_to_system_id(project_id): :param str project_id: Project Id. """ - return "{prefix}.{prj_id}".format(prefix=settings.PORTAL_PROJECTS_SYSTEM_PREFIX, prj_id=project_id) + return f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index f91fa0080d..9f9036f02b 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -6,36 +6,28 @@ import json import logging -from django.http import HttpRequest, JsonResponse from hashlib import sha256 -from django.contrib.auth.decorators import login_required +from pathlib import Path + from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.db import models, transaction +from django.http import HttpRequest, JsonResponse +from django.http.multipartparser import MultiPartParser from django.utils.decorators import method_decorator -from portal.libs.agave.utils import service_account -from portal.utils import check_group_membership, get_client_ip -from portal.utils.decorators import agave_jwt_login -from portal.exceptions.api import ApiException -from portal.views.base import BaseApiView -from portal.apps.projects.managers.base import ProjectsManager -from portal.apps.projects.workspace_operations.shared_workspace_operations import ( - list_projects, - get_project, - create_shared_workspace, - update_project, - get_workspace_role, - change_user_role, - add_user_to_workspace, - remove_user, - transfer_ownership, - increment_workspace_count, -) -from portal.apps.search.tasks import tapis_project_listing_indexer -from portal.libs.elasticsearch.indexes import IndexedProject from elasticsearch_dsl import Q + +from portal.apps.projects.managers.base import ProjectsManager from portal.apps.projects.models.project_metadata import ProjectMetadata -from django.db import transaction +from portal.apps.projects.schema_models import constants from portal.apps.projects.schema_models.schema import SCHEMA_MAPPING -from django.db import models +from portal.apps.projects.tasks import sync_files_without_metadata +from portal.apps.projects.workspace_operations.graph_operations import ( + add_node_to_project, + build_project_tree, + get_node_from_path, + initialize_project_graph, +) from portal.apps.projects.workspace_operations.project_meta_operations import ( create_entity_metadata, create_project_metadata, @@ -45,18 +37,27 @@ patch_file_obj_entity, patch_project_entity, ) -from portal.libs.agave.operations import mkdir -from pathlib import Path -from portal.apps.projects.schema_models import constants -from portal.apps.projects.workspace_operations.graph_operations import ( - add_node_to_project, - initialize_project_graph, - get_node_from_path, - build_project_tree, +from portal.apps.projects.workspace_operations.shared_workspace_operations import ( + add_user_to_workspace, + change_user_role, + create_shared_workspace, + get_project, + get_workspace_role, + increment_workspace_count, + list_projects, + remove_user, + transfer_ownership, + update_project, ) -from portal.apps.projects.tasks import sync_files_without_metadata +from portal.apps.search.tasks import tapis_project_listing_indexer +from portal.exceptions.api import ApiException +from portal.libs.agave.operations import mkdir +from portal.libs.agave.utils import service_account +from portal.libs.elasticsearch.indexes import IndexedProject from portal.libs.files.file_processing import resize_cover_image -from django.http.multipartparser import MultiPartParser +from portal.utils import check_group_membership, get_client_ip +from portal.utils.decorators import agave_jwt_login +from portal.views.base import BaseApiView LOGGER = logging.getLogger(__name__) METRICS = logging.getLogger(f"metrics.{__name__}") diff --git a/server/portal/apps/projects/views_unit_test.py b/server/portal/apps/projects/views_unit_test.py index a90e5b87a9..b322a801dc 100644 --- a/server/portal/apps/projects/views_unit_test.py +++ b/server/portal/apps/projects/views_unit_test.py @@ -1,18 +1,20 @@ -import pytest +import json from hashlib import sha256 +from unittest.mock import MagicMock + +import pytest +from django.conf import settings +from django.contrib.auth.models import Group +from django.test import override_settings +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart +from tapipy.tapis import TapisResult + from portal.apps.projects.managers.base import ProjectsManager from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps.projects.schema_models import constants from portal.apps.projects.views import get_project_client, get_project_for_user from portal.apps.search.tasks import tapis_project_listing_indexer from portal.libs.elasticsearch.indexes import IndexedProject -from mock import MagicMock -import json -from tapipy.tapis import TapisResult -from django.conf import settings -from django.contrib.auth.models import Group -from django.test import override_settings -from django.test.client import encode_multipart, BOUNDARY, MULTIPART_CONTENT -from portal.apps.projects.schema_models import constants @pytest.fixture diff --git a/server/portal/apps/projects/workspace_operations/datacite_operations.py b/server/portal/apps/projects/workspace_operations/datacite_operations.py index 4435d632f9..875f3f24ad 100644 --- a/server/portal/apps/projects/workspace_operations/datacite_operations.py +++ b/server/portal/apps/projects/workspace_operations/datacite_operations.py @@ -1,8 +1,8 @@ import datetime -from typing import Optional import json -import requests + import networkx as nx +import requests from django.conf import settings @@ -119,7 +119,7 @@ def get_datacite_json(pub_graph: nx.DiGraph): return datacite_json -def upsert_datacite_json(datacite_json: dict, doi: Optional[str] = None): +def upsert_datacite_json(datacite_json: dict, doi: str | None = None): """ Create a draft DOI in datacite with the specified metadata. If a DOI is specified, the metadata for that DOI is updated instead. diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index a9a2ac5114..0dc8f13cac 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -1,11 +1,13 @@ -from typing import Any, Dict -import networkx as nx -from django.db import transaction -import uuid import copy +import uuid +from typing import Any + +import networkx as nx from django.conf import settings -from portal.apps.projects.schema_models import constants +from django.db import transaction + from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps.projects.schema_models import constants def _get_next_child_order(graph: nx.DiGraph, parent_node: str) -> int: @@ -89,7 +91,7 @@ def traverse_graph(project_graph, root_node, path_components): return {"id": current_node, **project_graph.nodes[current_node]} -def get_node_from_path(project_id: str, path: str) -> Dict[str, Any]: +def get_node_from_path(project_id: str, path: str) -> dict[str, Any]: """Return the node ID for the parent of a node with the given path.""" graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) @@ -105,7 +107,7 @@ def get_node_from_path(project_id: str, path: str) -> Dict[str, Any]: return node -def get_root_node(project_id: str) -> Dict[str, Any]: +def get_root_node(project_id: str) -> dict[str, Any]: """Return the root node for a project graph.""" graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) project_graph = nx.node_link_graph(graph_model.value) diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index 16e040f217..511c118119 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -1,15 +1,17 @@ import operator import os import uuid -from django.db import models, transaction -from django.conf import settings from pathlib import Path from typing import get_args + import networkx as nx -from portal.apps.projects.schema_models.schema import SCHEMA_MAPPING -from portal.apps.projects.schema_models import constants +from django.conf import settings +from django.db import models, transaction + from portal.apps.projects.models.project_metadata import ProjectMetadata -from portal.apps.projects.schema_models.base_metadata import PartialEntityWithFiles, FileObj +from portal.apps.projects.schema_models import constants +from portal.apps.projects.schema_models.base_metadata import FileObj, PartialEntityWithFiles +from portal.apps.projects.schema_models.schema import SCHEMA_MAPPING from portal.apps.projects.workspace_operations.graph_operations import ( get_node_from_path, get_node_from_uuid, @@ -71,7 +73,8 @@ def get_value(project_id, path): def get_ordered_value(name, value): """ Return the metadata in the order defined in the Pydantic model. - Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be updated to use camelCase. + Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be + updated to use camelCase. """ schema = SCHEMA_MAPPING.get(name) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index f49575a043..f948637658 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -1,25 +1,26 @@ -from typing import Optional -from django.conf import settings -import logging import json +import logging from io import StringIO -from portal.apps.projects.workspace_operations.shared_workspace_operations import remove_user -from portal.apps.projects.models.project_metadata import ProjectMetadata + import networkx as nx from celery import shared_task +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.mail import send_mail +from django.db import transaction + +from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps.projects.schema_models import constants -from portal.libs.agave.utils import user_account, service_account -from portal.apps.publications.models import Publication, PublicationRequest from portal.apps.projects.workspace_operations.datacite_operations import ( get_datacite_json, - upsert_datacite_json, publish_datacite_doi, + upsert_datacite_json, ) -from django.db import transaction from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes +from portal.apps.projects.workspace_operations.shared_workspace_operations import remove_user +from portal.apps.publications.models import Publication, PublicationRequest from portal.apps.search.tasks import index_publication -from django.contrib.auth import get_user_model -from django.core.mail import send_mail +from portal.libs.agave.utils import service_account, user_account logger = logging.getLogger(__name__) @@ -111,7 +112,9 @@ def publication_request_callback( with transaction.atomic(): # Commented out cleanup to prevent breaking admin role functionality # user_client.systems.unShareSystem(systemId=source_system_id, users=[portal_admin_username]) - # user_client.systems.revokeUserPerms(systemId=source_system_id, userName=portal_admin_username, permissions=["READ", "MODIFY", "EXECUTE"]) + # user_client.systems.revokeUserPerms( + # systemId=source_system_id, userName=portal_admin_username, permissions=["READ", "MODIFY", "EXECUTE"] + # ) # user_client.files.deletePermissions(systemId=source_system_id, username=portal_admin_username, path="/") # logger.info(f'Removed service account from workspace {source_workspace_id}') @@ -190,7 +193,7 @@ def archive_publication_files(project_id: str): @shared_task(bind=True, max_retries=3, queue="default") -def publish_project(self, project_id: str, version: Optional[int] = 1): +def publish_project(self, project_id: str, version: int | None = 1): review_system_prefix = settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX published_system_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py index b91dc790e5..36f0f9d40e 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py @@ -4,18 +4,18 @@ import requests from django.conf import settings +from django.core.exceptions import MultipleObjectsReturned +from tapipy.errors import BaseTapyException, NotFoundError + from portal.apps.projects.models import LegacyProjectMetadata from portal.apps.projects.workspace_operations.shared_workspace_operations import ( - create_workspace_system, add_user_to_workspace, + create_workspace_system, ) from portal.libs.agave.utils import service_account - +from portal.settings import settings_custom from portal.settings.settings_secret import _AGAVE_SUPER_TOKEN as v2_token from portal.settings.settings_secret import _AGAVE_TENANT_BASEURL as v2_url -from portal.settings import settings_custom -from django.core.exceptions import MultipleObjectsReturned -from tapipy.errors import NotFoundError, BaseTapyException ROLE_MAP = { "USER": "writer", @@ -26,8 +26,11 @@ def get_role(project_id, username): - system_id = f"{getattr(settings_custom, '_PORTAL_PROJECTS_SYSTEM_PREFIX_V2', settings.PORTAL_PROJECTS_SYSTEM_PREFIX)}.{project_id}" - headers = {"Authorization": "Bearer {}".format(v2_token)} + system_prefix = getattr( + settings_custom, "_PORTAL_PROJECTS_SYSTEM_PREFIX_V2", settings.PORTAL_PROJECTS_SYSTEM_PREFIX + ) + system_id = f"{system_prefix}.{project_id}" + headers = {"Authorization": f"Bearer {v2_token}"} req = requests.get(f"{v2_url}/systems/v2/{system_id}/roles/{username}", headers=headers) if req.status_code != 200: return None diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index c6bcb1023c..80bd9aa984 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -1,15 +1,16 @@ # from portal.utils.encryption import createKeyPair -from portal.libs.agave.utils import service_account -from portal.apps.projects.models.project_metadata import ProjectMetadata -from portal.apps.projects.schema_models import constants -from tapipy.tapis import Tapis +import logging from typing import Literal -from django.db import transaction + from django.conf import settings from django.contrib.auth import get_user_model -from portal.apps.projects.workspace_operations.project_meta_operations import create_project_metadata, get_ordered_value +from django.db import transaction +from tapipy.tapis import Tapis -import logging +from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps.projects.schema_models import constants +from portal.apps.projects.workspace_operations.project_meta_operations import create_project_metadata, get_ordered_value +from portal.libs.agave.utils import service_account logger = logging.getLogger(__name__) diff --git a/server/portal/apps/public_data/urls.py b/server/portal/apps/public_data/urls.py index 96229804d6..15b1941444 100644 --- a/server/portal/apps/public_data/urls.py +++ b/server/portal/apps/public_data/urls.py @@ -4,8 +4,10 @@ """ import re -from django.urls import re_path + from django.conf import settings +from django.urls import re_path + from portal.apps.public_data.views import IndexView app_name = "public_data" diff --git a/server/portal/apps/public_data/views.py b/server/portal/apps/public_data/views.py index 60c651f79a..5e96bdd450 100644 --- a/server/portal/apps/public_data/views.py +++ b/server/portal/apps/public_data/views.py @@ -1,10 +1,12 @@ import json import logging + import networkx as nx -from django.views.generic.base import TemplateView from django.conf import settings -from portal.apps.publications.models import Publication +from django.views.generic.base import TemplateView + from portal.apps.projects.workspace_operations.datacite_operations import get_datacite_json +from portal.apps.publications.models import Publication logger = logging.getLogger(__name__) @@ -40,7 +42,7 @@ class IndexView(TemplateView): template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) project_id = kwargs.get("project_id") if project_id: try: @@ -61,4 +63,4 @@ def get_context_data(self, **kwargs): return context def dispatch(self, request, *args, **kwargs): - return super(IndexView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) diff --git a/server/portal/apps/publications/models.py b/server/portal/apps/publications/models.py index 3c4caf8d56..5b90394894 100644 --- a/server/portal/apps/publications/models.py +++ b/server/portal/apps/publications/models.py @@ -5,11 +5,13 @@ """ import logging + from django.conf import settings +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils import timezone + from portal.apps.projects.models.project_metadata import ProjectMetadata -from django.core.serializers.json import DjangoJSONEncoder # pylint: disable=invalid-name logger = logging.getLogger(__name__) diff --git a/server/portal/apps/publications/urls.py b/server/portal/apps/publications/urls.py index ee71a5bfcf..af861b6cc5 100644 --- a/server/portal/apps/publications/urls.py +++ b/server/portal/apps/publications/urls.py @@ -1,8 +1,9 @@ """Publications API Urls""" -from portal.apps.publications import views from django.urls import path +from portal.apps.publications import views + app_name = "publications_api" urlpatterns = [ path("publication-request/", views.PublicationRequestView.as_view(), name="publication_request"), diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index cf83b194b0..e98e7f2f38 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -6,30 +6,31 @@ import json import logging -from django.contrib.auth.decorators import login_required + from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist +from django.db import models, transaction from django.http import JsonResponse from django.utils.decorators import method_decorator -from portal.exceptions.api import ApiException -from portal.views.base import BaseApiView -from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_workspace +from elasticsearch_dsl import Q + +from portal.apps.notifications.models import Notification +from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps.projects.views import get_project_for_user from portal.apps.projects.workspace_operations.project_publish_operations import ( copy_graph_and_files_for_review_system, publish_project, - update_and_cleanup_review_project, send_publication_rejected_email_to_authors, send_publication_reviewed_email_to_reviewers, + update_and_cleanup_review_project, ) -from django.db import transaction -from portal.apps.notifications.models import Notification +from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_workspace from portal.apps.publications.models import Publication, PublicationRequest -from portal.apps.projects.models.project_metadata import ProjectMetadata -from portal.apps.projects.views import get_project_for_user -from django.db import models -from django.core.exceptions import ObjectDoesNotExist -from django.contrib.auth import get_user_model +from portal.exceptions.api import ApiException from portal.libs.elasticsearch.docs.base import IndexedPublication -from elasticsearch_dsl import Q +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) diff --git a/server/portal/apps/request_access/api/unit_test.py b/server/portal/apps/request_access/api/unit_test.py index 19a3095d0c..87aaba88a1 100644 --- a/server/portal/apps/request_access/api/unit_test.py +++ b/server/portal/apps/request_access/api/unit_test.py @@ -1,6 +1,7 @@ -import pytest import json import os + +import pytest from django.conf import settings diff --git a/server/portal/apps/request_access/api/urls.py b/server/portal/apps/request_access/api/urls.py index 3360ba9855..774adfce04 100644 --- a/server/portal/apps/request_access/api/urls.py +++ b/server/portal/apps/request_access/api/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.request_access.api import views app_name = "request_access_api" diff --git a/server/portal/apps/request_access/api/views.py b/server/portal/apps/request_access/api/views.py index 651f35b0cd..8c12e030f1 100644 --- a/server/portal/apps/request_access/api/views.py +++ b/server/portal/apps/request_access/api/views.py @@ -1,9 +1,11 @@ import logging -from django.http import JsonResponse, HttpResponseBadRequest + from django.conf import settings +from django.http import HttpResponseBadRequest, JsonResponse +from pytas.http import TASClient + from portal.apps.tickets import utils from portal.views.base import BaseApiView -from pytas.http import TASClient logger = logging.getLogger(__name__) @@ -32,7 +34,7 @@ def post(self, request): else: return JsonResponse({"message": "Incorrect password"}, status=401) except Exception as e: - logger.error("Incorrect password for user: {user}. {exc}".format(user=username, exc=e)) + logger.error(f"Incorrect password for user: {username}. {e}") return JsonResponse({"message": "Incorrect password"}, status=401) if email is None or problem_description is None: diff --git a/server/portal/apps/request_access/urls.py b/server/portal/apps/request_access/urls.py index 724dbdf2ca..a6d2a4b54a 100644 --- a/server/portal/apps/request_access/urls.py +++ b/server/portal/apps/request_access/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.request_access.views import IndexView app_name = "request_access" diff --git a/server/portal/apps/request_access/views.py b/server/portal/apps/request_access/views.py index 7001e55b2b..46dfb9336f 100644 --- a/server/portal/apps/request_access/views.py +++ b/server/portal/apps/request_access/views.py @@ -1,6 +1,6 @@ -from django.views.generic.base import TemplateView -from django.shortcuts import redirect from django.conf import settings +from django.shortcuts import redirect +from django.views.generic.base import TemplateView class IndexView(TemplateView): @@ -15,9 +15,9 @@ def dispatch(self, request, *args, **kwargs): if request.user.is_authenticated: return redirect("/workbench/dashboard/") - return super(IndexView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/search/management/commands/reindex-files.py b/server/portal/apps/search/management/commands/reindex_files.py similarity index 94% rename from server/portal/apps/search/management/commands/reindex-files.py rename to server/portal/apps/search/management/commands/reindex_files.py index f421d3f75a..1cb81d5cb5 100644 --- a/server/portal/apps/search/management/commands/reindex-files.py +++ b/server/portal/apps/search/management/commands/reindex_files.py @@ -1,8 +1,9 @@ -from django.core.management import BaseCommand -from django.conf import settings import elasticsearch +from django.conf import settings +from django.core.management import BaseCommand from elasticsearch_dsl import Index from elasticsearch_dsl.connections import connections # noqa: F401 + from portal.libs.elasticsearch.indexes import setup_files_index @@ -11,7 +12,7 @@ class Command(BaseCommand): This command reindexes all documents in the default files index in order to apply new mappings/analyzers. It does NOT crawl Agave for file metadata, it only uses data that already exists in the file index. Usage is as simple as - running `./manage.py reindex-files`. + running `./manage.py reindex_files`. This works by resetting the index aliased as settings.ES_REINDEX_INDEX_ALIAS (applying any new mappings/analyzers defined in the portal.libs.elasticsearch.docs.base.IndexedFile @@ -44,9 +45,8 @@ def handle(self, *args, **options): if not swap_only: confirm = input( - 'This will delete any documents in the index "{}" and recreate the index. Continue? (Y/n) '.format( - reindex_index_alias - ) + f'This will delete any documents in the index "{reindex_index_alias}" and recreate the index. ' + "Continue? (Y/n) " ) if confirm != "Y": self.stdout.write("Aborting reindex.") diff --git a/server/portal/apps/search/management/commands/unit_test.py b/server/portal/apps/search/management/commands/unit_test.py index d4e3597a88..40126e6e21 100644 --- a/server/portal/apps/search/management/commands/unit_test.py +++ b/server/portal/apps/search/management/commands/unit_test.py @@ -1,13 +1,14 @@ -from mock import patch, MagicMock -from django.test import TestCase +from unittest.mock import MagicMock, patch + from django.core.management import call_command +from django.test import TestCase class TestSwapReindex(TestCase): def setUp(self): - self.patch_setup = patch("portal.apps.search.management.commands.reindex-files.setup_files_index") - self.patch_connections = patch("portal.apps.search.management.commands.reindex-files.connections") - self.patch_elasticsearch = patch("portal.apps.search.management.commands.reindex-files.elasticsearch") + self.patch_setup = patch("portal.apps.search.management.commands.reindex_files.setup_files_index") + self.patch_connections = patch("portal.apps.search.management.commands.reindex_files.connections") + self.patch_elasticsearch = patch("portal.apps.search.management.commands.reindex_files.elasticsearch") self.mock_setup = self.patch_setup.start() self.mock_connections = self.patch_connections.start() @@ -17,25 +18,25 @@ def setUp(self): self.addCleanup(self.patch_connections.stop) self.addCleanup(self.patch_elasticsearch.stop) - @patch("portal.apps.search.management.commands.reindex-files.input") + @patch("portal.apps.search.management.commands.reindex_files.input") def test_raises_when_user_does_not_proceed(self, mock_input): mock_input.return_value = "n" with self.assertRaises(SystemExit): - call_command("reindex-files") + call_command("reindex_files") - @patch("portal.apps.search.management.commands.reindex-files.Index") - @patch("portal.apps.search.management.commands.reindex-files.input") + @patch("portal.apps.search.management.commands.reindex_files.Index") + @patch("portal.apps.search.management.commands.reindex_files.input") def test_raises_exception_when_no_index(self, mock_input, mock_index): mock_input.return_value = "Y" mock_index.return_value.get_alias.return_value.keys.side_effect = Exception with self.assertRaises(SystemExit): - call_command("reindex-files") + call_command("reindex_files") - @patch("portal.apps.search.management.commands.reindex-files.Index") - @patch("portal.apps.search.management.commands.reindex-files.input") + @patch("portal.apps.search.management.commands.reindex_files.Index") + @patch("portal.apps.search.management.commands.reindex_files.input") def test_performs_reindex_from_default_to_reindex(self, mock_input, mock_index): mock_input.return_value = "Y" @@ -44,18 +45,18 @@ def test_performs_reindex_from_default_to_reindex(self, mock_input, mock_index): mock_client = MagicMock() self.mock_elasticsearch.Elasticsearch.return_value = mock_client - call_command("reindex-files") + call_command("reindex_files") self.mock_elasticsearch.helpers.reindex.assert_called_with(mock_client, "DEFAULT_NAME", "REINDEX_NAME") - @patch("portal.apps.search.management.commands.reindex-files.Index") - @patch("portal.apps.search.management.commands.reindex-files.input") + @patch("portal.apps.search.management.commands.reindex_files.Index") + @patch("portal.apps.search.management.commands.reindex_files.input") def test_performs_swap_with_correct_args(self, mock_input, mock_index): mock_input.return_value = "Y" mock_index.return_value.get_alias.return_value.keys.side_effect = [["DEFAULT_NAME"], ["REINDEX_NAME"]] - call_command("reindex-files") + call_command("reindex_files") mock_alias = { "actions": [ @@ -67,8 +68,8 @@ def test_performs_swap_with_correct_args(self, mock_input, mock_index): } self.mock_elasticsearch.Elasticsearch().indices.update_aliases.assert_called_with(mock_alias) - @patch("portal.apps.search.management.commands.reindex-files.Index") - @patch("portal.apps.search.management.commands.reindex-files.input") + @patch("portal.apps.search.management.commands.reindex_files.Index") + @patch("portal.apps.search.management.commands.reindex_files.input") def test_cleanup(self, mock_input, mock_index): mock_input.return_value = "Y" @@ -79,6 +80,6 @@ def test_cleanup(self, mock_input, mock_index): ] opts = {"cleanup": True} - call_command("reindex-files", **opts) + call_command("reindex_files", **opts) self.assertEqual(mock_index.return_value.delete.call_count, 1) diff --git a/server/portal/apps/search/tasks.py b/server/portal/apps/search/tasks.py index 4ec1e44756..bb94b86295 100644 --- a/server/portal/apps/search/tasks.py +++ b/server/portal/apps/search/tasks.py @@ -1,11 +1,13 @@ import logging -from django.conf import settings + from celery import shared_task -from portal.libs.agave.utils import user_account, service_account -from portal.libs.elasticsearch.utils import index_listing, index_project_listing +from django.conf import settings +from elasticsearch.exceptions import NotFoundError + from portal.apps.projects.models.metadata import LegacyProjectMetadata +from portal.libs.agave.utils import service_account, user_account from portal.libs.elasticsearch.docs.base import IndexedProject, IndexedPublication -from elasticsearch.exceptions import NotFoundError +from portal.libs.elasticsearch.utils import index_listing, index_project_listing logger = logging.getLogger(__name__) @@ -28,8 +30,8 @@ def tapis_indexer( ): return - from portal.libs.elasticsearch.utils import index_level from portal.libs.agave.utils import walk_levels + from portal.libs.elasticsearch.utils import index_level client = user_account(access_token) if access_token else service_account() @@ -40,7 +42,7 @@ def tapis_indexer( filePath, folders, files = walk_levels(client, systemId, filePath, ignore_hidden=ignore_hidden).__next__() index_level(filePath, folders, files, systemId, reindex=reindex) except Exception as exc: - logger.error("Error walking files under system {} and path {}".format(systemId, filePath)) + logger.error(f"Error walking files under system {systemId} and path {filePath}") raise self.retry(exc=exc) if recurse: diff --git a/server/portal/apps/signals/receivers.py b/server/portal/apps/signals/receivers.py index 383a76a19e..10748393f5 100644 --- a/server/portal/apps/signals/receivers.py +++ b/server/portal/apps/signals/receivers.py @@ -1,15 +1,17 @@ -from django.dispatch import receiver -from portal.apps.signals.signals import portal_event +import copy +import logging + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +from django.contrib.auth import get_user_model from django.db.models.signals import post_save +from django.dispatch import receiver + from portal.apps.notifications.models import Notification from portal.apps.onboarding.models import SetupEvent from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.search.tasks import index_project -from django.contrib.auth import get_user_model -import logging -import copy -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer +from portal.apps.signals.signals import portal_event channel_layer = get_channel_layer() diff --git a/server/portal/apps/site_search/api/unit_test.py b/server/portal/apps/site_search/api/unit_test.py index ab71870ced..0a9c228293 100644 --- a/server/portal/apps/site_search/api/unit_test.py +++ b/server/portal/apps/site_search/api/unit_test.py @@ -1,6 +1,7 @@ -from mock import MagicMock -from tapipy.errors import BaseTapyException +from unittest.mock import MagicMock + import pytest +from tapipy.errors import BaseTapyException @pytest.fixture diff --git a/server/portal/apps/site_search/api/urls.py b/server/portal/apps/site_search/api/urls.py index 843151754c..b7e84a8285 100644 --- a/server/portal/apps/site_search/api/urls.py +++ b/server/portal/apps/site_search/api/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.site_search.api.views import SiteSearchApiView app_name = "site_search" diff --git a/server/portal/apps/site_search/api/views.py b/server/portal/apps/site_search/api/views.py index e2ae182a2d..0f2bf92bc4 100644 --- a/server/portal/apps/site_search/api/views.py +++ b/server/portal/apps/site_search/api/views.py @@ -1,11 +1,13 @@ +import logging + +from django.conf import settings from django.http import JsonResponse from elasticsearch_dsl import Search +from tapipy.errors import BaseTapyException + from portal.libs.agave.operations import search as search_operation -from portal.views.base import BaseApiView -from django.conf import settings from portal.libs.agave.utils import service_account -import logging -from tapipy.errors import BaseTapyException +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -111,6 +113,6 @@ def _handle_tapis_ssh_exception(self, e): # in case of these error types, user is not authenticated # or does not have access do not fail the entire search # request, log the issue. - logger.exception("Error retrieving search results due to TAPIS SSH related error: {}".format(str(e))) + logger.exception(f"Error retrieving search results due to TAPIS SSH related error: {str(e)}") else: raise diff --git a/server/portal/apps/site_search/urls.py b/server/portal/apps/site_search/urls.py index 5a22c7a7de..97cb4c24d7 100644 --- a/server/portal/apps/site_search/urls.py +++ b/server/portal/apps/site_search/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.site_search.views import IndexView app_name = "site_search" diff --git a/server/portal/apps/site_search/views.py b/server/portal/apps/site_search/views.py index 6d37d651be..6e2862a19c 100644 --- a/server/portal/apps/site_search/views.py +++ b/server/portal/apps/site_search/views.py @@ -1,5 +1,5 @@ -from django.views.generic.base import TemplateView from django.conf import settings +from django.views.generic.base import TemplateView class IndexView(TemplateView): @@ -10,10 +10,10 @@ class IndexView(TemplateView): template_name = "portal/apps/workbench/index.html" def dispatch(self, request, *args, **kwargs): - return super(IndexView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context["setup_complete"] = ( False if self.request.user.is_anonymous else self.request.user.profile.setup_complete ) diff --git a/server/portal/apps/system_monitor/unit_test.py b/server/portal/apps/system_monitor/unit_test.py index 1bf320ff4b..9360d318dd 100644 --- a/server/portal/apps/system_monitor/unit_test.py +++ b/server/portal/apps/system_monitor/unit_test.py @@ -1,6 +1,7 @@ -import pytest import json import os + +import pytest from django.conf import settings from django.http import Http404 diff --git a/server/portal/apps/system_monitor/urls.py b/server/portal/apps/system_monitor/urls.py index 9e608820fc..ff4c2f8b1b 100644 --- a/server/portal/apps/system_monitor/urls.py +++ b/server/portal/apps/system_monitor/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.system_monitor import views app_name = "system_monitor" diff --git a/server/portal/apps/system_monitor/views.py b/server/portal/apps/system_monitor/views.py index b708af2619..cbdaf974c8 100644 --- a/server/portal/apps/system_monitor/views.py +++ b/server/portal/apps/system_monitor/views.py @@ -1,10 +1,12 @@ -from django.conf import settings -from portal.views.base import BaseApiView -from django.http import JsonResponse -import requests import json import logging +import requests +from django.conf import settings +from django.http import JsonResponse + +from portal.views.base import BaseApiView + logger = logging.getLogger(__name__) @@ -41,16 +43,14 @@ def get(self, request, system_name=None): systems_json = requests.get(settings.SYSTEM_MONITOR_URL).json() for sys in requested_systems: if sys not in systems_json: - logger.info("System information for {} is missing. Assuming not operational status.".format(sys)) + logger.info(f"System information for {sys} is missing. Assuming not operational status.") systems.append(_get_unoperational_system(sys)) continue try: system = System(systems_json[sys]).to_dict() systems.append(system) except Exception: - logger.exception( - "Problem gather system information for {}: Assuming not operational status".format(sys) - ) + logger.exception(f"Problem gather system information for {sys}: Assuming not operational status") systems.append(_get_unoperational_system(sys)) return JsonResponse(systems, safe=False) diff --git a/server/portal/apps/tickets/api/unit_test.py b/server/portal/apps/tickets/api/unit_test.py index 16972b382d..2db43adff0 100644 --- a/server/portal/apps/tickets/api/unit_test.py +++ b/server/portal/apps/tickets/api/unit_test.py @@ -1,7 +1,8 @@ -import pytest +import io import json import os -import io + +import pytest from django.conf import settings @@ -171,7 +172,7 @@ def test_tickets_get_history(client, authenticated_user, mock_rtutil): assert response.status_code == 200 result = json.loads(response.content) assert len(result["ticket_history"]) == 5 - full_name = "{} {}".format(authenticated_user.first_name, authenticated_user.last_name) + full_name = f"{authenticated_user.first_name} {authenticated_user.last_name}" assert result["ticket_history"][2]["Creator"] == full_name assert result["ticket_history"][3]["Creator"] == "RT System" assert result["ticket_history"][4]["Creator"] == full_name @@ -190,7 +191,7 @@ def test_tickets_get_history_handle_service_accounts( response = client.get("/api/tickets/1/history") assert response.status_code == 200 result = json.loads(response.content) - full_name = "{} {}".format(authenticated_user.first_name, authenticated_user.last_name) + full_name = f"{authenticated_user.first_name} {authenticated_user.last_name}" assert result["ticket_history"][2]["Creator"] == full_name assert result["ticket_history"][3]["Creator"] == "RT System" assert result["ticket_history"][4]["Creator"] == full_name @@ -209,7 +210,7 @@ def test_tickets_post_history_reply_with_text(client, authenticated_user, mock_r mock_rtutil.replyToTicket.assert_called_with( ticket_id=1, files=[], - reply_text="reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username), + reply_text=f"reply text\n[Reply submitted on behalf of {authenticated_user.username}]", ) @@ -222,7 +223,7 @@ def test_tickets_post_history_reply_with_multiline_text( mock_rtutil.replyToTicket.assert_called_with( ticket_id=1, files=[], - reply_text="reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username), + reply_text=f"reply text\n[Reply submitted on behalf of {authenticated_user.username}]", ) @@ -250,4 +251,4 @@ def test_tickets_post_history_reply_with_text_and_attachment( _, kwargs = mock_rtutil.replyToTicket.call_args assert kwargs["ticket_id"] == 1 assert len(kwargs["files"]) == 1 - assert kwargs["reply_text"] == "reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username) + assert kwargs["reply_text"] == f"reply text\n[Reply submitted on behalf of {authenticated_user.username}]" diff --git a/server/portal/apps/tickets/api/urls.py b/server/portal/apps/tickets/api/urls.py index b9678d5e71..6f857772cf 100644 --- a/server/portal/apps/tickets/api/urls.py +++ b/server/portal/apps/tickets/api/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.tickets.api import views app_name = "portal_tickets_api" diff --git a/server/portal/apps/tickets/api/views.py b/server/portal/apps/tickets/api/views.py index 47f48d6288..d92c2c155d 100644 --- a/server/portal/apps/tickets/api/views.py +++ b/server/portal/apps/tickets/api/views.py @@ -1,17 +1,18 @@ import logging import re from functools import wraps -from django.core.files.base import ContentFile -from django.http import JsonResponse, HttpResponseBadRequest, HttpResponse + +from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.utils.decorators import method_decorator from django.core.exceptions import PermissionDenied -from portal.apps.tickets import rtUtil -from portal.apps.tickets import utils -from portal.views.base import BaseApiView +from django.core.files.base import ContentFile +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse +from django.utils.decorators import method_decorator + +from portal.apps.tickets import rtUtil, utils from portal.exceptions.api import ApiException -from django.conf import settings +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -131,7 +132,7 @@ def _get_ticket_history(self, rt, requesting_username, ticket_id): known_user = get_user_model().objects.filter(username=entry["Creator"]).first() if known_user: - entry["Creator"] = "{} {}".format(known_user.first_name, known_user.last_name) + entry["Creator"] = f"{known_user.first_name} {known_user.last_name}" return ticket_history def _get_matching_history_entry(self, ticket_history, content): @@ -150,7 +151,7 @@ def post(self, request, ticket_id): return HttpResponseBadRequest() # Add information on which user submitted this reply (as this is being done by a service account) - modified_reply = reply + "\n[Reply submitted on behalf of {}]".format(request.user.username) + modified_reply = reply + f"\n[Reply submitted on behalf of {request.user.username}]" attachments = [(f.name, ContentFile(f.read()), f.content_type) for f in request.FILES.getlist("attachments")] diff --git a/server/portal/apps/tickets/rtUtil.py b/server/portal/apps/tickets/rtUtil.py index a9d871d4bc..7d47bc4d6f 100644 --- a/server/portal/apps/tickets/rtUtil.py +++ b/server/portal/apps/tickets/rtUtil.py @@ -1,6 +1,7 @@ +from datetime import datetime + import rt from django.conf import settings -from datetime import datetime from requests.auth import HTTPBasicAuth diff --git a/server/portal/apps/tickets/unit_test.py b/server/portal/apps/tickets/unit_test.py index eacd97a0a0..19ff5b6d07 100644 --- a/server/portal/apps/tickets/unit_test.py +++ b/server/portal/apps/tickets/unit_test.py @@ -1,8 +1,8 @@ import pytest -from django.http import HttpResponse -from django.http import HttpRequest -from portal.apps.tickets.utils import get_recaptcha_verification +from django.http import HttpRequest, HttpResponse + from portal.apps.tickets import rtUtil +from portal.apps.tickets.utils import get_recaptcha_verification @pytest.fixture(autouse=True) diff --git a/server/portal/apps/tickets/urls.py b/server/portal/apps/tickets/urls.py index 10812fa59f..273968d352 100644 --- a/server/portal/apps/tickets/urls.py +++ b/server/portal/apps/tickets/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.tickets import views app_name = "tickets" diff --git a/server/portal/apps/tickets/utils.py b/server/portal/apps/tickets/utils.py index e09166da4c..d3500b42e2 100644 --- a/server/portal/apps/tickets/utils.py +++ b/server/portal/apps/tickets/utils.py @@ -1,7 +1,8 @@ -from portal.apps.tickets import rtUtil -from django.http import JsonResponse, HttpResponseBadRequest -from django.conf import settings import requests +from django.conf import settings +from django.http import HttpResponseBadRequest, JsonResponse + +from portal.apps.tickets import rtUtil METADATA_HEADER = "*** Ticket Metadata ***" @@ -12,20 +13,20 @@ def create_ticket(username, first_name, last_name, email, cc, subject, problem_d if subject is None or email is None or problem_description is None: return HttpResponseBadRequest() - metadata = "{}\n\n".format(METADATA_HEADER) - metadata += "Client info:\n{}\n\n".format(info) + metadata = f"{METADATA_HEADER}\n\n" + metadata += f"Client info:\n{info}\n\n" for key in ["HTTP_REFERER", "HTTP_USER_AGENT", "HTTP_HOST"]: metadata += "{}:\n{}\n\n".format(key, meta.get(key, "None")) if username: - metadata += "authenticated_user:\n{}\n\n".format(username) - metadata += "authenticated_user_email:\n{}\n\n".format(email) - metadata += "authenticated_user_first_name:\n{}\n\n".format(first_name) - metadata += "authenticated_user_last_name:\n{}\n\n".format(last_name) + metadata += f"authenticated_user:\n{username}\n\n" + metadata += f"authenticated_user_email:\n{email}\n\n" + metadata += f"authenticated_user_first_name:\n{first_name}\n\n" + metadata += f"authenticated_user_last_name:\n{last_name}\n\n" else: - metadata += "user_first_name:\n{}\n\n".format(first_name) - metadata += "user_last_name:\n{}\n\n".format(last_name) + metadata += f"user_first_name:\n{first_name}\n\n" + metadata += f"user_last_name:\n{last_name}\n\n" problem_description += "\n\n" + metadata diff --git a/server/portal/apps/tickets/views.py b/server/portal/apps/tickets/views.py index 93b6d51e9e..af98ea3001 100644 --- a/server/portal/apps/tickets/views.py +++ b/server/portal/apps/tickets/views.py @@ -1,7 +1,7 @@ -from django.views.decorators.csrf import ensure_csrf_cookie -from django.shortcuts import redirect, render -from django.contrib.auth.decorators import login_required from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.shortcuts import redirect, render +from django.views.decorators.csrf import ensure_csrf_cookie @login_required diff --git a/server/portal/apps/users/tasks.py b/server/portal/apps/users/tasks.py index 1c67807ea4..d8df36abbb 100644 --- a/server/portal/apps/users/tasks.py +++ b/server/portal/apps/users/tasks.py @@ -1,7 +1,9 @@ import json -from django.conf import settings + from celery import shared_task +from django.conf import settings from pytas.http import TASClient + from portal.libs.elasticsearch.docs.base import IndexedAllocation from portal.libs.elasticsearch.utils import get_sha256_hash diff --git a/server/portal/apps/users/unit_test.py b/server/portal/apps/users/unit_test.py index f3a8a72819..0559a305a3 100644 --- a/server/portal/apps/users/unit_test.py +++ b/server/portal/apps/users/unit_test.py @@ -1,16 +1,18 @@ import json import os +from unittest.mock import patch + import pytest -from mock import patch -from django.test import TestCase -from django.contrib.auth import get_user_model from django.conf import settings -from pytas.http import TASClient +from django.contrib.auth import get_user_model +from django.test import TestCase from elasticsearch.exceptions import NotFoundError +from pytas.http import TASClient from zeep.exceptions import Fault + from portal.apps.auth.models import TapisOAuthToken -from portal.apps.users.utils import get_allocations from portal.apps.users.tasks import get_tas_allocations +from portal.apps.users.utils import get_allocations class AttrDict(dict): @@ -60,7 +62,10 @@ def test_auth_view_noauth(self): def test_usage_view(self, mocked_file): # TODO: this is hideous, there must be a better way to write that or # re-write the route to be less disgusting. - mocked_file.search.return_value.filter.return_value.extra.return_value.execute.return_value.to_dict.return_value = { + mocked_search = mocked_file.search.return_value + mocked_filter = mocked_search.filter.return_value + mocked_extra = mocked_filter.extra.return_value + mocked_extra.execute.return_value.to_dict.return_value = { "aggregations": {"total_storage_bytes": {"value": 10}} } self.client.login(username="test", password="test") @@ -76,12 +81,12 @@ def test_usage_view_noauth(self): class TestGetAllocations(TestCase): def setUp(self): - super(TestGetAllocations, self).setUp() + super().setUp() self.mock_tas_patcher = patch("portal.apps.users.tasks.TASClient", spec=TASClient) self.mock_tas = self.mock_tas_patcher.start() def tearDown(self): - super(TestGetAllocations, self).tearDown() + super().tearDown() self.mock_tas_patcher.stop() @patch("portal.apps.users.utils.IndexedAllocation") @@ -218,7 +223,7 @@ def tas_delete_user_error_response(): def test_add_user(client, requests_mock, authenticated_user, tas_add_user_response): - requests_mock.post("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_add_user_response) + requests_mock.post(f"{settings.TAS_URL}/v1/projects/1234/users/5678", json=tas_add_user_response) response = client.post("/api/users/team/manage/1234/5678") assert response.status_code == 200 assert response.json() == {"response": "ok"} @@ -230,13 +235,13 @@ def test_add_user_unauthenticated(client): def test_add_user_failure(client, requests_mock, authenticated_user, tas_add_user_error_response): - requests_mock.post("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_add_user_error_response) + requests_mock.post(f"{settings.TAS_URL}/v1/projects/1234/users/5678", json=tas_add_user_error_response) response = client.post("/api/users/team/manage/1234/5678") assert response.status_code == 400 def test_delete_user(client, requests_mock, authenticated_user, tas_delete_user_response): - requests_mock.delete("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_delete_user_response) + requests_mock.delete(f"{settings.TAS_URL}/v1/projects/1234/users/5678", json=tas_delete_user_response) response = client.delete("/api/users/team/manage/1234/5678") assert response.status_code == 200 assert response.json() == {"response": "ok"} @@ -248,7 +253,7 @@ def test_delete_user_unauthenticated(client): def test_delete_user_failure(client, requests_mock, authenticated_user, tas_delete_user_error_response): - requests_mock.delete("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_delete_user_error_response) + requests_mock.delete(f"{settings.TAS_URL}/v1/projects/1234/users/5678", json=tas_delete_user_error_response) response = client.delete("/api/users/team/manage/1234/5678") assert response.status_code == 400 diff --git a/server/portal/apps/users/urls.py b/server/portal/apps/users/urls.py index 447d956522..d8c0fca3c4 100644 --- a/server/portal/apps/users/urls.py +++ b/server/portal/apps/users/urls.py @@ -1,14 +1,15 @@ from django.urls import path, re_path + from portal.apps.users.views import ( - SearchView, - AuthenticatedView, - UsageView, + AllocationManagementView, AllocationsView, + AllocationUsageView, + AuthenticatedView, + SearchView, + TasUsersView, TeamView, + UsageView, UserDataView, - TasUsersView, - AllocationUsageView, - AllocationManagementView, ) app_name = "users" diff --git a/server/portal/apps/users/utils.py b/server/portal/apps/users/utils.py index d7a236cafb..dd11a1fc3c 100644 --- a/server/portal/apps/users/utils.py +++ b/server/portal/apps/users/utils.py @@ -1,14 +1,16 @@ -from django.db.models import Q +import logging + +import requests from django.conf import settings +from django.db.models import Q +from elasticsearch.exceptions import NotFoundError from pytas.http import TASClient + +from portal.exceptions.api import ApiException from portal.libs.elasticsearch.docs.base import IndexedAllocation -from elasticsearch.exceptions import NotFoundError from portal.libs.elasticsearch.utils import get_sha256_hash -from .tasks import index_allocations, get_tas_allocations -import logging -import requests -from portal.exceptions.api import ApiException +from .tasks import get_tas_allocations, index_allocations logger = logging.getLogger(__name__) @@ -57,7 +59,7 @@ def get_allocations(username, force=False): """ try: if force: - logger.info("Forcing TAS allocation retrieval for user:{}".format(username)) + logger.info(f"Forcing TAS allocation retrieval for user:{username}") raise NotFoundError result = {"hosts": {}, "portal_alloc": None, "active": [], "inactive": []} result.update(IndexedAllocation.from_username(username).value.to_dict()) @@ -79,7 +81,7 @@ def get_project_users_from_name(project_name): : rtype: list """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/projects/name/{1}/users".format(settings.TAS_URL, project_name), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/projects/name/{project_name}/users", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -94,7 +96,7 @@ def get_project_users_from_id(project_id): : rtype: list """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/projects/{1}/users".format(settings.TAS_URL, project_id), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/projects/{project_id}/users", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -109,7 +111,7 @@ def get_project_from_name(project_name): : rtype: dict """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/projects/name/{1}".format(settings.TAS_URL, project_name), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/projects/name/{project_name}", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -124,7 +126,7 @@ def get_project_from_id(project_id): : rtype: dict """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/projects/{1}".format(settings.TAS_URL, project_id), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/projects/{project_id}", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -148,7 +150,7 @@ def get_user_data(username): def get_per_user_allocation_usage(allocation_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get("{0}/v1/allocations/{1}/usage".format(settings.TAS_URL, allocation_id), auth=auth) + r = requests.get(f"{settings.TAS_URL}/v1/allocations/{allocation_id}/usage", auth=auth) resp = r.json() if resp["status"] == "success": return resp["result"] @@ -158,7 +160,7 @@ def get_per_user_allocation_usage(allocation_id): def add_user(project_id, user_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - uri = "{0}/v1/projects/{1}/users/{2}".format(settings.TAS_URL, project_id, user_id) + uri = f"{settings.TAS_URL}/v1/projects/{project_id}/users/{user_id}" r = requests.post(uri, auth=auth) resp = r.json() if resp["status"] != "success": @@ -168,7 +170,7 @@ def add_user(project_id, user_id): def remove_user(project_id, user_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.delete("{0}/v1/projects/{1}/users/{2}".format(settings.TAS_URL, project_id, user_id), auth=auth) + r = requests.delete(f"{settings.TAS_URL}/v1/projects/{project_id}/users/{user_id}", auth=auth) resp = r.json() if resp["status"] != "success": raise ApiException("Failed to delete user: '{}'".format(resp["message"])) @@ -179,7 +181,5 @@ def check_user_groups(username, groups): try: return any(user["username"] == str(username) for group in groups for user in get_project_users_from_name(group)) except Exception as e: - logger.error( - "Issue checking user groups for user:{} which failed with the following exception:{}".format(username, e) - ) + logger.error(f"Issue checking user groups for user:{username} which failed with the following exception:{e}") return False diff --git a/server/portal/apps/users/views.py b/server/portal/apps/users/views.py index 5374ec15d7..6fef39ec3c 100644 --- a/server/portal/apps/users/views.py +++ b/server/portal/apps/users/views.py @@ -1,32 +1,33 @@ -import logging -import requests import json -from zeep import Client -from zeep.transports import Transport -from zeep.cache import InMemoryCache -from zeep.exceptions import Fault +import logging -from portal.views.base import BaseApiView -from portal.apps.users import utils as users_utils +import requests +from django.conf import settings from django.contrib.auth import get_user_model -from django.forms.models import model_to_dict -from django.http import HttpResponseNotFound, HttpResponseBadRequest, JsonResponse -from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist +from django.forms.models import model_to_dict +from django.http import HttpResponseBadRequest, HttpResponseNotFound, JsonResponse from django.utils.decorators import method_decorator -from django.conf import settings from elasticsearch_dsl import Q -from portal.libs.elasticsearch.docs.base import IndexedFile from pytas.http import TASClient +from zeep import Client +from zeep.cache import InMemoryCache +from zeep.exceptions import Fault +from zeep.transports import Transport + +from portal.apps.users import utils as users_utils from portal.apps.users.utils import ( + add_user, get_allocations, - get_user_data, get_per_user_allocation_usage, - add_user, - remove_user, get_project_from_id, get_project_users_from_id, + get_user_data, + remove_user, ) +from portal.libs.elasticsearch.docs.base import IndexedFile +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -129,12 +130,14 @@ class AllocationsView(BaseApiView): def get(self, request): """Returns active user allocations on TACC resources - : returns: {'response': {'active': allocations, 'portal_alloc': settings.PORTAL_ALLOCATION, 'inactive': inactive, 'hosts': hosts}} + : returns: {'response': {'active': allocations, + 'portal_alloc': settings.PORTAL_ALLOCATION, 'inactive': inactive, + 'hosts': hosts}} : rtype: dict """ data = get_allocations(request.user.username) - # This line iterates through all the projects in the active allocations and filters out the ones that are excluded in settings. + # Filter active allocations to remove projects excluded in settings. filtered_projects = [ project for project in data["active"] if project.get("projectName") not in settings.ALLOCATIONS_TO_EXCLUDE ] @@ -244,7 +247,8 @@ def put(self, request): tas_client.service.EditProjectUser(user_id, user_role) except Exception: raise Exception( - f"Error assigning user: {user_id} new role: {user_role} to project name:id : {project_name}:{project_id}" + f"Error assigning user: {user_id} new role: {user_role} " + f"to project name:id : {project_name}:{project_id}" ) return JsonResponse({"response": "ok"}) @@ -260,11 +264,11 @@ def get(self, request, allocation_id): @method_decorator(login_required, name="dispatch") class AllocationManagementView(BaseApiView): def post(self, request, project_id, user_id): - logger.info("Adding {} to TAS project {}".format(user_id, project_id)) + logger.info(f"Adding {user_id} to TAS project {project_id}") add_user(project_id, user_id) return JsonResponse({"response": "ok"}) def delete(self, request, project_id, user_id): - logger.info("Deleting {} to TAS project {}".format(user_id, project_id)) + logger.info(f"Deleting {user_id} to TAS project {project_id}") remove_user(project_id, user_id) return JsonResponse({"response": "ok"}) diff --git a/server/portal/apps/webhooks/callback.py b/server/portal/apps/webhooks/callback.py index e627346657..ae43f95bfe 100644 --- a/server/portal/apps/webhooks/callback.py +++ b/server/portal/apps/webhooks/callback.py @@ -1,7 +1,7 @@ from abc import abstractmethod -class WebhookCallback(object): +class WebhookCallback: """class WebhookCallback An abstract base class for executing callback functions upon diff --git a/server/portal/apps/webhooks/fields.py b/server/portal/apps/webhooks/fields.py index f44929bae0..3a820871f5 100644 --- a/server/portal/apps/webhooks/fields.py +++ b/server/portal/apps/webhooks/fields.py @@ -4,7 +4,6 @@ from django.contrib.postgres.fields import JSONField as DjangoJSONField from django.db.models import Field - # Hack for loading Field class appropriate to either Postgres or SQLite (testing db) # from https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1 diff --git a/server/portal/apps/webhooks/migrations/0001_initial.py b/server/portal/apps/webhooks/migrations/0001_initial.py index dd7f654449..4121be0924 100644 --- a/server/portal/apps/webhooks/migrations/0001_initial.py +++ b/server/portal/apps/webhooks/migrations/0001_initial.py @@ -1,9 +1,11 @@ # Generated by Django 2.2.7 on 2020-09-17 19:05 import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + import portal.apps.webhooks.fields diff --git a/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py b/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py index a120000022..eae4eb6b65 100644 --- a/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py +++ b/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py @@ -1,9 +1,10 @@ # Generated by Django 3.2.19 on 2023-05-04 22:02 import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py b/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py index 528a2948b6..cd310fc2c8 100644 --- a/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py +++ b/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py @@ -1,6 +1,7 @@ # Generated by Django 2.2.28 on 2022-12-21 21:14 from django.db import migrations + import portal.utils.fields diff --git a/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py b/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py index b59163d643..4fd68584df 100644 --- a/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py +++ b/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.10 on 2024-03-19 16:38 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/webhooks/models.py b/server/portal/apps/webhooks/models.py index 60a78a87da..1ee5b7b378 100644 --- a/server/portal/apps/webhooks/models.py +++ b/server/portal/apps/webhooks/models.py @@ -1,5 +1,5 @@ -from django.db import models from django.conf import settings +from django.db import models from django.utils import timezone diff --git a/server/portal/apps/webhooks/unit_test.py b/server/portal/apps/webhooks/unit_test.py index 1668f2f9aa..215dbd4233 100644 --- a/server/portal/apps/webhooks/unit_test.py +++ b/server/portal/apps/webhooks/unit_test.py @@ -1,16 +1,18 @@ import json import os +from unittest.mock import MagicMock, patch from urllib.parse import urlencode -from mock import patch, MagicMock -from django.test import TestCase, TransactionTestCase, override_settings + from django.contrib.auth import get_user_model from django.db.models import signals +from django.test import TestCase, TransactionTestCase, override_settings from django.urls import reverse from tapipy.tapis import TapisResult + from portal.apps.notifications.models import Notification from portal.apps.signals.receivers import send_notification_ws -from portal.libs.exceptions import PortalLibException from portal.apps.webhooks.views import validate_tapis_job +from portal.libs.exceptions import PortalLibException class TestValidateTapisJob(TestCase): diff --git a/server/portal/apps/webhooks/urls.py b/server/portal/apps/webhooks/urls.py index 4828be8b07..3dbe55f849 100644 --- a/server/portal/apps/webhooks/urls.py +++ b/server/portal/apps/webhooks/urls.py @@ -1,8 +1,8 @@ """Webhooks URLs""" from django.urls import path -from portal.apps.webhooks import views +from portal.apps.webhooks import views app_name = "webhooks" urlpatterns = [ diff --git a/server/portal/apps/webhooks/utils.py b/server/portal/apps/webhooks/utils.py index 53486ec920..5c6f033ced 100644 --- a/server/portal/apps/webhooks/utils.py +++ b/server/portal/apps/webhooks/utils.py @@ -1,12 +1,14 @@ -from portal.apps.webhooks.callback import WebhookCallback -from portal.apps.webhooks.models import ExternalCall -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist +import logging import random import string -from inspect import isclass from importlib import import_module -import logging +from inspect import isclass + +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist + +from portal.apps.webhooks.callback import WebhookCallback +from portal.apps.webhooks.models import ExternalCall logger = logging.getLogger(__name__) @@ -24,7 +26,7 @@ def register_webhook(callback=None, callback_data=None, user=None): external_call = ExternalCall.objects.create( callback=callback, callback_data=callback_data, user=user, webhook_id=get_webhook_id() ) - return "{}/webhooks/callbacks/{}/".format(settings.VANITY_BASE_URL, external_call.webhook_id) + return f"{settings.VANITY_BASE_URL}/webhooks/callbacks/{external_call.webhook_id}/" def validate_webhook(webhook_id): @@ -46,10 +48,10 @@ def load_callback(callback_name): module = import_module(module_str) call = getattr(module, callable_str) if not isclass(call): - raise ValueError("{callback_name} is not a class".format(callback_name=callback_name)) + raise ValueError(f"{callback_name} is not a class") callback_instance = call() if not isinstance(callback_instance, WebhookCallback): - raise ValueError("{callback_name} is not a subclass of WebhookCallback".format(callback_name=callback_name)) + raise ValueError(f"{callback_name} is not a subclass of WebhookCallback") return callback_instance diff --git a/server/portal/apps/webhooks/utils_unit_test.py b/server/portal/apps/webhooks/utils_unit_test.py index 0cabee9330..65ed0a4716 100644 --- a/server/portal/apps/webhooks/utils_unit_test.py +++ b/server/portal/apps/webhooks/utils_unit_test.py @@ -1,8 +1,8 @@ -from portal.apps.webhooks.models import ExternalCall -from portal.apps.webhooks.callback import WebhookCallback -from portal.apps.webhooks.utils import load_callback, register_webhook, validate_webhook, execute_callback import pytest +from portal.apps.webhooks.callback import WebhookCallback +from portal.apps.webhooks.models import ExternalCall +from portal.apps.webhooks.utils import execute_callback, load_callback, register_webhook, validate_webhook pytestmark = pytest.mark.django_db @@ -13,7 +13,7 @@ def callback(self, external_call, webhook_request): assert webhook_request == "mock_request" -class InvalidCallback(object): +class InvalidCallback: pass diff --git a/server/portal/apps/webhooks/views.py b/server/portal/apps/webhooks/views.py index c7a3595b98..4cef360508 100644 --- a/server/portal/apps/webhooks/views.py +++ b/server/portal/apps/webhooks/views.py @@ -1,25 +1,23 @@ import json import logging +from django.conf import settings from django.contrib.auth import get_user_model -from django.views.decorators.csrf import csrf_exempt -from django.utils.decorators import method_decorator +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.http import HttpResponse, HttpResponseBadRequest -from django.core.exceptions import ObjectDoesNotExist - +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt from requests import HTTPError from tapipy.errors import BaseTapyException from portal.apps.notifications.models import Notification from portal.apps.search.tasks import tapis_indexer -from portal.views.base import BaseApiView -from portal.libs.exceptions import PortalLibException -from portal.exceptions.api import ApiException -from portal.apps.webhooks.utils import validate_webhook, execute_callback +from portal.apps.webhooks.utils import execute_callback, validate_webhook from portal.apps.workspace.api.utils import check_job_for_timeout - -from django.conf import settings +from portal.exceptions.api import ApiException +from portal.libs.exceptions import PortalLibException +from portal.views.base import BaseApiView logger = logging.getLogger(__name__) @@ -43,9 +41,8 @@ def validate_tapis_job(job_uuid, job_owner, disallowed_states=[]): # Validate the job UUID against the owner if job_data.owner != job_owner: logger.error( - "Tapis job (owner='{}', status='{}) for this event (owner='{}') is not valid".format( - job_data.owner, job_data.status, job_owner - ) + f"Tapis job (owner='{job_data.owner}', status='{job_data.status}) " + " for this event (owner='{job_owner}') is not valid" ) raise PortalLibException("Unable to find a related valid job for this notification.") @@ -94,9 +91,7 @@ def post(self, request, *args, **kwargs): # Do nothing on job status not in portal notification states if job_status not in settings.PORTAL_JOB_NOTIFICATION_STATES: logger.info( - "Job UUID {} for owner {} entered {} state (no notification sent)".format( - job_uuid, username, job_status - ) + f"Job UUID {job_uuid} for owner {username} entered {job_status} state (no notification sent)" ) return HttpResponse("OK") @@ -104,7 +99,7 @@ def post(self, request, *args, **kwargs): if job_status == job_old_status: return HttpResponse("OK") - logger.info("JOB STATUS CHANGE: UUID={} status={}".format(job_uuid, job_status)) + logger.info(f"JOB STATUS CHANGE: UUID={job_uuid} status={job_status}") event_data = { Notification.EVENT_TYPE: "job", @@ -121,7 +116,7 @@ def post(self, request, *args, **kwargs): event_data[Notification.EXTRA]["status"] = job_details.status try: - logger.info("Indexing job output for job={}".format(job_uuid)) + logger.info(f"Indexing job output for job={job_uuid}") tapis_indexer.apply_async( kwargs={ @@ -131,7 +126,7 @@ def post(self, request, *args, **kwargs): } ) except Exception as e: - logger.exception("Error starting async task to index job output: {}".format(e)) + logger.exception(f"Error starting async task to index job output: {e}") with transaction.atomic(): Notification.objects.create(**event_data) @@ -179,9 +174,7 @@ def post(self, request, *args, **kwargs): try: valid_state = validate_tapis_job(job_uuid, job_owner, TERMINAL_JOB_STATES) if not valid_state: - raise PortalLibException( - "Interactive Job UUID {} for user {} was in invalid state".format(job_uuid, job_owner) - ) + raise PortalLibException(f"Interactive Job UUID {job_uuid} for user {job_owner} was in invalid state") event_data[Notification.EXTRA] = { "name": valid_state.name, "status": valid_state.status, diff --git a/server/portal/apps/webhooks/views_unit_test.py b/server/portal/apps/webhooks/views_unit_test.py index ac138459a0..84574a0679 100644 --- a/server/portal/apps/webhooks/views_unit_test.py +++ b/server/portal/apps/webhooks/views_unit_test.py @@ -1,9 +1,10 @@ -from portal.apps.webhooks.utils import register_webhook -from portal.apps.webhooks.callback import WebhookCallback -from django.conf import settings import json + import pytest +from django.conf import settings +from portal.apps.webhooks.callback import WebhookCallback +from portal.apps.webhooks.utils import register_webhook pytestmark = pytest.mark.django_db diff --git a/server/portal/apps/workbench/api/unit_test.py b/server/portal/apps/workbench/api/unit_test.py index b60650a8ff..86562a6d8d 100644 --- a/server/portal/apps/workbench/api/unit_test.py +++ b/server/portal/apps/workbench/api/unit_test.py @@ -1,4 +1,5 @@ import json + from django.conf import settings diff --git a/server/portal/apps/workbench/api/urls.py b/server/portal/apps/workbench/api/urls.py index 3c41256e3e..1682ed3e48 100644 --- a/server/portal/apps/workbench/api/urls.py +++ b/server/portal/apps/workbench/api/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from portal.apps.workbench.api import views app_name = "workbench_api" diff --git a/server/portal/apps/workbench/urls.py b/server/portal/apps/workbench/urls.py index 6079bbbafa..af074d3381 100644 --- a/server/portal/apps/workbench/urls.py +++ b/server/portal/apps/workbench/urls.py @@ -4,6 +4,7 @@ """ from django.urls import re_path + from portal.apps.workbench.views import IndexView app_name = "workbench" diff --git a/server/portal/apps/workbench/views.py b/server/portal/apps/workbench/views.py index 3835c2fa3a..ab5c72103f 100644 --- a/server/portal/apps/workbench/views.py +++ b/server/portal/apps/workbench/views.py @@ -1,7 +1,7 @@ -from django.views.generic.base import TemplateView +from django.conf import settings from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator -from django.conf import settings +from django.views.generic.base import TemplateView @method_decorator(login_required, name="dispatch") @@ -13,10 +13,10 @@ class IndexView(TemplateView): template_name = "portal/apps/workbench/index.html" def dispatch(self, request, *args, **kwargs): - return super(IndexView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): - context = super(IndexView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context["setup_complete"] = self.request.user.profile.setup_complete context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/workspace/admin.py b/server/portal/apps/workspace/admin.py index 35507b699a..d1d9c8bf7f 100644 --- a/server/portal/apps/workspace/admin.py +++ b/server/portal/apps/workspace/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin -from portal.apps.workspace.models import AppTrayEntry, AppTrayCategory + +from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry @admin.register(AppTrayCategory) diff --git a/server/portal/apps/workspace/api/handlers/tapis_handlers.py b/server/portal/apps/workspace/api/handlers/tapis_handlers.py index 1d504edd2f..062b1b837e 100644 --- a/server/portal/apps/workspace/api/handlers/tapis_handlers.py +++ b/server/portal/apps/workspace/api/handlers/tapis_handlers.py @@ -1,6 +1,5 @@ import logging - logger = logging.getLogger(__name__) diff --git a/server/portal/apps/workspace/api/lookups.py b/server/portal/apps/workspace/api/lookups.py index b3fa1ebd3f..f31adb3efa 100644 --- a/server/portal/apps/workspace/api/lookups.py +++ b/server/portal/apps/workspace/api/lookups.py @@ -2,11 +2,12 @@ :synopsis: Function to lookup manager classes """ -from importlib import import_module import logging +from importlib import import_module + from django.conf import settings -from portal.exceptions.api import ApiException +from portal.exceptions.api import ApiException logger = logging.getLogger(__name__) diff --git a/server/portal/apps/workspace/api/unit_test.py b/server/portal/apps/workspace/api/unit_test.py index 14cb57f6ea..1e52cd4984 100644 --- a/server/portal/apps/workspace/api/unit_test.py +++ b/server/portal/apps/workspace/api/unit_test.py @@ -1,8 +1,9 @@ import json +from unittest.mock import patch + import pytest -from mock import patch -from django.test import TestCase from django.contrib.auth import get_user_model +from django.test import TestCase @pytest.mark.django_db(transaction=True) @@ -20,7 +21,7 @@ def tearDown(self): def test_job_history_get(self): job_uuid = "032142c3-ac6a-42cb-841e-fbc26a2d951c-007" self.mock_tapis_client.jobs.getJobHistory.return_value = "mock_response" - response = self.client.get("/api/workspace/jobs/{}/history".format(job_uuid)) + response = self.client.get(f"/api/workspace/jobs/{job_uuid}/history") self.mock_tapis_client.jobs.getJobHistory.assert_called_with( jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{self.client.session.session_key}"}, diff --git a/server/portal/apps/workspace/api/urls.py b/server/portal/apps/workspace/api/urls.py index 680035ceb1..f26e242a40 100644 --- a/server/portal/apps/workspace/api/urls.py +++ b/server/portal/apps/workspace/api/urls.py @@ -1,8 +1,8 @@ """Workpace API Urls""" from django.urls import re_path -from portal.apps.workspace.api import views +from portal.apps.workspace.api import views app_name = "workspace_api" urlpatterns = [ diff --git a/server/portal/apps/workspace/api/utils.py b/server/portal/apps/workspace/api/utils.py index 46746b78d8..c23f070621 100644 --- a/server/portal/apps/workspace/api/utils.py +++ b/server/portal/apps/workspace/api/utils.py @@ -2,8 +2,10 @@ import json import logging + from django.conf import settings -from tapipy.errors import BaseTapyException, UnauthorizedError, ForbiddenError +from tapipy.errors import BaseTapyException, ForbiddenError, UnauthorizedError + from portal.apps.onboarding.steps.system_access_v3 import create_system_credentials_with_tms from portal.exceptions.api import ApiException @@ -13,7 +15,8 @@ def get_tapis_timeout_error_messages(job_id): return [ "JOBS_EARLY_TERMINATION Job terminated by Tapis because: TIME_EXPIRED", - f'JOBS_USER_APP_FAILURE The user application ({job_id}) ended with remote status "TIMEOUT" and returned exit code: 0:0.', + f"JOBS_USER_APP_FAILURE The user application ({job_id}) ended with remote " + 'status "TIMEOUT" and returned exit code: 0:0.', ] diff --git a/server/portal/apps/workspace/api/utils_unit_test.py b/server/portal/apps/workspace/api/utils_unit_test.py index fe76ec4913..a6b3fe30d9 100644 --- a/server/portal/apps/workspace/api/utils_unit_test.py +++ b/server/portal/apps/workspace/api/utils_unit_test.py @@ -1,6 +1,8 @@ -from tapipy.tapis import TapisResult +from unittest.mock import patch + from tapipy.errors import UnauthorizedError -from mock import patch +from tapipy.tapis import TapisResult + from portal.apps.workspace.api.utils import ( push_keys_required_if_not_credentials_ensured, ) @@ -10,7 +12,8 @@ def test_push_keys_required_if_not_credentials_ensured_successful_credential_cre authenticated_user, mock_tapis_client ): """ - Test that the push_keys_required_if_not_credentials_ensured function returns False when the user has system credentials + Test that the push_keys_required_if_not_credentials_ensured function + returns False when the user has system credentials and the system does not require keys to be pushed. """ system = { diff --git a/server/portal/apps/workspace/api/views.py b/server/portal/apps/workspace/api/views.py index 8f62fbd857..d80fe9aefe 100644 --- a/server/portal/apps/workspace/api/views.py +++ b/server/portal/apps/workspace/api/views.py @@ -3,34 +3,35 @@ :synopsys: Views to handle Workspace API """ -import logging import json +import logging from urllib.parse import urlparse -from django.http import JsonResponse + from django.conf import settings from django.contrib.auth.decorators import login_required -from django.utils.decorators import method_decorator -from django.urls import reverse -from django.db.models.functions import Coalesce from django.core.exceptions import PermissionDenied +from django.db.models.functions import Coalesce +from django.http import JsonResponse +from django.urls import reverse +from django.utils.decorators import method_decorator from tapipy.tapis import TapisResult -from portal.views.base import BaseApiView -from portal.exceptions.api import ApiException + +from portal.apps.datafiles.utils import evaluate_datafiles_storage_system from portal.apps.licenses.models import LICENSE_TYPES, get_license_info -from portal.libs.agave.utils import service_account -from portal.libs.agave.serializers import BaseTapisResultSerializer +from portal.apps.workspace.api.utils import check_job_for_timeout, push_keys_required_if_not_credentials_ensured # TODOv3: dropV2Jobs -from portal.apps.workspace.models import JobSubmission -from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry -from .handlers.tapis_handlers import tapis_get_handler -from portal.apps.workspace.api.utils import check_job_for_timeout, push_keys_required_if_not_credentials_ensured +from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry, JobSubmission +from portal.exceptions.api import ApiException +from portal.libs.agave.serializers import BaseTapisResultSerializer +from portal.libs.agave.utils import service_account from portal.utils import get_client_ip -from portal.apps.datafiles.utils import evaluate_datafiles_storage_system +from portal.views.base import BaseApiView +from .handlers.tapis_handlers import tapis_get_handler logger = logging.getLogger(__name__) -METRICS = logging.getLogger("metrics.{}".format(__name__)) +METRICS = logging.getLogger(f"metrics.{__name__}") def _app_license_type(app_def): @@ -277,14 +278,16 @@ def has_timeout_message(job): qs_upper = query_string.upper() if query_string else "" qs_title = query_string.title() if query_string else "" - # TODO WP-1116: all status search add-ons to be removed and added to drop-down feature on frontend + # TODO WP-1116: all status search add-ons to be removed and added to + # drop-down feature on frontend if status_searches: enhanced_status_conditions = [] for status in status_searches: if status == "FINISHED": enhanced_status_conditions.append( - "(status = 'FINISHED' OR (status = 'FAILED' AND (lastMessage LIKE '%TIME_EXPIRED%' OR lastMessage LIKE '%TIMEOUT%')))" + "(status = 'FINISHED' OR (status = 'FAILED' AND " + "(lastMessage LIKE '%TIME_EXPIRED%' OR lastMessage LIKE '%TIMEOUT%')))" ) elif status == "FAILED": enhanced_status_conditions.append("(status = 'FAILED')") @@ -336,7 +339,8 @@ def has_timeout_message(job): # (ii) FAILED jobs that are interactive and have the timeout/expired message (will be shown as FINISHED on UI) # For "Failed" search, want to get all - # (i) FAILED jobs except those that were interactive and have the timeout/expired message (excluded because they are shown as FINISHED on UI) + # (i) FAILED jobs except those that were interactive and have the timeout/expired message + # (excluded because they are shown as FINISHED on UI) is_finished = "FINISHED" in status_searches is_failed = "FAILED" in status_searches @@ -437,7 +441,7 @@ def post(self, request, *args, **kwargs): if job_uuid and job_action: if job_action == "resubmit": - logger.info("user:{} is resubmitting job uuid:{}".format(username, job_uuid)) + logger.info(f"user:{username} is resubmitting job uuid:{job_uuid}") data = tapis.jobs.resubmitJob( jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} ) @@ -461,7 +465,7 @@ def post(self, request, *args, **kwargs): ) elif job_action == "cancel": - logger.info("user:{} is canceling/stopping job uuid:{}".format(username, job_uuid)) + logger.info(f"user:{username} is canceling/stopping job uuid:{job_uuid}") data = tapis.jobs.cancelJob( jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} ) @@ -485,9 +489,8 @@ def post(self, request, *args, **kwargs): ) else: raise ApiException( - "user:{} is trying to run an unsupported job action: {} for job uuid: {}".format( - username, job_action, job_uuid - ), + f"user:{username} is trying to run an unsupported job action: {job_action}" + f" for job uuid: {job_uuid}", status=400, ) @@ -501,15 +504,13 @@ def post(self, request, *args, **kwargs): elif not job_post: raise ApiException( - "user:{} is submitting a request with no job body.".format( - username, - ), + f"user:{username} is submitting a request with no job body.", status=400, ) # submit job else: - logger.info("processing job submission for user:{}: {}".format(username, job_post)) + logger.info(f"processing job submission for user:{username}: {job_post}") # Provide default job archive configuration if none is provided and portal has default system if settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM: @@ -540,13 +541,16 @@ def post(self, request, *args, **kwargs): if lic is None: raise ApiException("You are missing the required license for this application.") - # TODOv3: Multistring licenses break environment variables. Determine how to handle multistring licenses, if needed at all. + # TODOv3: Multistring licenses break environment variables. + # Determine how to handle multistring licenses, if needed at all. # https://jira.tacc.utexas.edu/browse/WP-70 # license_var = { # "key": "_license", # "value": lic.license_as_str() # } - # job_post['parameterSet']['envVariables'] = job_post['parameterSet'].get('envVariables', []) + [license_var] + # job_post["parameterSet"]["envVariables"] = job_post["parameterSet"].get("envVariables", []) + [ + # license_var + # ] # Test file listing on relevant systems to determine whether keys need to be pushed manually for system_id in list(filter(None, [job_post.get("archiveSystemId"), execSystemId])): @@ -596,7 +600,7 @@ def post(self, request, *args, **kwargs): } ] - logger.info("user:{} is submitting job:{}".format(username, job_post)) + logger.info(f"user:{username} is submitting job:{job_post}") response = tapis.jobs.submitJob( **job_post, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} ) @@ -636,11 +640,11 @@ def get(self, request, *args, **kwargs): user_role = request.GET.get("user_role") system_id = request.GET.get("system_id") if roles: - logger.info("user:{} tapis.systems.listRoles system_id:{}".format(request.user.username, system_id)) + logger.info(f"user:{request.user.username} tapis.systems.listRoles system_id:{system_id}") agc = service_account() data = agc.systems.listRoles(systemId=system_id) elif user_role: - logger.info("user:{} tapis.systems.getRoleForUser system_id:{}".format(request.user.username, system_id)) + logger.info(f"user:{request.user.username} tapis.systems.getRoleForUser system_id:{system_id}") agc = service_account() data = agc.systems.getRoleForUser(systemId=system_id, username=request.user.username) return JsonResponse({"response": data}) @@ -649,7 +653,7 @@ def post(self, request, *args, **kwargs): body = json.loads(request.body) role = body["role"] system_id = body["system_id"] - logger.info("user:{} tapis.systems.updateRole system_id:{}".format(request.user.username, system_id)) + logger.info(f"user:{request.user.username} tapis.systems.updateRole system_id:{system_id}") role_body = {"username": request.user.username, "role": role} agc = service_account() data = agc.systems.updateRole(systemId=system_id, body=role_body) @@ -824,7 +828,8 @@ def get(self, request): """ tabs, html_definitions = self.getPublicApps(request.user) - # Add "My Apps" tab with all of the user's private and shared apps that are enabled, or all available apps if no public apps are enabled + # Add "My Apps" tab with all of the user's private and shared apps + # that are enabled, or all available apps if no public apps are enabled my_apps = self.getPrivateApps(request.user) shared_apps = self.getSharedApps(request.user) @@ -843,7 +848,7 @@ def get(self, request, operation=None): return JsonResponse({"message": "This view requires authentication."}, status=403) get_params = request.GET.dict() - logger.info("user:%s op:%s query_params:%s" % (request.user.username, operation, get_params)) + logger.info(f"user:{request.user.username} op:{operation} query_params:{get_params}") response = tapis_get_handler(client, operation, **get_params) return JsonResponse({"data": response}) diff --git a/server/portal/apps/workspace/api/views_unit_test.py b/server/portal/apps/workspace/api/views_unit_test.py index 832dd3aadc..a8b8489688 100644 --- a/server/portal/apps/workspace/api/views_unit_test.py +++ b/server/portal/apps/workspace/api/views_unit_test.py @@ -1,13 +1,13 @@ -from django.conf import settings -from portal.apps.workspace.api.views import AppsTrayView -from portal.apps.workspace.models import AppTrayCategory -from portal.apps.workspace.models import JobSubmission import json import os + import pytest -from tapipy.tapis import TapisResult +from django.conf import settings from django.core.management import call_command +from tapipy.tapis import TapisResult +from portal.apps.workspace.api.views import AppsTrayView +from portal.apps.workspace.models import AppTrayCategory, JobSubmission pytest.mark.django_db(transaction=True) @@ -187,7 +187,7 @@ def test_job_post_is_logged_for_metrics( # Ensure metric-related logging is being performed logging_metric_mock.assert_called_with( - "user:{} is submitting job:{}".format(authenticated_user.username, tapis_job_submission) + f"user:{authenticated_user.username} is submitting job:{tapis_job_submission}" ) diff --git a/server/portal/apps/workspace/management/commands/import-apps.py b/server/portal/apps/workspace/management/commands/import_apps.py similarity index 95% rename from server/portal/apps/workspace/management/commands/import-apps.py rename to server/portal/apps/workspace/management/commands/import_apps.py index e9b86da5be..cdffd51e16 100644 --- a/server/portal/apps/workspace/management/commands/import-apps.py +++ b/server/portal/apps/workspace/management/commands/import_apps.py @@ -1,10 +1,11 @@ import logging -from django.core.management import BaseCommand + from django.conf import settings +from django.core.management import BaseCommand from tapipy.errors import NotFoundError -from portal.libs.agave.utils import service_account -from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry +from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry +from portal.libs.agave.utils import service_account logger = logging.getLogger(__name__) @@ -63,10 +64,10 @@ def import_apps(self, portal_names): appId=app.get("id"), ) - logger.info("Imported {}".format(app_entry)) + logger.info(f"Imported {app_entry}") except Exception: logger.exception("Error importing application") - logger.info("Following app could not be imported: {}".format(app)) + logger.info(f"Following app could not be imported: {app}") def handle(self, *args, **options): if options["clean"]: diff --git a/server/portal/apps/workspace/management/commands/import-jobs.py b/server/portal/apps/workspace/management/commands/import_jobs.py similarity index 95% rename from server/portal/apps/workspace/management/commands/import-jobs.py rename to server/portal/apps/workspace/management/commands/import_jobs.py index 3cc45d29a3..32e1918466 100644 --- a/server/portal/apps/workspace/management/commands/import-jobs.py +++ b/server/portal/apps/workspace/management/commands/import_jobs.py @@ -1,8 +1,9 @@ -from django.core.management import BaseCommand -from portal.libs.agave.utils import service_account +import dateutil.parser from django.contrib.auth import get_user_model +from django.core.management import BaseCommand + from portal.apps.workspace.models import JobSubmission -import dateutil.parser +from portal.libs.agave.utils import service_account class Command(BaseCommand): @@ -34,4 +35,4 @@ def handle(self, *args, **options): offset += 100 done = len(jobs) < 100 total += len(jobs) - print("{} jobs for {}".format(total, user.username)) + print(f"{total} jobs for {user.username}") diff --git a/server/portal/apps/workspace/management/commands/unit_test.py b/server/portal/apps/workspace/management/commands/unit_test.py index a1466309ab..ab1d72f040 100644 --- a/server/portal/apps/workspace/management/commands/unit_test.py +++ b/server/portal/apps/workspace/management/commands/unit_test.py @@ -1,8 +1,10 @@ -from mock import patch +from unittest.mock import patch + import pytest -from django.test import TransactionTestCase from django.contrib.auth import get_user_model from django.core.management import call_command +from django.test import TransactionTestCase + from portal.apps.workspace.models import JobSubmission @@ -11,14 +13,14 @@ class TestImportJobs(TransactionTestCase): fixtures = ["users"] def setUp(self): - self.mock_client_patcher = patch("portal.apps.workspace.management.commands.import-jobs.service_account") + self.mock_client_patcher = patch("portal.apps.workspace.management.commands.import_jobs.service_account") self.mock_client = self.mock_client_patcher.start() self.user = get_user_model().objects.get(username="username") def tearDown(self): self.mock_client_patcher.stop() - @patch("portal.apps.workspace.management.commands.import-jobs.get_user_model") + @patch("portal.apps.workspace.management.commands.import_jobs.get_user_model") def test_import(self, mock_user_model): mock_user_model.return_value.objects.all.return_value = [self.user] JobSubmission.objects.create(jobId="1234", user=self.user) @@ -27,7 +29,7 @@ def test_import(self, mock_user_model): {"id": "5678", "created": "2019-10-29T19:30:13Z"}, ] - call_command("import-jobs") + call_command("import_jobs") result = JobSubmission.objects.all().filter(user=self.user) self.assertEqual(len(result), 2) diff --git a/server/portal/apps/workspace/migrations/0001_initial.py b/server/portal/apps/workspace/migrations/0001_initial.py index c4ef3f9a6c..b767d435c0 100644 --- a/server/portal/apps/workspace/migrations/0001_initial.py +++ b/server/portal/apps/workspace/migrations/0001_initial.py @@ -1,9 +1,10 @@ # Generated by Django 2.2.7 on 2019-11-05 19:50 import datetime + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py b/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py index 0b38d84553..9b1cb5af7d 100644 --- a/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py +++ b/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py @@ -1,9 +1,9 @@ # Generated by Django 3.2.19 on 2023-05-04 22:05 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py b/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py index df8bfbfb31..4f4a403987 100644 --- a/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py +++ b/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py @@ -1,7 +1,7 @@ # Generated by Django 2.2.7 on 2020-02-18 21:15 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py b/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py index 25f43b388b..ce3ec93f94 100644 --- a/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py +++ b/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py @@ -1,7 +1,7 @@ # Generated by Django 2.2.16 on 2020-11-17 17:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py b/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py index f42683aefd..0650a7c6b4 100644 --- a/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py +++ b/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py @@ -1,6 +1,7 @@ # Generated by Django 2.2.28 on 2022-12-21 21:14 from django.db import migrations + import portal.utils.fields diff --git a/server/portal/apps/workspace/models.py b/server/portal/apps/workspace/models.py index 2b61566370..3528f42f3f 100644 --- a/server/portal/apps/workspace/models.py +++ b/server/portal/apps/workspace/models.py @@ -1,5 +1,5 @@ -from django.db import models from django.conf import settings +from django.db import models from django.utils import timezone @@ -29,7 +29,7 @@ class AppTrayCategory(models.Model): ) def __str__(self): - return "%s" % (self.category) + return f"{self.category}" class AppTrayEntry(models.Model): @@ -48,8 +48,8 @@ class AppTrayEntry(models.Model): def __str__(self): if self.appType == "html": - return "%s: %s (HTML)" % (self.label, self.appId) - return "%s%s%s" % ( + return f"{self.label}: {self.appId} (HTML)" + return "{}{}{}".format( f"{self.label}: " if self.label else "", self.appId, f"-{self.version}" if self.version else "", diff --git a/server/portal/apps/workspace/models_unit_test.py b/server/portal/apps/workspace/models_unit_test.py index 6a1fc6cd96..21b3e0b25b 100644 --- a/server/portal/apps/workspace/models_unit_test.py +++ b/server/portal/apps/workspace/models_unit_test.py @@ -1,4 +1,4 @@ -from portal.apps.workspace.models import JobSubmission, AppTrayCategory, AppTrayEntry +from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry, JobSubmission def test_job_submission_model(django_db_reset_sequences, regular_user): diff --git a/server/portal/apps/workspace/unit_test.py b/server/portal/apps/workspace/unit_test.py index e22af05f19..ed6f5bf8f1 100644 --- a/server/portal/apps/workspace/unit_test.py +++ b/server/portal/apps/workspace/unit_test.py @@ -1,10 +1,11 @@ import json import os -from mock import patch -from django.test import TestCase +from unittest.mock import patch + +import pytest from django.conf import settings from django.contrib.auth import get_user_model -import pytest +from django.test import TestCase @pytest.mark.django_db(transaction=True) @@ -13,7 +14,7 @@ class TestAppsApiViews(TestCase): @classmethod def setUpClass(cls): - super(TestAppsApiViews, cls).setUpClass() + super().setUpClass() cls.mock_client_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client") cls.mock_client = cls.mock_client_patcher.start() cls.mock_get_user_data_patcher = patch("portal.apps.users.utils.get_user_data") @@ -24,7 +25,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - super(TestAppsApiViews, cls).tearDownClass() + super().tearDownClass() cls.mock_get_user_data_patcher.stop() cls.mock_client_patcher.stop() diff --git a/server/portal/apps/workspace/urls.py b/server/portal/apps/workspace/urls.py index 0220e5165d..9585b452db 100644 --- a/server/portal/apps/workspace/urls.py +++ b/server/portal/apps/workspace/urls.py @@ -1,6 +1,7 @@ """Workspace URLs""" from django.urls import re_path + from portal.apps.workspace import views urlpatterns = [ diff --git a/server/portal/apps/workspace/views.py b/server/portal/apps/workspace/views.py index ee83bfe391..a95d861b8c 100644 --- a/server/portal/apps/workspace/views.py +++ b/server/portal/apps/workspace/views.py @@ -3,10 +3,10 @@ :synopsis: Views to handle Workspace """ -from django.views.generic.base import TemplateView +from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator from django.views.decorators.csrf import ensure_csrf_cookie -from django.contrib.auth.decorators import login_required +from django.views.generic.base import TemplateView @method_decorator(login_required, name="dispatch") @@ -18,4 +18,4 @@ class WorkspaceView(TemplateView): @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): """Overwrite dispatch to ensure csrf cookie""" - return super(WorkspaceView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) diff --git a/server/portal/asgi.py b/server/portal/asgi.py index 620514d02a..f4d5d683e1 100644 --- a/server/portal/asgi.py +++ b/server/portal/asgi.py @@ -4,12 +4,13 @@ """ import os + import django +from channels.auth import AuthMiddlewareStack from channels.routing import ProtocolTypeRouter, URLRouter from django.core.asgi import get_asgi_application -from channels.auth import AuthMiddlewareStack -import portal.apps.notifications.routing +import portal.apps.notifications.routing os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings") django.setup() diff --git a/server/portal/celery.py b/server/portal/celery.py index 5ac2f1f9f9..a436ee3d90 100644 --- a/server/portal/celery.py +++ b/server/portal/celery.py @@ -1,4 +1,5 @@ import os + from celery import Celery from celery.schedules import crontab from django.conf import settings @@ -28,4 +29,4 @@ @app.task(bind=True) def debug_task(self): - print(("Request: {0!r}".format(self.request))) + print(f"Request: {self.request!r}") diff --git a/server/portal/exceptions/api.py b/server/portal/exceptions/api.py index df8564ac4d..613b3fca08 100644 --- a/server/portal/exceptions/api.py +++ b/server/portal/exceptions/api.py @@ -44,7 +44,7 @@ def __init__(self, message=None, status=None, extra=None, *args, **kwargs): ... response=e.response ... ) """ - super(ApiException, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) response = self.response or Response() response.status_code = status or response.status_code response.reason = message or response.reason diff --git a/server/portal/libs/agave/exceptions.py b/server/portal/libs/agave/exceptions.py index afaba55e04..fe75d25a86 100644 --- a/server/portal/libs/agave/exceptions.py +++ b/server/portal/libs/agave/exceptions.py @@ -4,11 +4,12 @@ """ import logging + from portal.libs.exceptions import PortalLibException # pylint: disable=invalid-name logger = logging.getLogger(__name__) -METRICS = logging.getLogger("metrics.{}".format(__name__)) +METRICS = logging.getLogger(f"metrics.{__name__}") # pylint: enable=invalid-name diff --git a/server/portal/libs/agave/models/applications.py b/server/portal/libs/agave/models/applications.py index 79a71ecdd8..35341ba23f 100644 --- a/server/portal/libs/agave/models/applications.py +++ b/server/portal/libs/agave/models/applications.py @@ -3,10 +3,12 @@ :synopsis: Classes to represent Agave Applications """ -from collections import namedtuple import logging +from collections import namedtuple + from cached_property import cached_property_with_ttl -from portal.libs.agave.exceptions import ValidationError, CreationError, DeletionError, APIError + +from portal.libs.agave.exceptions import APIError, CreationError, DeletionError, ValidationError from portal.libs.agave.models.base import BaseAgaveResource from portal.libs.agave.models.permissions import ApplicationPermissions @@ -76,7 +78,7 @@ def __init__(self, client, id=None, load=True, ignore_error=404, **kwargs): wrapped.update(**kwargs) - super(Application, self).__init__(client, **wrapped) + super().__init__(client, **wrapped) self.id = getattr(self, "id", None) self.name = getattr(self, "name", None) @@ -126,7 +128,7 @@ def permissions(self): return ApplicationPermissions(self._ac, pems, self) def __str__(self): - return "{id}".format(id=self.id) + return f"{self.id}" # def __repr__(self): # return '{class_name}(id={id}, label={label})'.format( @@ -191,7 +193,7 @@ def validate_execution_type(self): """Validate self.execution_type""" types = self._EXECUTION_TYPE if self.execution_type not in types: - raise ValidationError("'execution_type' should be one of: {types}".format(types=types)) + raise ValidationError(f"'execution_type' should be one of: {types}") def validate_version(self): """Validate self.version""" @@ -212,7 +214,7 @@ def validate_parallelism(self): """Validate self.parallelism""" types = self._PARALLELISM if self.parallelism not in types: - raise ValidationError("'parallelism' should be one of: {types}".format(types=types)) + raise ValidationError(f"'parallelism' should be one of: {types}") def validate_short_description(self): """Validate self.short_description""" diff --git a/server/portal/libs/agave/models/permissions.py b/server/portal/libs/agave/models/permissions.py index 7089897d34..acf5e3dc00 100644 --- a/server/portal/libs/agave/models/permissions.py +++ b/server/portal/libs/agave/models/permissions.py @@ -4,6 +4,7 @@ """ import logging + from portal.libs.agave.exceptions import CreationError # pylint: disable=invalid-name @@ -11,7 +12,7 @@ # pylint: enable=invalid-name -class Permission(object): +class Permission: """A single permission""" READ = "READ" @@ -58,14 +59,13 @@ def to_dict(self): def __str__(self): """String -> self.username [R,W,E]""" - return "{username} {recursive}[{read}, {write}, {execute}]".format( - username=self.username, recursive=self.recursive, read=self.read, write=self.write, execute=self.execute - ) + return f"{self.username} {self.recursive}[{self.read}, {self.write}, {self.execute}]" def __repr__(self): """Repr -> Permissions(username, R, W, E)""" - return ("Permissions({username},recursive={recursive},read={read},write={write},execute={execute})").format( - username=self.username, recursive=self.recursive, read=self.read, write=self.write, execute=self.execute + return ( + f"Permissions({self.username},recursive={self.recursive},read={self.read}," + f"write={self.write},execute={self.execute})" ) def __eq__(self, other): @@ -79,7 +79,7 @@ def __eq__(self, other): ) -class Permissions(object): +class Permissions: """Permissions""" def __init__(self, client, permissions): @@ -173,7 +173,7 @@ def __init__(self, client, permissions, parent): agave's pems endpoint. :param parent: Agave File object. """ - super(FilePermissions, self).__init__(client, permissions) + super().__init__(client, permissions) self.parent = parent def save(self): @@ -198,7 +198,7 @@ def __init__(self, client, permissions, parent): agave's pems endpoint. :param parent: Agave Metadata object. """ - super(MetadataPermissions, self).__init__(client, permissions) + super().__init__(client, permissions) self.parent = parent def save(self): @@ -223,7 +223,7 @@ def __init__(self, client, permissions, parent): agave's pems endpoint. :param parent: Agave Application object. """ - super(ApplicationPermissions, self).__init__(client, permissions) + super().__init__(client, permissions) self.parent = parent def save(self): diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 48e3ffb753..650c99a0d0 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -1,19 +1,22 @@ -import os import io +import logging +import os +import urllib +from pathlib import Path from urllib.parse import quote + +import httpx from django.conf import settings from django.db import transaction -import logging -import urllib from elasticsearch_dsl import Q -import httpx -from portal.libs.elasticsearch.indexes import IndexedFile -from portal.apps.search.tasks import tapis_indexer, tapis_listing_indexer -from portal.exceptions.api import ApiException -from portal.libs.agave.utils import text_preview, get_file_size, increment_file_name -from portal.libs.agave.filter_mapping import filter_mapping -from pathlib import Path from tapipy.errors import BaseTapyException + +from portal.apps.projects.schema_models import constants +from portal.apps.projects.workspace_operations.graph_operations import ( + get_node_from_path, + get_or_create_trash_entity, + get_root_node, +) from portal.apps.projects.workspace_operations.project_meta_operations import ( add_file_associations, create_file_obj, @@ -25,13 +28,11 @@ patch_file_association, remove_file_obj_by_path, ) -from portal.apps.projects.schema_models import constants -from portal.apps.projects.workspace_operations.graph_operations import ( - get_or_create_trash_entity, - get_root_node, - get_node_from_path, -) - +from portal.apps.search.tasks import tapis_indexer, tapis_listing_indexer +from portal.exceptions.api import ApiException +from portal.libs.agave.filter_mapping import filter_mapping +from portal.libs.agave.utils import get_file_size, increment_file_name, text_preview +from portal.libs.elasticsearch.indexes import IndexedFile logger = logging.getLogger(__name__) @@ -468,9 +469,7 @@ def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, m def makepublic(client, src_system, src_path, dest_path="/", *args, **kwargs): - dest_system = next( - (sys["system"] for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["scheme"] == "public") - ) + dest_system = next(sys["system"] for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["scheme"] == "public") return copy(client, src_system, src_path, dest_system, dest_path, *args, **kwargs) @@ -663,11 +662,11 @@ def preview(client, system, path, max_uses=3, lifetime=600, **kwargs): file_type = "object" elif file_ext in settings.SUPPORTED_MS_OFFICE: file_type = "ms-office" - url = "https://view.officeapps.live.com/op/view.aspx?src={}".format(url) + url = f"https://view.officeapps.live.com/op/view.aspx?src={url}" elif file_ext in settings.SUPPORTED_IPYNB_PREVIEW_EXTS: file_type = "ipynb" tmp = url.replace("https://", "") - url = "https://nbviewer.jupyter.org/urls/{tmp}".format(tmp=tmp) + url = f"https://nbviewer.jupyter.org/urls/{tmp}" elif file_ext in settings.SUPPORTED_NEW_WINDOW_PREVIEW_EXTS: error = "This file type must be previewed in a new window." else: diff --git a/server/portal/libs/agave/operations_unit_test.py b/server/portal/libs/agave/operations_unit_test.py index 6e48dd3767..6de37a4e9f 100644 --- a/server/portal/libs/agave/operations_unit_test.py +++ b/server/portal/libs/agave/operations_unit_test.py @@ -1,10 +1,12 @@ -from mock import patch, MagicMock +from unittest.mock import MagicMock, patch + from django.test import TestCase -from tapipy.tapis import TapisResult from elasticsearch_dsl import Q from elasticsearch_dsl.response import Hit -from portal.libs.agave.operations import listing, search, mkdir, move, copy, rename, makepublic +from tapipy.tapis import TapisResult + from portal.exceptions.api import ApiException +from portal.libs.agave.operations import copy, listing, makepublic, mkdir, move, rename, search class TestOperations(TestCase): @@ -79,7 +81,7 @@ def test_search(self, mock_search, mock_listing): mock_search().query().filter.assert_called_with(~Q("query_string", query="\\/.Trash\\/", fields=["path"])) mock_search().query().filter().filter.assert_called_with("prefix", **{"path._exact": "path"}) mock_search().query().filter().filter().filter.assert_called_with("term", **{"system._exact": "test.system"}) - mock_search().query().filter().filter().filter().extra.assert_called_with(from_=int(0), size=int(100)) + mock_search().query().filter().filter().filter().extra.assert_called_with(from_=0, size=100) self.assertEqual( search_res, {"listing": [{"system": "test.system", "path": "/path/to/file"}], "reachedEnd": True, "count": 1}, diff --git a/server/portal/libs/agave/serializers.py b/server/portal/libs/agave/serializers.py index ec8a2c66e4..e869f74d87 100644 --- a/server/portal/libs/agave/serializers.py +++ b/server/portal/libs/agave/serializers.py @@ -4,8 +4,9 @@ wrapps an agave object into a dict. """ -import logging import json +import logging + from tapipy.tapis import TapisResult # pylint: disable=invalid-name diff --git a/server/portal/libs/agave/unit_test.py b/server/portal/libs/agave/unit_test.py index 98b36f88aa..54a9163299 100644 --- a/server/portal/libs/agave/unit_test.py +++ b/server/portal/libs/agave/unit_test.py @@ -3,16 +3,18 @@ :synopsis: Unit tests for Agave libraries. """ +import copy +import json import logging import os -import json -import copy -from mock import patch, call -from django.test import TestCase +from unittest.mock import call, patch + from django.conf import settings -from portal.libs.agave import utils as AgaveUtils +from django.test import TestCase from tapipy.tapis import TapisResult +from portal.libs.agave import utils as AgaveUtils + # pylint: disable=invalid-name logger = logging.getLogger(__name__) # pylint: enable=invalid-name @@ -23,7 +25,7 @@ class TestAgaveUtils(TestCase): @classmethod def setUpClass(cls): - super(TestAgaveUtils, cls).setUpClass() + super().setUpClass() cls.magave_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) cls.magave = cls.magave_patcher.start() diff --git a/server/portal/libs/agave/utils.py b/server/portal/libs/agave/utils.py index ba77bc81bf..5e5e0f6f7c 100644 --- a/server/portal/libs/agave/utils.py +++ b/server/portal/libs/agave/utils.py @@ -5,9 +5,10 @@ import logging import os + +import requests from django.conf import settings from tapipy.tapis import Tapis -import requests logger = logging.getLogger(__name__) @@ -116,8 +117,7 @@ def walk_levels(client, system, path, bottom_up=False, ignore_hidden=False): if not bottom_up: yield (path, folders, files) for child in folders: - for child_path, child_folders, child_files in walk_levels(client, system, child["path"], bottom_up=bottom_up): - yield (child_path, child_folders, child_files) + yield from walk_levels(client, system, child["path"], bottom_up=bottom_up) if bottom_up: yield (path, folders, files) @@ -168,13 +168,13 @@ def increment_file_name(listing, file_name): inc = 1 _ext = os.path.splitext(file_name)[1] _name = os.path.splitext(file_name)[0] - _inc = "({})".format(inc) - file_name = "{}{}{}".format(_name, _inc, _ext) + _inc = f"({inc})" + file_name = f"{_name}{_inc}{_ext}" while any(x.name for x in listing if x.name == file_name): inc += 1 - _inc = "({})".format(inc) - file_name = "{}{}{}".format(_name, _inc, _ext) + _inc = f"({inc})" + file_name = f"{_name}{_inc}{_ext}" return file_name diff --git a/server/portal/libs/agave/utils_unit_test.py b/server/portal/libs/agave/utils_unit_test.py index 95b46e1b02..316d2f52a0 100644 --- a/server/portal/libs/agave/utils_unit_test.py +++ b/server/portal/libs/agave/utils_unit_test.py @@ -1,7 +1,8 @@ import pytest -from portal.libs.agave.utils import text_preview, get_file_size from tapipy.tapis import TapisResult +from portal.libs.agave.utils import get_file_size, text_preview + POSTIT_URL = "https://tapis.example/postit/something" diff --git a/server/portal/libs/elasticsearch/analyzers.py b/server/portal/libs/elasticsearch/analyzers.py index 75e99857f5..f23361399c 100644 --- a/server/portal/libs/elasticsearch/analyzers.py +++ b/server/portal/libs/elasticsearch/analyzers.py @@ -4,8 +4,8 @@ """ import logging -from elasticsearch_dsl import analyzer, token_filter, tokenizer +from elasticsearch_dsl import analyzer, token_filter, tokenizer logger = logging.getLogger(__name__) diff --git a/server/portal/libs/elasticsearch/docs/base.py b/server/portal/libs/elasticsearch/docs/base.py index 2020e4228f..26d4f56459 100644 --- a/server/portal/libs/elasticsearch/docs/base.py +++ b/server/portal/libs/elasticsearch/docs/base.py @@ -3,15 +3,17 @@ :synopsis: Wrapper classes for ES different doc types. """ -import logging import datetime +import logging + from django.conf import settings from elasticsearch import Elasticsearch -from elasticsearch_dsl import Document, Date, Object, Text, Long, Boolean, Keyword +from elasticsearch_dsl import Boolean, Date, Document, Keyword, Long, Object, Text + from portal.libs.elasticsearch.analyzers import ( - path_analyzer, file_analyzer, file_pattern_analyzer, + path_analyzer, reverse_file_analyzer, ) from portal.libs.elasticsearch.utils import file_uuid_sha256, get_sha256_hash @@ -81,14 +83,14 @@ def save(self, *args, **kwargs): Sets `lastUpdated` attribute on save. Otherwise see elasticsearch_dsl.Document.save() """ self.lastUpdated = datetime.datetime.now() - return super(IndexedFile, self).save(*args, **kwargs) + return super().save(*args, **kwargs) def update(self, *args, **kwargs): """ Sets `lastUpdated` attribute on save. Otherwise see elasticsearch_dsl.Document.update() """ lastUpdated = datetime.datetime.now() - return super(IndexedFile, self).update(lastUpdated=lastUpdated, *args, **kwargs) + return super().update(lastUpdated=lastUpdated, *args, **kwargs) @classmethod def from_path(cls, system, path): diff --git a/server/portal/libs/elasticsearch/docs/unit_test.py b/server/portal/libs/elasticsearch/docs/unit_test.py index faf3d74f13..ca9bfc9092 100644 --- a/server/portal/libs/elasticsearch/docs/unit_test.py +++ b/server/portal/libs/elasticsearch/docs/unit_test.py @@ -1,6 +1,8 @@ -from mock import patch, MagicMock +from unittest.mock import MagicMock, patch + from django.test import TestCase -from portal.libs.elasticsearch.docs.base import IndexedFile, IndexedAllocation, IndexedProject + +from portal.libs.elasticsearch.docs.base import IndexedAllocation, IndexedFile, IndexedProject class TestIndexedFile(TestCase): diff --git a/server/portal/libs/elasticsearch/indexes.py b/server/portal/libs/elasticsearch/indexes.py index 34795ef411..8e829c3904 100644 --- a/server/portal/libs/elasticsearch/indexes.py +++ b/server/portal/libs/elasticsearch/indexes.py @@ -3,13 +3,14 @@ :synopsis: ElasticSearch Index setup """ -from datetime import datetime import logging +from datetime import datetime + from django.conf import settings from elasticsearch_dsl import Index -from portal.libs.elasticsearch.docs.base import IndexedFile, IndexedAllocation, IndexedProject, IndexedPublication -from portal.libs.elasticsearch.analyzers import file_query_analyzer +from portal.libs.elasticsearch.analyzers import file_query_analyzer +from portal.libs.elasticsearch.docs.base import IndexedAllocation, IndexedFile, IndexedProject, IndexedPublication logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ def setup_indexes(doc_type, reindex=False, force=False): index with that alias and the provided name. """ baseName = settings.ES_INDEX_PREFIX.format(doc_type) - indexName = "{}-{}".format(baseName, index_time_string()) + indexName = f"{baseName}-{index_time_string()}" alias = baseName if reindex: alias += "-reindex" diff --git a/server/portal/libs/elasticsearch/unit_test.py b/server/portal/libs/elasticsearch/unit_test.py index 3427a62a3c..305929fd78 100644 --- a/server/portal/libs/elasticsearch/unit_test.py +++ b/server/portal/libs/elasticsearch/unit_test.py @@ -1,16 +1,17 @@ -from mock import patch, call +from unittest.mock import call, patch + from django.test import TestCase from elasticsearch_dsl import Q from elasticsearch_dsl.response.hit import Hit -from portal.libs.elasticsearch.indexes import setup_files_index, setup_projects_index, setup_indexes +from portal.libs.elasticsearch.indexes import setup_files_index, setup_indexes, setup_projects_index from portal.libs.elasticsearch.utils import ( - index_listing, - index_level, + delete_recursive, file_uuid_sha256, - walk_children, grouper, - delete_recursive, + index_level, + index_listing, + walk_children, ) diff --git a/server/portal/libs/elasticsearch/utils.py b/server/portal/libs/elasticsearch/utils.py index 5db76a63f9..a517b78251 100644 --- a/server/portal/libs/elasticsearch/utils.py +++ b/server/portal/libs/elasticsearch/utils.py @@ -2,14 +2,16 @@ Utilities to help on elastic search implementations. """ -import os -import logging import datetime +import logging +import os +from hashlib import sha256 +from itertools import zip_longest + from elasticsearch.helpers import bulk from elasticsearch_dsl import Q from elasticsearch_dsl.connections import get_connection -from hashlib import sha256 -from itertools import zip_longest + # from portal.apps.projects.models import ProjectMetadata # pylint: disable=invalid-name @@ -60,7 +62,7 @@ def file_uuid_sha256(system, path): """ if not path.startswith("/"): - path = "/{}".format(path) + path = f"/{path}" # str representation of the hash of e.g. "cep.home.user/path/to/file" return sha256((system + path).encode()).hexdigest() @@ -111,8 +113,7 @@ def walk_children(system, path, include_parent=False, recurse=False): else: search = search.filter(basepath_query) - for hit in search.scan(): - yield hit + yield from search.scan() def delete_recursive(system, path): diff --git a/server/portal/libs/exceptions.py b/server/portal/libs/exceptions.py index 5df5895c80..f7768bae7f 100644 --- a/server/portal/libs/exceptions.py +++ b/server/portal/libs/exceptions.py @@ -7,7 +7,7 @@ # pylint: disable=invalid-name logger = logging.getLogger(__name__) -METRICS = logging.getLogger("metrics.{}".format(__name__)) +METRICS = logging.getLogger(f"metrics.{__name__}") # pylint: enable=invalid-name diff --git a/server/portal/libs/files/file_processing.py b/server/portal/libs/files/file_processing.py index d2635f30a8..4927797a6a 100644 --- a/server/portal/libs/files/file_processing.py +++ b/server/portal/libs/files/file_processing.py @@ -1,12 +1,13 @@ -import numpy as np +import csv import io -import os import logging -from matplotlib import pyplot as plt -import csv -import matplotlib.animation as anim +import os import tempfile + +import matplotlib.animation as anim +import numpy as np import tifffile as tiff +from matplotlib import pyplot as plt from PIL import Image logger = logging.getLogger(__name__) diff --git a/server/portal/libs/googledrive/files.py b/server/portal/libs/googledrive/files.py index e8d324b3c8..8d3a978859 100644 --- a/server/portal/libs/googledrive/files.py +++ b/server/portal/libs/googledrive/files.py @@ -3,7 +3,7 @@ logger = logging.getLogger(__name__) -class GoogleDriveFile(object): +class GoogleDriveFile: """Represents a google drive file""" SUPPORTED_IMAGE_PREVIEW_EXTS = [ @@ -147,12 +147,12 @@ def name(self): def path(self): if self._parent: if self._parent.name == "My Drive": - path = "/{}".format(self.name) + path = f"/{self.name}" else: path = "/".join([self._parent.path, self.name]) elif "parents" in self._item: parent = self - path = "{}".format(self.name) + path = f"{self.name}" while True: try: self._path_collection.insert( @@ -164,7 +164,7 @@ def path(self): .execute() ) parent_name = "" if parent.name == "My Drive" else parent.name - path = "{}/{}".format(parent_name, path) + path = f"{parent_name}/{path}" except (AttributeError, KeyError): break else: diff --git a/server/portal/libs/googledrive/operations.py b/server/portal/libs/googledrive/operations.py index 575b5dad85..67edfd20cc 100644 --- a/server/portal/libs/googledrive/operations.py +++ b/server/portal/libs/googledrive/operations.py @@ -1,9 +1,9 @@ -import os import io -import magic import logging -from googleapiclient.http import MediaIoBaseUpload, MediaIoBaseDownload +import os +import magic +from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload logger = logging.getLogger(__name__) @@ -15,8 +15,8 @@ def listing(client, system, path, offset=None, limit=100, nextPageToken=None, *a listing_call = ( client.files() .list( - q="'{}' in parents and trashed=False".format(path), - fields="files({}), nextPageToken".format(fields), + q=f"'{path}' in parents and trashed=False", + fields=f"files({fields}), nextPageToken", pageSize=limit, pageToken=nextPageToken, ) @@ -53,10 +53,8 @@ def search(client, system, path, offset=None, limit=100, nextPageToken=None, que listing_call = ( client.files() .list( - q=("'{path}' in parents and trashed=False and name contains '{query_string}'").format( - path=path, query_string=query_string - ), - fields="files({}), nextPageToken".format(fields), + q=(f"'{path}' in parents and trashed=False and name contains '{query_string}'"), + fields=f"files({fields}), nextPageToken", pageSize=limit, pageToken=nextPageToken, ) diff --git a/server/portal/libs/googledrive/operations_unit_test.py b/server/portal/libs/googledrive/operations_unit_test.py index db1979ada1..be8d37b3a6 100644 --- a/server/portal/libs/googledrive/operations_unit_test.py +++ b/server/portal/libs/googledrive/operations_unit_test.py @@ -1,6 +1,7 @@ -import pytest import io -from mock import MagicMock +from unittest.mock import MagicMock + +import pytest @pytest.fixture diff --git a/server/portal/libs/transfer/operations_unit_test.py b/server/portal/libs/transfer/operations_unit_test.py index b4cd6f36b3..03f3f1699c 100644 --- a/server/portal/libs/transfer/operations_unit_test.py +++ b/server/portal/libs/transfer/operations_unit_test.py @@ -1,6 +1,7 @@ -import pytest import io -from mock import MagicMock, call +from unittest.mock import MagicMock, call + +import pytest @pytest.fixture diff --git a/server/portal/middleware.py b/server/portal/middleware.py index eb3e2864e0..5c342ba503 100644 --- a/server/portal/middleware.py +++ b/server/portal/middleware.py @@ -1,8 +1,9 @@ -from django.contrib import messages +import logging + from django.conf import settings -from termsandconditions.models import TermsAndConditions +from django.contrib import messages from termsandconditions.middleware import TermsAndConditionsRedirectMiddleware, is_path_protected -import logging +from termsandconditions.models import TermsAndConditions logger = logging.getLogger(__name__) @@ -28,9 +29,9 @@ def process_request(self, request): request, "

Please Accept the Terms of Use

" "You have not yet agreed to the current Terms of Use. " - 'Please CLICK HERE to review and ' + f'Please CLICK HERE to review and ' "accept the Terms of Use.
Acceptance of the Terms of " "Use is required for continued use of the portal " - "resources." % accept_url, + "resources.", ) return None diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index c607f6919e..5d0b90c498 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -10,11 +10,13 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ """ +import logging import os import uuid -import logging from hashlib import sha256 + from kombu import Exchange, Queue + from portal.settings import settings_secret logger = logging.getLogger(__file__) @@ -318,7 +320,8 @@ def portal_filter(record): "metrics": { "format": "[METRICS] %(levelname)s %(module)s %(name)s.%(funcName)s:%(lineno)s:" " %(message)s user=%(user)s ip=%(ip)s agent=%(agent)s sessionId=%(sessionId)s op=%(operation)s" - " info=%(info)s timestamp=%(asctime)s trackingId=portals.%(sessionId)s guid=%(logGuid)s portal=%(portal)s tenant=%(tenant)s" + " info=%(info)s timestamp=%(asctime)s trackingId=portals.%(sessionId)s guid=%(logGuid)s" + " portal=%(portal)s tenant=%(tenant)s" }, }, "handlers": { diff --git a/server/portal/settings/unit_test_settings.py b/server/portal/settings/unit_test_settings.py index 3e01e885ee..5910b5f300 100644 --- a/server/portal/settings/unit_test_settings.py +++ b/server/portal/settings/unit_test_settings.py @@ -12,7 +12,6 @@ import os - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/server/portal/urls.py b/server/portal/urls.py index cf0d26d509..0c2473b924 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -18,17 +18,17 @@ """ from django.conf import settings -from django.contrib import admin from django.conf.urls.static import static +from django.contrib import admin from django.contrib.staticfiles.urls import staticfiles_urlpatterns -from portal.apps.auth.views import tapis_oauth as login -from portal.views.views import project_version as portal_version +from django.urls import include, path, re_path from django.views.generic import RedirectView from django.views.generic.base import TemplateView -from django.urls import path, re_path, include from impersonate import views as impersonate_views -from portal.views.views import health_check -from portal.views.views import serve_docs + +from portal.apps.auth.views import tapis_oauth as login +from portal.views.views import health_check, serve_docs +from portal.views.views import project_version as portal_version admin.autodiscover() diff --git a/server/portal/utils/decorators.py b/server/portal/utils/decorators.py index f5246c1d51..d071b280b3 100644 --- a/server/portal/utils/decorators.py +++ b/server/portal/utils/decorators.py @@ -6,9 +6,10 @@ import logging import time from functools import wraps + from django.http import JsonResponse -from portal.utils.jwt_auth import login_user_agave_jwt +from portal.utils.jwt_auth import login_user_agave_jwt logger = logging.getLogger(__name__) diff --git a/server/portal/utils/encryption.py b/server/portal/utils/encryption.py index 6e61b5f70e..f7c51286e0 100644 --- a/server/portal/utils/encryption.py +++ b/server/portal/utils/encryption.py @@ -3,12 +3,13 @@ :synopsis: Utilities to handle encryption and ssh keys """ -import logging import base64 -from Crypto.PublicKey import RSA +import logging + +from Crypto import Random from Crypto.Cipher import AES from Crypto.Hash import SHA256 -from Crypto import Random +from Crypto.PublicKey import RSA from django.conf import settings # pylint: disable=invalid-name diff --git a/server/portal/utils/exceptions.py b/server/portal/utils/exceptions.py index fb73a3b0a9..1c840dca2a 100644 --- a/server/portal/utils/exceptions.py +++ b/server/portal/utils/exceptions.py @@ -33,7 +33,7 @@ class PortalException(RequestException): """ def __init__(self, message=None, status=None, extra=None, *args, **kwargs): - super(PortalException, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) response = self.response or Response() response.status_code = status or response.status_code response.reason = message or response.reason @@ -46,4 +46,4 @@ class ApiMethodNotAllowed(PortalException): """Custom 405 Method Not Allowed Exception""" def __init__(self, extra=None, *args, **kwargs): - super(ApiMethodNotAllowed, self).__init__(message="Method Not Allowed", status=405, extra=extra) + super().__init__(message="Method Not Allowed", status=405, extra=extra) diff --git a/server/portal/utils/fields.py b/server/portal/utils/fields.py index 4645da92c3..8cf9bb130e 100644 --- a/server/portal/utils/fields.py +++ b/server/portal/utils/fields.py @@ -4,7 +4,6 @@ from django.contrib.postgres.fields import JSONField as DjangoJSONField from django.db.models import Field - # NOTE: TODO_django: Django v3 introduces a native JSONField, so this file will be obsolete. # https://docs.djangoproject.com/en/3.2/ref/models/fields/#django.db.models.JSONField diff --git a/server/portal/utils/jwt_auth.py b/server/portal/utils/jwt_auth.py index 32b35802e0..b454ff7a68 100644 --- a/server/portal/utils/jwt_auth.py +++ b/server/portal/utils/jwt_auth.py @@ -5,15 +5,14 @@ import logging from base64 import b64decode -from six import text_type -from django.conf import settings -from django.contrib.auth import get_user_model -from django.contrib.auth import login -from django.core.exceptions import ObjectDoesNotExist + import jwt as pyjwt -from cryptography.hazmat.primitives.serialization import load_der_public_key from cryptography.exceptions import UnsupportedAlgorithm - +from cryptography.hazmat.primitives.serialization import load_der_public_key +from django.conf import settings +from django.contrib.auth import get_user_model, login +from django.core.exceptions import ObjectDoesNotExist +from six import text_type LOGGER = logging.getLogger(__name__) diff --git a/server/portal/utils/translations.py b/server/portal/utils/translations.py index 57f8be959a..0a649b4dae 100644 --- a/server/portal/utils/translations.py +++ b/server/portal/utils/translations.py @@ -1,10 +1,9 @@ +import copy import logging -import urllib.request -import urllib.parse import urllib.error +import urllib.parse +import urllib.request from urllib.parse import urlparse -import copy - LOGGER = logging.getLogger(__name__) @@ -19,7 +18,7 @@ def url_parse_inputs(job): if isinstance(value, str): parsed = urlparse(value) if parsed.scheme: - job["inputs"][key] = "{}://{}{}".format(parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) + job["inputs"][key] = f"{parsed.scheme}://{parsed.netloc}{urllib.parse.quote(parsed.path)}" else: job["inputs"][key] = urllib.parse.quote(parsed.path) else: @@ -27,7 +26,7 @@ def url_parse_inputs(job): parsed_values = [] for input in value: parsed = urlparse(input) - input = "{}://{}{}".format(parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) + input = f"{parsed.scheme}://{parsed.netloc}{urllib.parse.quote(parsed.path)}" parsed_values.append(input) job["inputs"][key] = parsed_values return job diff --git a/server/portal/utils/unit_test.py b/server/portal/utils/unit_test.py index 5747537030..c7aa4c0dc2 100644 --- a/server/portal/utils/unit_test.py +++ b/server/portal/utils/unit_test.py @@ -1,11 +1,13 @@ -from django.test import TestCase, override_settings -from mock import patch, Mock -from django.contrib.auth.models import Group +from datetime import timedelta +from unittest.mock import Mock, patch + from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.test import TestCase, override_settings + from portal.utils import check_group_membership -from portal.utils.translations import url_parse_inputs from portal.utils.jwt_auth import login_user_agave_jwt -from datetime import timedelta +from portal.utils.translations import url_parse_inputs class TestTranslations(TestCase): @@ -15,7 +17,7 @@ class TestTranslations(TestCase): def setUp(self): """Setup.""" - super(TestTranslations, self).setUp() + super().setUp() self.user = get_user_model().objects.get(username="username") self.job = { "inputs": { diff --git a/server/portal/views/base.py b/server/portal/views/base.py index 30d849c3e1..aecac74c0c 100644 --- a/server/portal/views/base.py +++ b/server/portal/views/base.py @@ -4,13 +4,15 @@ """ import logging -from requests.exceptions import ConnectionError, HTTPError -from django.views.generic import View -from django.http import JsonResponse, Http404 + from django.core.exceptions import PermissionDenied -from portal.exceptions.api import ApiException +from django.http import Http404, JsonResponse +from django.views.generic import View +from requests.exceptions import ConnectionError, HTTPError from tapipy.errors import BaseTapyException +from portal.exceptions.api import ApiException + logger = logging.getLogger(__name__) @@ -26,7 +28,7 @@ def dispatch(self, request, *args, **kwargs): to be available in the logs and in Opbeat's UI. """ try: - return super(BaseApiView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) except (PermissionDenied, Http404) as e: # log information but re-raise exception to let django handle response logger.error(e, exc_info=True) diff --git a/server/portal/views/unit_test.py b/server/portal/views/unit_test.py index a9b120704a..ef19578d9b 100644 --- a/server/portal/views/unit_test.py +++ b/server/portal/views/unit_test.py @@ -1,11 +1,12 @@ +import json + import pytest -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +import requests +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.http import Http404 + from portal.exceptions.api import ApiException from portal.libs.exceptions import PortalLibException -import requests -import json - # route to be used for testing purposes API_ROUTE = "/api/system-monitor/" @@ -83,7 +84,7 @@ def test_connectionerror_httperror_with_response(ExceptionClass, status_code, cl @pytest.mark.parametrize("status_code", [403, 404, NON_403_404]) def test_connectionerror_httperror_non_json_content(ExceptionClass, status_code, client, api_method_mock): test_response = requests.Response() - test_response._content = "Non json error content".encode("utf-8") + test_response._content = b"Non json error content" test_response.status_code = status_code api_method_mock.side_effect = requests.exceptions.HTTPError(response=test_response) diff --git a/server/portal/views/views.py b/server/portal/views/views.py index c5643b08eb..c4d0083a60 100644 --- a/server/portal/views/views.py +++ b/server/portal/views/views.py @@ -1,9 +1,10 @@ import logging import os -from django.views.static import serve + from django.conf import settings from django.contrib.auth.decorators import login_required -from django.http import HttpResponse, JsonResponse, Http404 +from django.http import Http404, HttpResponse, JsonResponse +from django.views.static import serve logger = logging.getLogger(__name__) @@ -16,13 +17,13 @@ def project_version(request): if "ref:" in head: # we're on a branch branch = head.split(":")[1].strip() - with open(".git/{0}".format(branch)) as f: - version = "{}:{}".format(branch, f.readline()) + with open(f".git/{branch}") as f: + version = f"{branch}:{f.readline()}" else: # we're in a detached head, e.g., a tag. would be nice to show tag name... version = head - except IOError: + except OSError: logger.warning("Unable to read project version from git HEAD") version = "UNKNOWN" diff --git a/server/portal/wsgi.py b/server/portal/wsgi.py index 6d1f9a340c..355e780a79 100644 --- a/server/portal/wsgi.py +++ b/server/portal/wsgi.py @@ -8,6 +8,7 @@ """ import os + from django.core.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings")