diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3deab97a95..401b02367d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: pipx install poetry - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 cache: "poetry" @@ -57,7 +57,7 @@ jobs: pipx install poetry - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 cache: "poetry" @@ -67,10 +67,15 @@ jobs: cd server poetry install --with dev - - name: Server-Side Linting + # - name: Run Ruff check + # run: | + # cd server + # poetry run ruff check --output-format=github . + + - name: Run Ruff format check run: | cd server - poetry run flake8 + poetry run ruff format --check . Client_Side_Unit_Tests: runs-on: ubuntu-24.04 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..7128db8a57 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.5 + hooks: + # - id: ruff-check + # args: [--fix] + # files: ^server/ + - id: ruff-format + files: ^server/ diff --git a/README.md b/README.md index 0f57af8cd9..c39e83f2a6 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ The base Portal code for TACC WMA Workspace Portals ### Related Repositories: + - [Camino], a Docker container-based deployment scheme - [Core CMS], the base CMS code for TACC WMA CMS Websites - [Core Styles], the shared UI pattern code for TACC WMA CMS Websites @@ -12,9 +13,9 @@ The base Portal code for TACC WMA Workspace Portals ## Prerequisites for running the portal application: -* Docker > 28 -* Python 3.12.x -* Nodejs 24.x (LTS) +- Docker > 28 +- Python 3.12.x +- Nodejs 24.x (LTS) The Core Portal can be run using [Docker][1]. @@ -36,7 +37,7 @@ NOTE: This may require a computer restart to take effect. 3. Navigate to `./server/conf/nginx/certificates` 4. Select `ca.pem` 5. Under the "All" or "Certificates" tab,\ - Search for CEP and double click on the certificate + Search for CEP and double click on the certificate 6. In the Trust section, find the "When using this certificate" dropdown and select "Always Trust" 7. Close the window to save. @@ -50,55 +51,55 @@ NOTE: This may require a computer restart to take effect. #### Firefox UI 1. Go to preferences -3. Search for Authorities -4. Click on "View Certificates" under "Certificates" -5. On the Certificate Manager go to the "Authorities" tab -6. Click on "Import..." -7. Browse to `./server/conf/nginx/certificates` -8. Select `ca.pem` +2. Search for Authorities +3. Click on "View Certificates" under "Certificates" +4. On the Certificate Manager go to the "Authorities" tab +5. Click on "Import..." +6. Browse to `./server/conf/nginx/certificates` +7. Select `ca.pem` #### Firefox CLI (not tested) 1. `sudo apt-get install libnss3-tools` (or proper package manager) 2. `certutil -A -n "cepCA" -t "TCu,Cu,Tu" -i ca.pem -d ${DBDIR}` 3. `$DBDIR` differs from browser to browser for more info: - Chromium: https://chromium.googlesource.com/chromium/src/+/master/docs/linux_cert_management.md - Firefox: https://support.mozilla.org/en-US/kb/profiles-where-firefox-stores-user-data?redirectlocale=en-US&redirectslug=Profiles#How_to_find_your_profile + Chromium: https://chromium.googlesource.com/chromium/src/+/master/docs/linux_cert_management.md + Firefox: https://support.mozilla.org/en-US/kb/profiles-where-firefox-stores-user-data?redirectlocale=en-US&redirectslug=Profiles#How_to_find_your_profile ### Generating new Certs for Local development. + This operation should be done yearly when local certs expire, and the results committed to the repo. Typically the local CA should NOT be re=generated and added to your keychain, since we mint them to last several years. From the `server/conf/nginx/certificates` directory: + 1. `openssl req -newkey rsa:2048 -noenc -keyout cep.test.key -out cep.test.csr` 2. `openssl x509 -req -CA ca.pem -CAkey ca.key -in cep.test.csr -out cep.test.crt -days 365 -CAcreateserial -extfile cep.test.ext` -After this you will need to restart the Nginx container if it is running. + After this you will need to restart the Nginx container if it is running. ### Setup local access to the portal: - 1. Add a record to your local `hosts` file for `127.0.0.1 cep.test` - - `sudo vim /etc/hosts` +1. Add a record to your local `hosts` file for `127.0.0.1 cep.test` + - `sudo vim /etc/hosts` - 2. Do this step after going through the server and client code configuration steps in next section. +2. Do this step after going through the server and client code configuration steps in next section. - Direct your browser to `https://cep.test`. This will display the django CMS default page. To login to the portal, point your browser to `https://cep.test/login`. + Direct your browser to `https://cep.test`. This will display the django CMS default page. To login to the portal, point your browser to `https://cep.test/login`. - _NOTE: If when navigating to `https://cep.test` you see a "Server not found" error while on the VPN, follow these steps and try again:_ - 1. Open the Network app utility - 2. Select network connection you’re on (wifi, ethernet, etc) - 3. Go to “Advanced” - 4. Go to “TCP/IP” tab - 5. Under “Configure IPv6” dropdown, select “Link-local only” - 6. Hit “OK” - 7. Hit “Apply” + _NOTE: If when navigating to `https://cep.test` you see a "Server not found" error while on the VPN, follow these steps and try again:_ + 1. Open the Network app utility + 2. Select network connection you’re on (wifi, ethernet, etc) + 3. Go to “Advanced” + 4. Go to “TCP/IP” tab + 5. Under “Configure IPv6” dropdown, select “Link-local only” + 6. Hit “OK” + 7. Hit “Apply” - _NOTE: When logging in, make sure that you are going through SSL (`https://cep.test/login`). After succesful login, you can use the debug server at `https://cep.test`._ - - _NOTE: Evergreen browsers will no longer allow self-signed certificates. Currently Chrome and Firefox deny access to the local portal for this reason. A cert solution needs to be established in alignment with current TACC policies to resolve this._ + _NOTE: When logging in, make sure that you are going through SSL (`https://cep.test/login`). After succesful login, you can use the debug server at `https://cep.test`._ + _NOTE: Evergreen browsers will no longer allow self-signed certificates. Currently Chrome and Firefox deny access to the local portal for this reason. A cert solution needs to be established in alignment with current TACC policies to resolve this._ ### Code Configuration After you clone the repository locally, there are several configuration steps required to prepare the project. - #### Create settings and secrets ##### Portal @@ -106,7 +107,7 @@ After you clone the repository locally, there are several configuration steps re - Create `server/portal/settings/settings_secret.py` containing what is in `secret` field in the `Core Portal Settings Secret` entry secured on [UT Stache](https://stache.utexas.edu/entry/bedc97190d3a907cb44488785440595c) - Copy `server/conf/env_files/ngrok.sample.env` to `server/conf/env_files/ngrok.env` - - _Note: [Setup ngrok](#setting-up-notifications-locally) and update `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` in `ngrok.env` to enable webhook notifications locally_ + - _Note: [Setup ngrok](#setting-up-notifications-locally) and update `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` in `ngrok.env` to enable webhook notifications locally_ ##### CMS @@ -117,28 +118,60 @@ After you clone the repository locally, there are several configuration steps re - To override any standard or custom CMS settings, create a `server/conf/cms/settings_local.py` #### Build the image for the portal's django container: + make build + OR docker compose -f ./server/conf/docker/docker-compose.yml build - #### Start the development environment: + make start + OR docker compose -f ./server/conf/docker/docker-compose-dev.all.debug.yml up - #### Install client-side dependencies and bundle code: cd client npm ci npm run build -- _Notes: During local development you can also use `npm run dev` to set a live reload watch on your local system that will update the portal code in real-time. Again, make sure that you are using NodeJS LTS and not an earlier version. You will also need the port 3000 available locally._ +- _Notes: During local development you can also use `npm run dev` to set a live reload watch on your local system that will update the portal code in real-time. Again, make sure that you are using NodeJS LTS and not an earlier version. You will also need the port 3000 available locally._ + +- _Notes: If your settings.DEBUG is set to true, you will have to use `npm run dev` to have a functional app. In DEBUG setting, the requests are handled via [vite][2]._ + +#### Set up pre-commit hooks: + +Install `pre-commit` on macOS via Homebrew (bash) or script: + +```bash +brew install pre-commit +# or +curl https://pre-commit.com/install-local.py | python3 - +``` + +Or install via `pip` or `pipx`: + +```bash +pip install pre-commit +# or +pipx install pre-commit +``` -- _Notes: If your settings.DEBUG is set to true, you will have to use `npm run dev` to have a functional app. In DEBUG setting, the requests are handled via [vite][2]._ +Enable `pre-commit` in this repository to run hooks automatically on commit: + +```bash +pre-commit install +``` + +Optionally, run `pre-commit` against all files in the repo: + +```bash +pre-commit run --all-files +``` #### Initialize the application in the `core_portal_django` container: @@ -157,19 +190,20 @@ OR You may optionally create sample pages in the CMS at https://cep.test/. -*NOTE*: TACC VPN or physical connection to the TACC network is required to log-in to CMS using LDAP, otherwise the password set with `python3 manage.py createsuperuser` is used +_NOTE_: TACC VPN or physical connection to the TACC network is required to log-in to CMS using LDAP, otherwise the password set with `python3 manage.py createsuperuser` is used ### Setting up search index: Requirements: + - At least one page in CMS (see above). - At least [15% of free disk space](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html). - For Mac/Windows - - At least 4GB of RAM allocated to Docker (see Docker Desktop > Settings > Resources > Advanced). + - At least 4GB of RAM allocated to Docker (see Docker Desktop > Settings > Resources > Advanced). - For Linux (Locally) - - Run `sudo sysctl -w vm.max_map_count=2146999999` (The minimum required by [ES](https://www.elastic.co/guide/en/elasticsearch/reference/master/_maximum_map_count_check.html) is 262144 but it doesn't seem to work). - - Run `sudo sysctl -w vm.overcommit_memory=1`. - - Run `sudo sysctl -p` (In order to persist in `/etc/sysctl.conf`). + - Run `sudo sysctl -w vm.max_map_count=2146999999` (The minimum required by [ES](https://www.elastic.co/guide/en/elasticsearch/reference/master/_maximum_map_count_check.html) is 262144 but it doesn't seem to work). + - Run `sudo sysctl -w vm.overcommit_memory=1`. + - Run `sudo sysctl -p` (In order to persist in `/etc/sysctl.conf`). First, rebuild the cms search index: @@ -182,6 +216,7 @@ Then, use the django shell in the `core_portal_django` container— python3 manage.py shell —to run the following code to set up the search index: + ``` from portal.libs.elasticsearch.indexes import setup_files_index, setup_projects_index, setup_allocations_index setup_files_index(force=True) @@ -202,8 +237,8 @@ setup_allocations_index(force=True) ``` ngrok http 443 ``` -3. Then, take the `https` url generated by ngrok and paste it into the `WH_BASE_URL` setting in `settings_local.py` +3. Then, take the `https` url generated by ngrok and paste it into the `WH_BASE_URL` setting in `settings_local.py` ### Linting and Formatting Conventions @@ -211,27 +246,36 @@ Client-side code is linted (JavaScript via `eslint`, CSS via `stylelint`), and i 1. Navigate to `client/` directory. 1. Run `npm run lint`, which is the same as linting both languages independently: - - `npm run lint:js` - - `npm run lint:css` - - `npm run prettier:check` + - `npm run lint:js` + - `npm run lint:css` + - `npm run prettier:check` You may auto-fix your linting errors to conform with configured standards, for specific languages, via: + - `npm run lint:js -- --fix` - `npm run lint:css -- --fix` - `npm run prettier:fix` -Server-side Python code is linted via Flake8, and is also enforced on commits to the repo. To see server side linting errors, run `flake8` from the command line. +Server-side Python code is formatted and linted via Ruff (configured via `.pre-commit-config.yaml` and `pyproject.toml`), and is enforced on commits to the repo. To check for server-side linting and formatting errors, run `ruff check` and `ruff format --check` from the command line. To do so, run the following in the `core_portal_django` container: +```bash +ruff check . +ruff format --check . ``` -flake8 + +To automatically fix linting errors and format code: + +```bash +ruff check --fix . +ruff format . ``` ### Testing Server-side python testing is run through pytest. Start docker container first by `docker exec -it core_portal_django bash`, Then run `pytest -ra` from the `server` folder to run backend tests and display a report at the bottom of the output. -Client-side javascript testing is run through Jest. Run `npm run test`* from the `client` folder to ensure tests are running correctly. +Client-side javascript testing is run through Jest. Run `npm run test`\* from the `client` folder to ensure tests are running correctly. \* To run tests without console logging, run `npm run test -- --silent`. @@ -239,7 +283,6 @@ Client-side javascript testing is run through Jest. Run `npm run test`* from the Coverage is sent to codecov on commits to the repo (see Github Actions for branch to see branch coverage). Ideally we only merge positive code coverage changes to `main`. - #### Production Deployment The Core Portal runs in a Docker container as part of a set of services managed with Docker Compose. @@ -259,41 +302,44 @@ Deployments are initiated via [Jenkins](https://jenkins01.tacc.utexas.edu/view/W ### Contributing #### Development Workflow + We use a modifed version of [GitFlow](https://datasift.github.io/gitflow/IntroducingGitFlow.html) as our development workflow. Our [development site](https://dev.cep.tacc.utexas.edu) (accessible behind the TACC Network) is always up-to-date with `main`, while the [production site](https://prod.cep.tacc.utexas.edu) is built to a hashed commit tag. + - "Feature branches" contain major and minor updates, bug fixes and hot fixes, and other changes with respective branch prefixes: - - `feat/` for features and updates - - `fix/` for bugfixes and hotfixes - - `refactor/` for large internal changes - - `style/` for code style changes (white-space, formatting, etc.) - - `chore/` for no-op changes - - `docs/` for documentation - - `perf/` for performance improvements - - `test/` for test case updates - - or other "types" from [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/#summary) + - `feat/` for features and updates + - `fix/` for bugfixes and hotfixes + - `refactor/` for large internal changes + - `style/` for code style changes (white-space, formatting, etc.) + - `chore/` for no-op changes + - `docs/` for documentation + - `perf/` for performance improvements + - `test/` for test case updates + - or other "types" from [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/#summary) #### Testing Core Styles Changes Locally 1. Clone [Core Styles] (if you haven't already). 2. Tell project to temporarily use your [Core Styles] clone: - ```bash - npm link path-to/Core-Styles # e.g. npm link ../../Core-Styles - ``` + + ```bash + npm link path-to/Core-Styles # e.g. npm link ../../Core-Styles + ``` 3. Make changes in your [Core Styles] clone as necessary. 4. Test changes. - - Changes to imported files during `npm run dev` will trigger livereload. + - Changes to imported files during `npm run dev` will trigger livereload. 5. Commit successful changes to a [Core Styles] branch. - _Note: [If you run `npm install` or `npm ci`, the link is destroyed.](https://github.com/npm/cli/issues/2380#issuecomment-1029967927) Repeat the above steps to restore it._ #### Best Practices + Sign your commits ([see this link](https://help.github.com/en/github/authenticating-to-github/managing-commit-signature-verification) for help) ### Resources -* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo) -* [Tapis Project (Formerly Agave)](https://tacc-cloud.readthedocs.io/projects/agave/en/latest/) - +- [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo) +- [Tapis Project (Formerly Agave)](https://tacc-cloud.readthedocs.io/projects/agave/en/latest/) diff --git a/server/.flake8 b/server/.flake8 deleted file mode 100644 index 09fb230f86..0000000000 --- a/server/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -max-line-length = 160 -per-file-ignores = - # line too long on existing/generated migrations - */migrations/*.py: E501 diff --git a/server/.github/dependabot.yml b/server/.github/dependabot.yml new file mode 100644 index 0000000000..ac7e39086b --- /dev/null +++ b/server/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "weekly" diff --git a/server/conf/cms/secrets.sample.py b/server/conf/cms/secrets.sample.py index 5c1d470e6a..134a7a7607 100644 --- a/server/conf/cms/secrets.sample.py +++ b/server/conf/cms/secrets.sample.py @@ -2,25 +2,25 @@ # DJANGO SETTINGS ######################## -SECRET_KEY = 'replacethiswithareallysecureandcomplexsecretkeystring' -LOGIN_REDIRECT_URL = '/workbench/dashboard/' +SECRET_KEY = "replacethiswithareallysecureandcomplexsecretkeystring" +LOGIN_REDIRECT_URL = "/workbench/dashboard/" ######################## # ELASTICSEARCH ######################## -ES_AUTH = 'username:password' -ES_HOSTS = 'http://elasticsearch:9200' -ES_INDEX_PREFIX = 'cep-dev-{}' -ES_DOMAIN = 'https://cep.test' +ES_AUTH = "username:password" +ES_HOSTS = "http://elasticsearch:9200" +ES_INDEX_PREFIX = "cep-dev-{}" +ES_DOMAIN = "https://cep.test" -es_engine = 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine' +es_engine = "haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine" HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': es_engine, - 'URL': ES_HOSTS, - 'INDEX_NAME': ES_INDEX_PREFIX.format('cms'), - 'KWARGS': {'http_auth': ES_AUTH} + "default": { + "ENGINE": es_engine, + "URL": ES_HOSTS, + "INDEX_NAME": ES_INDEX_PREFIX.format("cms"), + "KWARGS": {"http_auth": ES_AUTH}, } } @@ -28,16 +28,16 @@ # RECAPTCHA SETTINGS ######################## -RECAPTCHA_PUBLIC_KEY = '' -RECAPTCHA_PRIVATE_KEY = '' -SILENCED_SYSTEM_CHECKS = ['captcha.recaptcha_test_key_error'] +RECAPTCHA_PUBLIC_KEY = "" +RECAPTCHA_PRIVATE_KEY = "" +SILENCED_SYSTEM_CHECKS = ["captcha.recaptcha_test_key_error"] ######################## # REDMINE TRACKER AUTH ######################## -RT_HOST = '' -RT_UN = '' -RT_PW = '' -RT_QUEUE = '' -RT_TAG = '' +RT_HOST = "" +RT_UN = "" +RT_PW = "" +RT_QUEUE = "" +RT_TAG = "" diff --git a/server/conftest.py b/server/conftest.py index c27ca68ca4..0e16506e89 100644 --- a/server/conftest.py +++ b/server/conftest.py @@ -10,7 +10,7 @@ @pytest.fixture def mock_tapis_client(mocker): - yield mocker.patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) + yield mocker.patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) @pytest.fixture @@ -20,54 +20,47 @@ def mock_googledrive_client(mocker): @pytest.fixture def regular_user(django_user_model, django_db_reset_sequences, mock_tapis_client): - django_user_model.objects.create_user(username="username", - password="password", - first_name="Firstname", - last_name="Lastname", - email="user@user.com") + django_user_model.objects.create_user( + username="username", password="password", first_name="Firstname", last_name="Lastname", email="user@user.com" + ) user = django_user_model.objects.get(username="username") TapisOAuthToken.objects.create( - user=user, - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user=user, access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447 + ) PortalProfile.objects.create(user=user) yield user @pytest.fixture def regular_user2(django_user_model, django_db_reset_sequences, mock_tapis_client): - django_user_model.objects.create_user(username="username2", - password="password", - first_name="Firstname2", - last_name="Lastname2", - email="user2@user.com") + django_user_model.objects.create_user( + username="username2", + password="password", + first_name="Firstname2", + last_name="Lastname2", + email="user2@user.com", + ) user = django_user_model.objects.get(username="username2") TapisOAuthToken.objects.create( - user=user, - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user=user, access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447 + ) PortalProfile.objects.create(user=user) yield user @pytest.fixture def regular_user_with_underscore(django_user_model, django_db_reset_sequences, mock_tapis_client): - django_user_model.objects.create_user(username="user_name", - password="password", - first_name="Firstname3", - last_name="Lastname3", - email="user_name@user.com") + django_user_model.objects.create_user( + username="user_name", + password="password", + first_name="Firstname3", + last_name="Lastname3", + email="user_name@user.com", + ) user = django_user_model.objects.get(username="user_name") TapisOAuthToken.objects.create( - user=user, - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user=user, access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447 + ) PortalProfile.objects.create(user=user) yield user @@ -80,16 +73,13 @@ def authenticated_user(client, regular_user): @pytest.fixture def staff_user(client, django_user_model, django_db_reset_sequences, mock_tapis_client): - django_user_model.objects.create_user(username='staff', password='password') - user = django_user_model.objects.get(username='staff') + django_user_model.objects.create_user(username="staff", password="password") + user = django_user_model.objects.get(username="staff") user.is_staff = True user.save() TapisOAuthToken.objects.create( - user=user, - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user=user, access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447 + ) PortalProfile.objects.create(user=user) yield user @@ -102,47 +92,47 @@ def authenticated_staff(client, staff_user): @pytest.fixture def tapis_indexer(mocker): - yield mocker.patch('portal.libs.agave.operations.tapis_indexer') + yield mocker.patch("portal.libs.agave.operations.tapis_indexer") @pytest.fixture def tapis_listing_indexer(mocker): - yield mocker.patch('portal.libs.agave.operations.tapis_listing_indexer') + yield mocker.patch("portal.libs.agave.operations.tapis_listing_indexer") @pytest.fixture def agave_storage_system_mock(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/agave/systems/storage.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/agave/systems/storage.json")) as f: yield json.load(f) @pytest.fixture def tapis_file_mock(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/agave/files/file.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/agave/files/file.json")) as f: yield json.load(f) @pytest.fixture def agave_file_listing_mock(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/agave/files/file-listing.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/agave/files/file-listing.json")) as f: yield json.load(f) @pytest.fixture def tapis_file_listing_mock(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/agave/files/tapis-file-listing.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/agave/files/tapis-file-listing.json")) as f: yield json.load(f) @pytest.fixture def agave_listing_mock(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/agave/files/listing.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/agave/files/listing.json")) as f: yield json.load(f) @pytest.fixture def tapis_tokens_create_mock(): - yield json.load(open(os.path.join(settings.BASE_DIR, 'fixtures/agave/auth/create-tokens-response.json'))) + yield json.load(open(os.path.join(settings.BASE_DIR, "fixtures/agave/auth/create-tokens-response.json"))) @pytest.fixture @@ -151,5 +141,5 @@ def text_file_fixture(): filename = os.path.join(temp_directory, "text_file.txt") with open(filename, "w") as text_file: text_file.write("this is the contents of my text file") - with open(filename, 'rb') as text_file: + with open(filename, "rb") as text_file: yield text_file diff --git a/server/manage.py b/server/manage.py index b88ce318ae..193144bdb4 100755 --- a/server/manage.py +++ b/server/manage.py @@ -1,11 +1,12 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -17,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/server/poetry.lock b/server/poetry.lock index a4497acff7..eeae957694 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.2 and should not be changed by hand. [[package]] name = "amqp" @@ -1091,18 +1091,6 @@ twisted = {version = ">=22.4", extras = ["tls"]} [package.extras] tests = ["black", "django", "flake8", "flake8-bugbear", "hypothesis", "mypy", "pytest", "pytest-asyncio", "pytest-cov", "tox"] -[[package]] -name = "decorator" -version = "5.2.1" -description = "Decorators for Humans" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"}, - {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, -] - [[package]] name = "django" version = "5.2.13" @@ -1221,23 +1209,6 @@ files = [ [package.extras] tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] -[[package]] -name = "flake8" -version = "6.1.0" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.8.1" -groups = ["dev"] -files = [ - {file = "flake8-6.1.0-py2.py3-none-any.whl", hash = "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"}, - {file = "flake8-6.1.0.tar.gz", hash = "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" -pyflakes = ">=3.1.0,<3.2.0" - [[package]] name = "fonttools" version = "4.60.1" @@ -1395,15 +1366,15 @@ files = [ ] [package.dependencies] -google-auth = ">=1.25.0,<3.0dev" -googleapis-common-protos = ">=1.56.2,<2.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" -requests = ">=2.18.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] +grpc = ["grpcio (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.33.2,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" @@ -1418,12 +1389,12 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.21.0,<3dev", markers = "python_version >= \"3\""} -google-auth = {version = ">=1.16.0,<3dev", markers = "python_version >= \"3\""} +google-api-core = {version = ">=1.21.0,<3.dev0", markers = "python_version >= \"3\""} +google-auth = {version = ">=1.16.0,<3.dev0", markers = "python_version >= \"3\""} google-auth-httplib2 = ">=0.0.3" -httplib2 = ">=0.15.0,<1dev" -six = ">=1.13.0,<2dev" -uritemplate = ">=3.0.0,<4dev" +httplib2 = ">=0.15.0,<1.dev0" +six = ">=1.13.0,<2.dev0" +uritemplate = ">=3.0.0,<4.dev0" [[package]] name = "google-auth" @@ -1445,7 +1416,7 @@ setuptools = ">=40.3.0" six = ">=1.9.0" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev) ; python_version >= \"3.6\"", "requests (>=2.20.0,<3.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0) ; python_version >= \"3.6\"", "requests (>=2.20.0,<3.0.0.dev0)"] pyopenssl = ["pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] @@ -1785,30 +1756,30 @@ files = [ [[package]] name = "ipython" -version = "9.12.0" +version = "9.17.1" description = "IPython: Productive Interactive Computing" optional = false -python-versions = ">=3.12" +python-versions = ">=3.11" groups = ["main"] files = [ - {file = "ipython-9.12.0-py3-none-any.whl", hash = "sha256:0f2701e8ee86e117e37f50563205d36feaa259d2e08d4a6bc6b6d74b18ce128d"}, - {file = "ipython-9.12.0.tar.gz", hash = "sha256:01daa83f504b693ba523b5a407246cabde4eb4513285a3c6acaff11a66735ee4"}, + {file = "ipython-9.17.1-py3-none-any.whl", hash = "sha256:6d1645743cfd1a07eb695d85aa2b5fa66721f8cbae9431d4049f7084bbf06509"}, + {file = "ipython-9.17.1.tar.gz", hash = "sha256:8919be8c27f20a6f4423145028063f6637b42a03ce57665bb12015ee1f073529"}, ] [package.dependencies] colorama = {version = ">=0.4.4", markers = "sys_platform == \"win32\""} -decorator = ">=5.1.0" ipython-pygments-lexers = ">=1.0.0" jedi = ">=0.18.2" matplotlib-inline = ">=0.1.6" pexpect = {version = ">4.6", markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\""} prompt_toolkit = ">=3.0.41,<3.1.0" +psutil = {version = ">=7", markers = "sys_platform != \"emscripten\" and sys_platform != \"cygwin\""} pygments = ">=2.14.0" stack_data = ">=0.6.0" traitlets = ">=5.13.0" [package.extras] -all = ["argcomplete (>=3.0)", "ipython[doc,matplotlib,terminal,test,test-extra]", "types-decorator"] +all = ["argcomplete (>=3.0)", "ipython[doc,matplotlib,test,test-extra]"] black = ["black"] doc = ["docrepr", "exceptiongroup", "intersphinx_registry", "ipykernel", "ipython[matplotlib,test]", "setuptools (>=80.0)", "sphinx (>=8.0)", "sphinx-rtd-theme (>=0.1.8)", "sphinx_toml (==0.0.4)", "typing_extensions"] matplotlib = ["matplotlib (>3.9)"] @@ -2056,7 +2027,7 @@ librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] mongodb = ["pymongo (==4.15.3)"] msgpack = ["msgpack (==1.1.2)"] pyro = ["pyro4 (==4.82)"] -qpid = ["qpid-python (==1.36.0-1)", "qpid-tools (==1.36.0-1)"] +qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"] redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"] slmq = ["softlayer_messaging (>=1.0.3)"] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] @@ -2470,18 +2441,6 @@ traitlets = "*" [package.extras] test = ["flake8", "nbdime", "nbval", "notebook", "pytest"] -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -groups = ["dev"] -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "mock" version = "5.2.0" @@ -3013,6 +2972,42 @@ files = [ {file = "protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff"}, ] +[[package]] +name = "psutil" +version = "7.2.2" +description = "Cross-platform lib for process and system monitoring." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "sys_platform != \"emscripten\" and sys_platform != \"cygwin\"" +files = [ + {file = "psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b"}, + {file = "psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea"}, + {file = "psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63"}, + {file = "psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312"}, + {file = "psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b"}, + {file = "psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9"}, + {file = "psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00"}, + {file = "psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9"}, + {file = "psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a"}, + {file = "psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf"}, + {file = "psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1"}, + {file = "psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841"}, + {file = "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486"}, + {file = "psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979"}, + {file = "psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9"}, + {file = "psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e"}, + {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8"}, + {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc"}, + {file = "psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988"}, + {file = "psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee"}, + {file = "psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372"}, +] + +[package.extras] +dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] +test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] + [[package]] name = "psycopg2" version = "2.9.11" @@ -3099,18 +3094,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.6.1,<0.7.0" -[[package]] -name = "pycodestyle" -version = "2.11.1" -description = "Python style guide checker" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, -] - [[package]] name = "pycparser" version = "3.0" @@ -3327,18 +3310,6 @@ files = [ [package.dependencies] typing-extensions = ">=4.14.1" -[[package]] -name = "pyflakes" -version = "3.1.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "pyflakes-3.1.0-py2.py3-none-any.whl", hash = "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774"}, - {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, -] - [[package]] name = "pygments" version = "2.20.0" @@ -3876,6 +3847,34 @@ dev = ["flake8", "flake8-bandit", "flake8-comprehensions", "flake8-docstrings", docs = ["sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme"] test = ["codecov", "coveralls", "nose"] +[[package]] +name = "ruff" +version = "0.16.5" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b"}, + {file = "ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3"}, + {file = "ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105"}, + {file = "ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29"}, + {file = "ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf"}, + {file = "ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91"}, + {file = "ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a"}, + {file = "ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef"}, + {file = "ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26"}, + {file = "ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f"}, + {file = "ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e"}, + {file = "ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b"}, +] + [[package]] name = "service-identity" version = "24.2.0" @@ -4429,5 +4428,5 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" -python-versions = "^3.12" -content-hash = "654eec4481ed4d1854e4894c5a069fa947c885b556fde29c9314ac61cc06fa9d" +python-versions = "==3.12.*" +content-hash = "20fbc03deb9531a04f401fd8a06ea9533608a9b853bc45688ca0044adabb87b9" diff --git a/server/portal/__init__.py b/server/portal/__init__.py index fb989c4e63..53f4ccb1d8 100644 --- a/server/portal/__init__.py +++ b/server/portal/__init__.py @@ -1,3 +1,3 @@ from .celery import app as celery_app -__all__ = ('celery_app',) +__all__ = ("celery_app",) diff --git a/server/portal/apps/_custom/drp/metadata_mappings.py b/server/portal/apps/_custom/drp/metadata_mappings.py index 289c80eae0..816387fd3f 100644 --- a/server/portal/apps/_custom/drp/metadata_mappings.py +++ b/server/portal/apps/_custom/drp/metadata_mappings.py @@ -1,58 +1,55 @@ """Mapping of metadata fields from old DRP database to new DRP metadata model.""" SAMPLE_POROUS_MEDIA_TYPE_MAPPINGS = { - 'Beads': 'beads', - 'BEAD': 'beads', - 'Sandstone': 'sandstone', - 'SAND': 'sandstone', - 'CARB': 'carbonate', - 'SOIL': 'soil', - 'FIBR': 'fibrous_media', - 'GRAN': 'granite', - 'COAL': 'coal', - 'Other': 'other', - 'OTHE': 'other', + "Beads": "beads", + "BEAD": "beads", + "Sandstone": "sandstone", + "SAND": "sandstone", + "CARB": "carbonate", + "SOIL": "soil", + "FIBR": "fibrous_media", + "GRAN": "granite", + "COAL": "coal", + "Other": "other", + "OTHE": "other", } SAMPLE_SOURCE_MAPPINGS = { - 'Artificial': 'artificial', - 'A': 'artificial', - 'Natural': 'natural', - 'N': 'natural', + "Artificial": "artificial", + "A": "artificial", + "Natural": "natural", + "N": "natural", } -ORIGIN_DATA_IS_SEGMENTED_MAPPING = { - 1: 'yes', - 2: 'no' -} +ORIGIN_DATA_IS_SEGMENTED_MAPPING = {1: "yes", 2: "no"} ORIGIN_DATA_VOXEL_UNIT_MAPPING = { - 'micrometer': 'micrometer', - 'um': 'micrometer', - 'mm': 'millimeter', - 'nm': 'nanometer', - 'other': 'other' + "micrometer": "micrometer", + "um": "micrometer", + "mm": "millimeter", + "nm": "nanometer", + "other": "other", } ANALYSIS_DATA_TYPE_MAPPING = { - 'Simulation': 'simulation', - 'GeometricAnalysis': 'geometric_analysis', - 'Other': 'other', + "Simulation": "simulation", + "GeometricAnalysis": "geometric_analysis", + "Other": "other", } FILE_IMAGE_TYPE_MAPPING = { - '8-bit': '8_bit', - '64-bit Real': '64_bit_real', - '16-bit Unsigned': '16_bit_unsigned', - '32-bit Real': '32_bit_real', - '32-bit Signed': '32_bit_signed', - '24-bit RGB': '24_bit_rgb', - '32-bit Unsigned': '32_bit_unsigned', + "8-bit": "8_bit", + "64-bit Real": "64_bit_real", + "16-bit Unsigned": "16_bit_unsigned", + "32-bit Real": "32_bit_real", + "32-bit Signed": "32_bit_signed", + "24-bit RGB": "24_bit_rgb", + "32-bit Unsigned": "32_bit_unsigned", } FILE_BYTE_ORDER_MAPPING = { - 'little-endian': 'little_endian', - 'big-endian': 'big_endian', + "little-endian": "little_endian", + "big-endian": "big_endian", } FILE_USE_BINARY_CORRECTION_MAPPING = { diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index be21b84c1e..172366a0fe 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -21,16 +21,30 @@ class DrpFileMetadata(BaseFileMetadata): ) is_advanced_image_file: Optional[bool] = False - image_type: Optional[Literal[ - '8_bit', '16_bit_signed', '16_bit_unsigned', '32_bit_signed', '32_bit_unsigned', '32_bit_real', '64_bit_real', - '24_bit_rgb', '24_bit_rgb_planar', '24_bit_bgr', '24_bit_integer', '32_bit_argb', '32_bit_abgr', '1_bit_bitmap', - ]] = None + image_type: Optional[ + Literal[ + "8_bit", + "16_bit_signed", + "16_bit_unsigned", + "32_bit_signed", + "32_bit_unsigned", + "32_bit_real", + "64_bit_real", + "24_bit_rgb", + "24_bit_rgb_planar", + "24_bit_bgr", + "24_bit_integer", + "32_bit_argb", + "32_bit_abgr", + "1_bit_bitmap", + ] + ] = None height: Optional[NonNegativeInt] = None width: Optional[NonNegativeInt] = None number_of_images: Optional[NonNegativeInt] = None offset_to_first_image: Optional[int] = None gap_between_images: Optional[int] = None - byte_order: Optional[Literal['big_endian', 'little_endian']] = None + byte_order: Optional[Literal["big_endian", "little_endian"]] = None use_binary_correction: Optional[bool] = None @@ -48,13 +62,7 @@ class DrpDatasetMetadata(BaseMetadataModel): name: str description: Optional[str] = None uuid: Optional[str] = None - data_type: Literal[ - "sample", - "origin_data", - "digital_dataset", - "analysis_data", - "file" - ] + data_type: Literal["sample", "origin_data", "digital_dataset", "analysis_data", "file"] file_objs: list[FileObj] = [] @@ -62,15 +70,7 @@ class DrpSampleMetadata(DrpDatasetMetadata): """Model for DRP Sample Metadata""" porous_media_type: Literal[ - "sandstone", - "soil", - "carbonate", - "granite", - "beads", - "fibrous_media", - "coal", - "energy_storage", - "other" + "sandstone", "soil", "carbonate", "granite", "beads", "fibrous_media", "coal", "energy_storage", "other" ] porous_media_other_description: Optional[str] = None @@ -93,12 +93,7 @@ class DrpSampleMetadata(DrpDatasetMetadata): grain_size_min: Optional[NonNegativeFloat] = None grain_size_max: Optional[NonNegativeFloat] = None grain_size_avg: Optional[NonNegativeFloat] = None - grain_size_units: Optional[Literal[ - "nanometer", - "micrometer", - "millimeter", - "other" - ]] = None + grain_size_units: Optional[Literal["nanometer", "micrometer", "millimeter", "other"]] = None porosity: Optional[float] = None geographical_location: Optional[str] = None date_of_collection: Optional[str] = None @@ -120,12 +115,7 @@ class DrpOriginDatasetMetadata(DrpDatasetMetadata): voxel_x: Optional[NonNegativeFloat] = None voxel_y: Optional[NonNegativeFloat] = None voxel_z: Optional[NonNegativeFloat] = None - voxel_units: Optional[Literal[ - "nanometer", - "micrometer", - "millimeter", - "other" - ]] = None + voxel_units: Optional[Literal["nanometer", "micrometer", "millimeter", "other"]] = None dimensionality: Optional[str] = None digital_dataset: Optional[str] = None external_uri: Optional[str] = None # TODO_DRP: Remove in new model @@ -136,12 +126,7 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): is_segmented: Literal["yes", "no"] dataset_type: Literal[ - "machine_learning", - "simulation", - "geometric_analysis", - "experimental", - "characterization", - "other" + "machine_learning", "simulation", "geometric_analysis", "experimental", "characterization", "other" ] external_uri: Optional[str] = None sample: str diff --git a/server/portal/apps/_custom/drp/urls.py b/server/portal/apps/_custom/drp/urls.py index 11c767b5dd..12a5d66559 100644 --- a/server/portal/apps/_custom/drp/urls.py +++ b/server/portal/apps/_custom/drp/urls.py @@ -2,11 +2,12 @@ .. module:: portal.apps.forms.urls :synopsis: Forms URLs """ + from django.urls import re_path from portal.apps._custom.drp.views import DigitalRocksSampleView, GenerateImagesView -app_name = 'custom' +app_name = "custom" urlpatterns = [ - re_path('^$', DigitalRocksSampleView.as_view(), name='drp'), - re_path('generate-images/', GenerateImagesView.as_view(), name='generate_images') + re_path("^$", DigitalRocksSampleView.as_view(), name="drp"), + re_path("generate-images/", GenerateImagesView.as_view(), name="generate_images"), ] diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 4d35fa48cf..c932c11186 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -15,12 +15,11 @@ class DigitalRocksSampleView(BaseApiView): - def get(self, request): - project_id = request.GET.get('project_id') - get_origin_data = request.GET.get('get_origin_data') + project_id = request.GET.get("project_id") + get_origin_data = request.GET.get("get_origin_data") - full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + full_project_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" graph_model = ProjectMetadata.objects.get( name=constants.PROJECT_GRAPH, base_project__value__projectId=full_project_id @@ -30,23 +29,21 @@ def get(self, request): sample_uuids = [] - for node_id in list(project_graph.successors('NODE_ROOT')): + for node_id in list(project_graph.successors("NODE_ROOT")): node = project_graph.nodes[node_id] - if (node.get('name') == constants.SAMPLE): - sample_uuids.append(node.get('uuid')) + if node.get("name") == constants.SAMPLE: + sample_uuids.append(node.get("uuid")) - samples = ProjectMetadata.objects.filter(uuid__in=sample_uuids).values('uuid', 'name', 'value') + samples = ProjectMetadata.objects.filter(uuid__in=sample_uuids).values("uuid", "name", "value") origin_data = [] - if get_origin_data == 'true': - origin_data = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, - name=constants.DIGITAL_DATASET).values('uuid', 'name', 'value') + if get_origin_data == "true": + origin_data = ProjectMetadata.objects.filter( + base_project__value__projectId=full_project_id, name=constants.DIGITAL_DATASET + ).values("uuid", "name", "value") - response_data = { - 'samples': list(samples), - 'origin_data': list(origin_data) - } + response_data = {"samples": list(samples), "origin_data": list(origin_data)} return JsonResponse({"response": response_data}) diff --git a/server/portal/apps/accounts/api/urls.py b/server/portal/apps/accounts/api/urls.py index 2cfc3f380e..7ca265b61c 100644 --- a/server/portal/apps/accounts/api/urls.py +++ b/server/portal/apps/accounts/api/urls.py @@ -2,11 +2,11 @@ .. :module:: apps.accounts.api.urls :synopsis: Manager handling anything pertaining to accounts """ + from django.urls import re_path from portal.apps.accounts.api.views.systems import SystemKeysView -app_name = 'portal_accounts_api' +app_name = "portal_accounts_api" urlpatterns = [ - re_path(r'^systems/(?P[\w.\-\/]+)/keys/?$', - SystemKeysView.as_view()), + re_path(r"^systems/(?P[\w.\-\/]+)/keys/?$", SystemKeysView.as_view()), ] diff --git a/server/portal/apps/accounts/api/views/systems.py b/server/portal/apps/accounts/api/views/systems.py index 17fc408adb..b50f4c8bf8 100644 --- a/server/portal/apps/accounts/api/views/systems.py +++ b/server/portal/apps/accounts/api/views/systems.py @@ -53,9 +53,7 @@ def push(self, request, system_id, body): if default_authn_method == "TMS_KEYS": try: - create_system_credentials_with_tms( - client, tapis_username, system_id - ) + create_system_credentials_with_tms(client, tapis_username, system_id) http_status = 200 result = "OK" except BaseTapyException as e: @@ -67,9 +65,7 @@ def push(self, request, system_id, body): http_status = e.response.status_code result = e.message elif default_authn_method == "PKI_KEYS": - logger.info( - f"Resetting credentials for user {tapis_username} on system {system_id}" - ) + logger.info(f"Resetting credentials for user {tapis_username} on system {system_id}") priv_key_str, publ_key_str = createKeyPair() success, result, http_status = AccountsManager.add_pub_key_to_resource( @@ -83,9 +79,7 @@ def push(self, request, system_id, body): ) if not success: - logger.error( - f"Failed to push keys for user {tapis_username} on system {system_id}: {result}" - ) + logger.error(f"Failed to push keys for user {tapis_username} on system {system_id}: {result}") return JsonResponse({"message": result}, status=http_status) create_system_credentials_with_keys( @@ -120,9 +114,7 @@ def push(self, request, system_id, body): tapis_system = client.systems.getSystem(systemId=system_id) portal_system = { - "name": tapis_system.notes.get( - "label", tapis_system.notes.get("title", tapis_system.id) - ), + "name": tapis_system.notes.get("label", tapis_system.notes.get("title", tapis_system.id)), "system": tapis_system.id, "scheme": "private", "api": "tapis", @@ -134,6 +126,4 @@ def push(self, request, system_id, body): request.user.tapis_oauth, portal_system, default_host_eval="HOME" ) - return JsonResponse( - {"system": evaluated_system, "message": result}, status=http_status - ) + return JsonResponse({"system": evaluated_system, "message": result}, status=http_status) diff --git a/server/portal/apps/accounts/apps.py b/server/portal/apps/accounts/apps.py index 49c6f95d6a..484b30dbcc 100644 --- a/server/portal/apps/accounts/apps.py +++ b/server/portal/apps/accounts/apps.py @@ -2,4 +2,4 @@ class AccountsConfig(AppConfig): - name = 'portal.apps.accounts' + name = "portal.apps.accounts" diff --git a/server/portal/apps/accounts/integrations.py b/server/portal/apps/accounts/integrations.py index 3a35e78afa..89a63efdd9 100644 --- a/server/portal/apps/accounts/integrations.py +++ b/server/portal/apps/accounts/integrations.py @@ -4,7 +4,7 @@ logger = logging.getLogger(__name__) -INTEGRATION_APPS = [s['integration'] for s in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if 'integration' in s] +INTEGRATION_APPS = [s["integration"] for s in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if "integration" in s] def get_integrations(request): @@ -12,11 +12,14 @@ def get_integrations(request): for app in INTEGRATION_APPS: try: - mod = import_module('{}.integrations'.format(app)) + mod = import_module("{}.integrations".format(app)) app_integrations += mod.provide_integrations(request) except Exception as exc: - logger.warning('Call to module.provide_integrations fail for module: {app_name}. {exc}' - .format(app_name=app, exc=str(exc))) + logger.warning( + "Call to module.provide_integrations fail for module: {app_name}. {exc}".format( + app_name=app, exc=str(exc) + ) + ) return app_integrations diff --git a/server/portal/apps/accounts/managers/abstract.py b/server/portal/apps/accounts/managers/abstract.py index 7889fcf41b..afb9326b4c 100644 --- a/server/portal/apps/accounts/managers/abstract.py +++ b/server/portal/apps/accounts/managers/abstract.py @@ -22,12 +22,6 @@ class AbstractKeysManager: """ @abstractmethod - def add_public_key( - self, - system_id, - hostname, - port, - public_key - ): # pylint: disable=too-many-arguments + def add_public_key(self, system_id, hostname, port, public_key): # pylint: disable=too-many-arguments """Adds public key to `authorized_keys`""" return NotImplemented diff --git a/server/portal/apps/accounts/managers/accounts.py b/server/portal/apps/accounts/managers/accounts.py index e91559bd6f..c1f39af6db 100644 --- a/server/portal/apps/accounts/managers/accounts.py +++ b/server/portal/apps/accounts/managers/accounts.py @@ -2,14 +2,11 @@ .. :module:: apps.accounts.managers.accounts :synopsis: Manager handling anything pertaining to accounts """ + import logging from importlib import import_module from django.conf import settings -from paramiko.ssh_exception import ( - AuthenticationException, - ChannelException, - SSHException -) +from paramiko.ssh_exception import AuthenticationException, ChannelException, SSHException from portal.apps.accounts.managers.ssh_keys import KeyCannotBeAdded logger = logging.getLogger(__name__) @@ -24,23 +21,23 @@ def _lookup_keys_manager(username, password, token): """ mgr_str = getattr( settings, - 'PORTAL_KEYS_MANAGER', + "PORTAL_KEYS_MANAGER", ) - module_str, cls_str = mgr_str.rsplit('.', 1) + module_str, cls_str = mgr_str.rsplit(".", 1) module = import_module(module_str) cls = getattr(module, cls_str) return cls(username, password, token) def add_pub_key_to_resource( - user, - username, - password, - token, - system_id, - pub_key, - hostname=None, - port=22, + user, + username, + password, + token, + system_id, + pub_key, + hostname=None, + port=22, ): """Add Public Key to Remote Resource @@ -65,13 +62,7 @@ def add_pub_key_to_resource( hostname = sys.host transport = mgr.get_transport(hostname, port) - message = mgr.add_public_key( - system_id, - hostname, - pub_key, - port=port, - transport=transport - ) + message = mgr.add_public_key(system_id, hostname, pub_key, port=port, transport=transport) status = 200 except Exception as exc: # Catch all exceptions and set a status code for unknown exceptions @@ -88,10 +79,7 @@ def add_pub_key_to_resource( # May occur when system is down message = "KeyCannotBeAdded" # KeyCannnotBeAdded exception does not contain a message? status = 503 - except ( - ChannelException, - SSHException - ) as exc: + except (ChannelException, SSHException) as exc: # cannot ssh to system message = str(type(exc)) # paramiko exceptions do not contain a string message? status = 500 # Bad gateway diff --git a/server/portal/apps/accounts/managers/ssh_keys.py b/server/portal/apps/accounts/managers/ssh_keys.py index 00d3d2f3f2..e2d11d373d 100644 --- a/server/portal/apps/accounts/managers/ssh_keys.py +++ b/server/portal/apps/accounts/managers/ssh_keys.py @@ -18,6 +18,7 @@ class KeyCannotBeAdded(Exception): Exception raised when there is an error adding a public key to `~/.ssh/authorized_keys` """ + def __init__(self, msg, output, error_output, *args, **kwargs): super(KeyCannotBeAdded, self).__init__(*args, **kwargs) self.msg = msg @@ -25,11 +26,7 @@ def __init__(self, msg, output, error_output, *args, **kwargs): self.error_output = error_output def __str__(self): - return '{msg}: {output} \n {error}'.format( - msg=self.msg, - output=self.output, - error=self.error_output - ) + return "{msg}: {output} \n {error}".format(msg=self.msg, output=self.output, error=self.error_output) class KeysManager(AbstractKeysManager): @@ -47,32 +44,23 @@ def __init__(self, username, password, token): self.password = password self.token = token - def _ssh_prompt_handler( - self, - title, - instructions, - prompt_list - ): + def _ssh_prompt_handler(self, title, instructions, prompt_list): """SSH Prompt Handler This method handles SSH prompts from cloud resources """ answers = { - 'password': self.password, - 'tacc_token_code': self.token, - 'tacc_token': self.token, - f'totp_code_for_{self.username}': self.token + "password": self.password, + "tacc_token_code": self.token, + "tacc_token": self.token, + f"totp_code_for_{self.username}": self.token, } resp = [] - logger.debug('title: %s', title) - logger.debug('instructions: %s', instructions) - logger.debug('list: %s', prompt_list) + logger.debug("title: %s", title) + logger.debug("instructions: %s", instructions) + logger.debug("list: %s", prompt_list) for prmpt in prompt_list: - prmpt_str = prmpt[0]\ - .lower()\ - .strip()\ - .replace(' ', '_')\ - .replace(':', '') + prmpt_str = prmpt[0].lower().strip().replace(" ", "_").replace(":", "") resp.append(answers[prmpt_str]) return resp @@ -96,10 +84,7 @@ def _get_pub_key_comment(self, system_id): :return str: comment """ - comment = '{username}@{system_id}'.format( - username=self.username, - system_id=system_id - ) + comment = "{username}@{system_id}".format(username=self.username, system_id=system_id) return comment def _get_add_pub_key_command(self, system_id, public_key): @@ -111,25 +96,17 @@ def _get_add_pub_key_command(self, system_id, public_key): :return str: command """ comment = self._get_pub_key_comment(system_id) - string = ' '.join([public_key, comment]) + string = " ".join([public_key, comment]) command = ( 'if [ ! -f "~/.ssh/authorized_keys" ]; then ' - 'mkdir -p ~/.ssh/ && touch ~/.ssh/authorized_keys ' - '&& chmod 0600 ~/.ssh/authorized_keys; fi && ' + "mkdir -p ~/.ssh/ && touch ~/.ssh/authorized_keys " + "&& chmod 0600 ~/.ssh/authorized_keys; fi && " 'grep -q -F "{string}" ~/.ssh/authorized_keys || ' - 'echo "{string}" >> ~/.ssh/authorized_keys').format( - string=string - ) + 'echo "{string}" >> ~/.ssh/authorized_keys' + ).format(string=string) return command - def add_public_key( - self, - system_id, - hostname, - public_key, - port=22, - transport=None - ): # pylint: disable=too-many-arguments, arguments-differ + def add_public_key(self, system_id, hostname, public_key, port=22, transport=None): # pylint: disable=too-many-arguments, arguments-differ """Adds public key to `authorized_keys` :param str sytem_id: System Id @@ -151,24 +128,20 @@ def add_public_key( status = channel.recv_exit_status() output = channel.makefile() stderr = channel.makefile_stderr() - output_lines = '' + output_lines = "" for line in output.readlines(): - output_lines += line + '\n' + output_lines += line + "\n" logger.debug(line) if status == -1: - logger.info('No response from the server') + logger.info("No response from the server") elif status == 0: - logger.info('Public key added successfully to {}'.format(hostname)) + logger.info("Public key added successfully to {}".format(hostname)) elif status > 0: - error_lines = '' + error_lines = "" for line in stderr.readlines(): - error_lines += line + '\n' + error_lines += line + "\n" - raise KeyCannotBeAdded( - 'Error adding public key', - output_lines, - error_lines - ) + raise KeyCannotBeAdded("Error adding public key", output_lines, error_lines) trans.close() return output_lines diff --git a/server/portal/apps/accounts/managers/unit_test.py b/server/portal/apps/accounts/managers/unit_test.py index f15f80244a..86c4e58b72 100644 --- a/server/portal/apps/accounts/managers/unit_test.py +++ b/server/portal/apps/accounts/managers/unit_test.py @@ -2,21 +2,14 @@ from portal.apps.accounts.managers.accounts import add_pub_key_to_resource from portal.apps.accounts.managers.ssh_keys import KeysManager from portal.apps.accounts.managers.ssh_keys import KeyCannotBeAdded -from paramiko.ssh_exception import ( - AuthenticationException, - ChannelException, - SSHException -) +from paramiko.ssh_exception import AuthenticationException, ChannelException, SSHException import pytest @pytest.fixture def mock_lookup_keys_manager(mocker): yield mocker.patch( - 'portal.apps.accounts.managers.accounts._lookup_keys_manager', - return_value=MagicMock( - spec=KeysManager - ) + "portal.apps.accounts.managers.accounts._lookup_keys_manager", return_value=MagicMock(spec=KeysManager) ) @@ -24,14 +17,16 @@ def _run_add_pub_key_to_resource(user): password = "testpassword" token = "123456" system_id = "portal-home.testuser" - pub_key = 'pubkey' + pub_key = "pubkey" hostname = "data.tacc.utexas.edu" return add_pub_key_to_resource(user, password, token, system_id, pub_key, hostname) # AuthenticationException occurs with bad password/token when trying to push keys def test_authentication_exception(regular_user, mock_lookup_keys_manager): - mock_lookup_keys_manager.return_value.add_public_key = MagicMock(side_effect=AuthenticationException("Authentication failed.")) + mock_lookup_keys_manager.return_value.add_public_key = MagicMock( + side_effect=AuthenticationException("Authentication failed.") + ) result, message, status = _run_add_pub_key_to_resource(regular_user) assert result is False assert status == 403 @@ -40,7 +35,9 @@ def test_authentication_exception(regular_user, mock_lookup_keys_manager): # Channel exception occurs when server is reachable but returns an error while paramiko is attempting to open a channel def test_channel_exception(regular_user, mock_lookup_keys_manager): - mock_lookup_keys_manager.return_value.add_public_key = MagicMock(side_effect=ChannelException(999, "Mock Channel Exception")) + mock_lookup_keys_manager.return_value.add_public_key = MagicMock( + side_effect=ChannelException(999, "Mock Channel Exception") + ) result, message, status = _run_add_pub_key_to_resource(regular_user) assert result is False assert status == 500 @@ -57,7 +54,8 @@ def test_ssh_exception(regular_user, mock_lookup_keys_manager): # KeyCannotBeAdded exception occurs when authorized_keys file cannot be modified def test_KeyCannotBeAdded_exception(regular_user, mock_lookup_keys_manager): mock_lookup_keys_manager.return_value.add_public_key = MagicMock( - side_effect=KeyCannotBeAdded("MockKeyCannotBeAdded", "MockOutput", "MockErrorOutput")) + side_effect=KeyCannotBeAdded("MockKeyCannotBeAdded", "MockOutput", "MockErrorOutput") + ) result, message, status = _run_add_pub_key_to_resource(regular_user) assert result is False assert status == 503 diff --git a/server/portal/apps/accounts/migrations/0001_initial.py b/server/portal/apps/accounts/migrations/0001_initial.py index 0c835fb38f..4b0674da93 100644 --- a/server/portal/apps/accounts/migrations/0001_initial.py +++ b/server/portal/apps/accounts/migrations/0001_initial.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -15,38 +14,57 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='PortalProfileNHInterests', + name="PortalProfileNHInterests", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('description', models.CharField(max_length=300)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("description", models.CharField(max_length=300)), ], ), migrations.CreateModel( - name='PortalProfileResearchActivities', + name="PortalProfileResearchActivities", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('description', models.CharField(max_length=300)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("description", models.CharField(max_length=300)), ], ), migrations.CreateModel( - name='PortalProfile', + name="PortalProfile", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('ethnicity', models.CharField(max_length=255)), - ('gender', models.CharField(max_length=255)), - ('setup_complete', models.BooleanField(default=False)), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='profile', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("ethnicity", models.CharField(max_length=255)), + ("gender", models.CharField(max_length=255)), + ("setup_complete", models.BooleanField(default=False)), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, related_name="profile", to=settings.AUTH_USER_MODEL + ), + ), ], ), migrations.CreateModel( - name='NotificationPreferences', + name="NotificationPreferences", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('announcements', models.BooleanField(default=True, verbose_name='Receive occasional announcements from sal.frontera')), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='notification_preferences', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "announcements", + models.BooleanField( + default=True, verbose_name="Receive occasional announcements from sal.frontera" + ), + ), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="notification_preferences", + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'permissions': (('view_notification_subscribers', 'Can view list of users subscribed to a notification type'),), + "permissions": ( + ("view_notification_subscribers", "Can view list of users subscribed to a notification type"), + ), }, ), ] diff --git a/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py b/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py index cab7eb6b28..4ae01d3240 100644 --- a/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py +++ b/server/portal/apps/accounts/migrations/0002_hostkeys_keys_sshkeys.py @@ -6,41 +6,57 @@ class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('accounts', '0001_initial'), + ("accounts", "0001_initial"), ] operations = [ migrations.CreateModel( - name='SSHKeys', + name="SSHKeys", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='ssh_keys', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="ssh_keys", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.CreateModel( - name='Keys', + name="Keys", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('system', models.TextField(unique=True)), - ('private', models.TextField()), - ('public', models.TextField()), - ('ssh_keys', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='accounts.SSHKeys')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("system", models.TextField(unique=True)), + ("private", models.TextField()), + ("public", models.TextField()), + ( + "ssh_keys", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to="accounts.SSHKeys" + ), + ), ], ), migrations.CreateModel( - name='HostKeys', + name="HostKeys", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('hostname', models.TextField()), - ('private', models.TextField()), - ('public', models.TextField()), - ('ssh_keys', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='accounts.SSHKeys')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("hostname", models.TextField()), + ("private", models.TextField()), + ("public", models.TextField()), + ( + "ssh_keys", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to="accounts.SSHKeys" + ), + ), ], options={ - 'unique_together': {('hostname', 'ssh_keys')}, + "unique_together": {("hostname", "ssh_keys")}, }, ), ] diff --git a/server/portal/apps/accounts/migrations/0003_auto_20200131_0207.py b/server/portal/apps/accounts/migrations/0003_auto_20200131_0207.py index d73882804d..bd668bd1c4 100644 --- a/server/portal/apps/accounts/migrations/0003_auto_20200131_0207.py +++ b/server/portal/apps/accounts/migrations/0003_auto_20200131_0207.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('accounts', '0002_hostkeys_keys_sshkeys'), + ("accounts", "0002_hostkeys_keys_sshkeys"), ] operations = [ migrations.AlterField( - model_name='notificationpreferences', - name='announcements', - field=models.BooleanField(default=True, verbose_name='Receive occasional announcements from Frontera'), + model_name="notificationpreferences", + name="announcements", + field=models.BooleanField(default=True, verbose_name="Receive occasional announcements from Frontera"), ), ] diff --git a/server/portal/apps/accounts/migrations/0004_auto_20200318_2055.py b/server/portal/apps/accounts/migrations/0004_auto_20200318_2055.py index 528e1e3649..177b386d23 100644 --- a/server/portal/apps/accounts/migrations/0004_auto_20200318_2055.py +++ b/server/portal/apps/accounts/migrations/0004_auto_20200318_2055.py @@ -4,30 +4,29 @@ class Migration(migrations.Migration): - dependencies = [ - ('accounts', '0003_auto_20200131_0207'), + ("accounts", "0003_auto_20200131_0207"), ] operations = [ migrations.AddField( - model_name='portalprofile', - name='bio', + model_name="portalprofile", + name="bio", field=models.CharField(blank=True, default=None, max_length=4096, null=True), ), migrations.AddField( - model_name='portalprofile', - name='orcid_id', + model_name="portalprofile", + name="orcid_id", field=models.CharField(blank=True, default=None, max_length=256, null=True), ), migrations.AddField( - model_name='portalprofile', - name='professional_level', + model_name="portalprofile", + name="professional_level", field=models.CharField(default=None, max_length=256, null=True), ), migrations.AddField( - model_name='portalprofile', - name='website', + model_name="portalprofile", + name="website", field=models.CharField(blank=True, default=None, max_length=256, null=True), ), ] diff --git a/server/portal/apps/accounts/migrations/0005_auto_20210316_1950.py b/server/portal/apps/accounts/migrations/0005_auto_20210316_1950.py index 83a7ded45e..1a34ecaa6f 100644 --- a/server/portal/apps/accounts/migrations/0005_auto_20210316_1950.py +++ b/server/portal/apps/accounts/migrations/0005_auto_20210316_1950.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('accounts', '0004_auto_20200318_2055'), + ("accounts", "0004_auto_20200318_2055"), ] operations = [ migrations.AlterField( - model_name='notificationpreferences', - name='announcements', - field=models.BooleanField(default=True, verbose_name='Receive occasional announcements'), + model_name="notificationpreferences", + name="announcements", + field=models.BooleanField(default=True, verbose_name="Receive occasional announcements"), ), ] diff --git a/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py b/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py index 183c3ebd79..f9e7fef17e 100644 --- a/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py +++ b/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py @@ -4,34 +4,33 @@ class Migration(migrations.Migration): - dependencies = [ - ('accounts', '0005_auto_20210316_1950'), + ("accounts", "0005_auto_20210316_1950"), ] operations = [ migrations.RemoveField( - model_name='portalprofile', - name='bio', + model_name="portalprofile", + name="bio", ), migrations.RemoveField( - model_name='portalprofile', - name='ethnicity', + model_name="portalprofile", + name="ethnicity", ), migrations.RemoveField( - model_name='portalprofile', - name='gender', + model_name="portalprofile", + name="gender", ), migrations.RemoveField( - model_name='portalprofile', - name='orcid_id', + model_name="portalprofile", + name="orcid_id", ), migrations.RemoveField( - model_name='portalprofile', - name='professional_level', + model_name="portalprofile", + name="professional_level", ), migrations.RemoveField( - model_name='portalprofile', - name='website', + model_name="portalprofile", + name="website", ), ] diff --git a/server/portal/apps/accounts/migrations/0007_portalprofile_institution_portalprofile_orcid_id.py b/server/portal/apps/accounts/migrations/0007_portalprofile_institution_portalprofile_orcid_id.py index 4d4fd77103..5f3fbd12ad 100644 --- a/server/portal/apps/accounts/migrations/0007_portalprofile_institution_portalprofile_orcid_id.py +++ b/server/portal/apps/accounts/migrations/0007_portalprofile_institution_portalprofile_orcid_id.py @@ -4,20 +4,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('accounts', '0006_auto_20231018_1927'), + ("accounts", "0006_auto_20231018_1927"), ] operations = [ migrations.AddField( - model_name='portalprofile', - name='institution', + model_name="portalprofile", + name="institution", field=models.CharField(max_length=255, null=True), ), migrations.AddField( - model_name='portalprofile', - name='orcid_id', + model_name="portalprofile", + name="orcid_id", field=models.CharField(max_length=255, null=True), ), ] diff --git a/server/portal/apps/accounts/models.py b/server/portal/apps/accounts/models.py index 766caf883d..50a6305265 100644 --- a/server/portal/apps/accounts/models.py +++ b/server/portal/apps/accounts/models.py @@ -2,6 +2,7 @@ .. :module:: apps.accounts.managers.models :synopsis: Account's models """ + import logging from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -21,11 +22,8 @@ class PortalProfile(models.Model): Extending the user model to store extra data """ - user = models.OneToOneField( - settings.AUTH_USER_MODEL, - related_name='profile', - on_delete=models.CASCADE - ) + + user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name="profile", on_delete=models.CASCADE) # Default to False. If PORTAL_USER_ACCOUNT_SETUP_STEPS is empty, # setup_complete will be set to True on first login setup_complete = models.BooleanField(default=False) @@ -35,11 +33,7 @@ class PortalProfile(models.Model): def send_mail(self, subject, body=None): """Send mail to user""" - send_mail(subject, - body, - settings.DEFAULT_FROM_EMAIL, - [self.user.email], - html_message=body) + send_mail(subject, body, settings.DEFAULT_FROM_EMAIL, [self.user.email], html_message=body) class NotificationPreferences(models.Model): @@ -48,38 +42,37 @@ class NotificationPreferences(models.Model): .. todo: Should we have a `Preferences` model and store there all different kinds of preferences? """ - user = models.OneToOneField(settings.AUTH_USER_MODEL, - related_name='notification_preferences', - on_delete=models.CASCADE) - announcements = models.BooleanField( - default=True, - verbose_name=_('Receive occasional announcements')) + + user = models.OneToOneField( + settings.AUTH_USER_MODEL, related_name="notification_preferences", on_delete=models.CASCADE + ) + announcements = models.BooleanField(default=True, verbose_name=_("Receive occasional announcements")) class Meta: - permissions = ( - ('view_notification_subscribers', - 'Can view list of users subscribed to a notification type'), - ) + permissions = (("view_notification_subscribers", "Can view list of users subscribed to a notification type"),) class PortalProfileNHInterests(models.Model): """Portal Profile NH Interests""" + description = models.CharField(max_length=300) class PortalProfileResearchActivities(models.Model): """Resesarch Activities""" + description = models.CharField(max_length=300) class SSHKeysManager(models.Manager): """SSHKeys Manager""" + def save_keys( - self, - user, - system_id, - priv_key, - pub_key, + self, + user, + system_id, + priv_key, + pub_key, ): """Saves a new set of keys for a specific system and user obj @@ -98,24 +91,14 @@ def save_keys( encrypted using AES """ try: - Keys.objects.get( - ssh_keys__user=user, - system=system_id - ) + Keys.objects.get(ssh_keys__user=user, system=system_id) except ObjectDoesNotExist: ssh_keys = super(SSHKeysManager, self).create(user=user) - Keys.objects.create( - ssh_keys=ssh_keys, - system=system_id, - private=priv_key, - public=pub_key - ) + Keys.objects.create(ssh_keys=ssh_keys, system=system_id, private=priv_key, public=pub_key) return ssh_keys raise ValueError( """A set of keys for system: '{system}' and username: '{username}' - already exists""".format( - system=system_id, - username=user.username) + already exists""".format(system=system_id, username=user.username) ) def update_keys(self, user, system_id, priv_key, pub_key): @@ -137,37 +120,25 @@ def update_keys(self, user, system_id, priv_key, pub_key): encrypted using AES """ try: - keys = Keys.objects.get( - ssh_keys__user=user, - system=system_id - ) + keys = Keys.objects.get(ssh_keys__user=user, system=system_id) except ObjectDoesNotExist: try: - ssh_keys = super( - SSHKeysManager, - self - ).get_queryset().get(user=user) + ssh_keys = super(SSHKeysManager, self).get_queryset().get(user=user) except ObjectDoesNotExist: - ssh_keys = super( - SSHKeysManager, - self - ).create(user=user) + ssh_keys = super(SSHKeysManager, self).create(user=user) keys = Keys.objects.create(ssh_keys=ssh_keys, system=system_id) keys.public = pub_key keys.private = priv_key keys.save() - return super( - SSHKeysManager, - self - ).get_queryset().get(user=user) + return super(SSHKeysManager, self).get_queryset().get(user=user) def save_hostname_keys( - self, - user, - hostname, - priv_key, - pub_key, + self, + user, + hostname, + priv_key, + pub_key, ): """Saves a new set of keys for a specific system and user obj @@ -186,24 +157,14 @@ def save_hostname_keys( encrypted using AES """ try: - HostKeys.objects.get( - ssh_keys__user=user, - hostname=hostname - ) + HostKeys.objects.get(ssh_keys__user=user, hostname=hostname) except ObjectDoesNotExist: ssh_keys = super(SSHKeysManager, self).create(user=user) - HostKeys.objects.create( - ssh_keys=ssh_keys, - hostname=hostname, - private=priv_key, - public=pub_key - ) + HostKeys.objects.create(ssh_keys=ssh_keys, hostname=hostname, private=priv_key, public=pub_key) return ssh_keys raise ValueError( """A set of keys for hostname: '{hostname}' and username: '{username}' - already exists""".format( - hostname=hostname, - username=user.username) + already exists""".format(hostname=hostname, username=user.username) ) def update_hostname_keys(self, user, hostname, priv_key, pub_key): @@ -225,30 +186,18 @@ def update_hostname_keys(self, user, hostname, priv_key, pub_key): encrypted using AES """ try: - keys = HostKeys.objects.get( - ssh_keys__user=user, - hostname=hostname - ) + keys = HostKeys.objects.get(ssh_keys__user=user, hostname=hostname) except ObjectDoesNotExist: try: - ssh_keys = super( - SSHKeysManager, - self - ).get_queryset().get(user=user) + ssh_keys = super(SSHKeysManager, self).get_queryset().get(user=user) except ObjectDoesNotExist: - ssh_keys = super( - SSHKeysManager, - self - ).create(user=user) + ssh_keys = super(SSHKeysManager, self).create(user=user) keys = HostKeys.objects.create(ssh_keys=ssh_keys, hostname=hostname) keys.public = pub_key keys.private = priv_key keys.save() - return super( - SSHKeysManager, - self - ).get_queryset().get(user=user) + return super(SSHKeysManager, self).get_queryset().get(user=user) class SSHKeys(models.Model): @@ -265,10 +214,8 @@ class SSHKeys(models.Model): developers think twice about doing something with this set of keys. """ - user = models.OneToOneField( - settings.AUTH_USER_MODEL, - related_name='ssh_keys', - on_delete=models.CASCADE) + + user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name="ssh_keys", on_delete=models.CASCADE) objects = SSHKeysManager() def for_system(self, system_id): @@ -336,7 +283,8 @@ class Keys(models.Model): if it changed. If it did then the save method will encrypt the key before saving it into the DB. """ - ssh_keys = models.ForeignKey(SSHKeys, related_name='+', on_delete=models.CASCADE) + + ssh_keys = models.ForeignKey(SSHKeys, related_name="+", on_delete=models.CASCADE) system = models.TextField(unique=True) private = models.TextField() public = models.TextField() @@ -356,17 +304,13 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ The keys need to be given as clear text strings and will be encrypted using AES """ - if (self.private != self._private or - self.pk is None): + if self.private != self._private or self.pk is None: self.private = EncryptionUtil.encrypt(self.private) super(Keys, self).save(*args, **kwargs) self._private = self.private def __str__(self): - return '{username}: {system}'.format( - username=self.ssh_keys.user.username, - system=self.system - ) + return "{username}: {system}".format(username=self.ssh_keys.user.username, system=self.system) class HostKeys(models.Model): @@ -378,12 +322,12 @@ class HostKeys(models.Model): """ hostname = models.TextField() - ssh_keys = models.ForeignKey(SSHKeys, related_name='+', on_delete=models.CASCADE) + ssh_keys = models.ForeignKey(SSHKeys, related_name="+", on_delete=models.CASCADE) private = models.TextField() public = models.TextField() class Meta: - unique_together = (('hostname', 'ssh_keys'),) + unique_together = (("hostname", "ssh_keys"),) def __init__(self, *args, **kwargs): super(HostKeys, self).__init__(*args, **kwargs) @@ -400,14 +344,10 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ The keys need to be given as clear text strings and will be encrypted using AES """ - if (self.private != self._private or - self.pk is None): + if self.private != self._private or self.pk is None: self.private = EncryptionUtil.encrypt(self.private) super(HostKeys, self).save(*args, **kwargs) self._private = self.private def __str__(self): - return '{username}: {host}'.format( - username=self.ssh_keys.user.username, - host=self.hostname - ) + return "{username}: {host}".format(username=self.ssh_keys.user.username, host=self.hostname) diff --git a/server/portal/apps/accounts/unit_test.py b/server/portal/apps/accounts/unit_test.py index 106c2b7c46..ca5a8e186d 100644 --- a/server/portal/apps/accounts/unit_test.py +++ b/server/portal/apps/accounts/unit_test.py @@ -5,20 +5,20 @@ def test_account_redirect(client): - response = client.get('/accounts/profile/') + response = client.get("/accounts/profile/") assert response.status_code == 302 - assert response.url == '/workbench/account/' + assert response.url == "/workbench/account/" @pytest.fixture def tas_user_history_request(requests_mock, authenticated_user): - history_url = f'{settings.TAS_URL}/v1/users/{authenticated_user.username}/history' + history_url = f"{settings.TAS_URL}/v1/users/{authenticated_user.username}/history" requests_mock.get(history_url, json={"status": "success", "result": "dummy"}) @pytest.fixture def tas_client(mocker): - tas_mock = mocker.patch('portal.apps.accounts.views.TASClient', autospec=True) + tas_mock = mocker.patch("portal.apps.accounts.views.TASClient", autospec=True) tas_client_mock = mocker.MagicMock() tas_client_mock.authenticate.return_value = True tas_mock.return_value = tas_client_mock @@ -26,25 +26,25 @@ def tas_client(mocker): def test_profile_data(client, tas_client, tas_user_history_request): - response = client.get('/accounts/api/profile/data/') + response = client.get("/accounts/api/profile/data/") assert response.status_code == 200 def test_profile_data_unauthenticated(client, tas_client): - response = client.get('/accounts/api/profile/data/') + response = client.get("/accounts/api/profile/data/") assert response.status_code == 302 # redirect to login def test_profile_data_unexpected(client, tas_client, tas_user_history_request): tas_client.get_user.side_effect = Exception - response = client.get('/accounts/api/profile/data/') + response = client.get("/accounts/api/profile/data/") assert response.status_code == 500 - assert response.json() == {'message': 'Unable to get profile.'} + assert response.json() == {"message": "Unable to get profile."} @pytest.mark.django_db def test_logout_redirects_correctly_and_logs_out(client, authenticated_user, settings): - response = client.get('/accounts/logout') + response = client.get("/accounts/logout") expected_url = "https://example.tapis.io/v3/oauth2/logout?redirect_url=https://testserver/cms/logout/" diff --git a/server/portal/apps/accounts/urls.py b/server/portal/apps/accounts/urls.py index 058f7ce67b..43648eaf4c 100644 --- a/server/portal/apps/accounts/urls.py +++ b/server/portal/apps/accounts/urls.py @@ -2,15 +2,16 @@ .. module:: portal.apps.accounts.urls :synopsis: Accounts URLs """ + from django.urls import re_path from portal.apps.accounts.views import LogoutView from portal.apps.accounts.views import accounts from portal.apps.accounts import views -app_name = 'portal_accounts' +app_name = "portal_accounts" urlpatterns = [ - re_path(r'^logout/?', LogoutView.as_view(), name='logout'), - re_path(r'^profile/', accounts, name='manage_profile'), - re_path(r'^api/profile/data/', views.get_profile_data, name='get_profile_data'), + re_path(r"^logout/?", LogoutView.as_view(), name="logout"), + re_path(r"^profile/", accounts, name="manage_profile"), + re_path(r"^api/profile/data/", views.get_profile_data, name="get_profile_data"), ] diff --git a/server/portal/apps/accounts/views.py b/server/portal/apps/accounts/views.py index 762f3c241f..374bed0859 100644 --- a/server/portal/apps/accounts/views.py +++ b/server/portal/apps/accounts/views.py @@ -1,6 +1,7 @@ """ Accounts views. """ + import logging import requests @@ -39,7 +40,7 @@ def dispatch(self, request, *args, **kwargs): def accounts(request): - response = redirect('/workbench/account/') + response = redirect("/workbench/account/") return response @@ -48,12 +49,12 @@ def get_user_history(username): Get user history from tas """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/users/{1}/history'.format(settings.TAS_URL, username), auth=auth) + r = requests.get("{0}/v1/users/{1}/history".format(settings.TAS_URL, username), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise Exception('Failed to get project users', resp['message']) + raise Exception("Failed to get project users", resp["message"]) @handle_uncaught_exceptions(message="Unable to get profile.") @@ -65,10 +66,7 @@ def get_profile_data(request): django_user = request.user tas = TASClient( baseURL=settings.TAS_URL, - credentials={ - 'username': settings.TAS_CLIENT_KEY, - 'password': settings.TAS_CLIENT_SECRET - } + credentials={"username": settings.TAS_CLIENT_KEY, "password": settings.TAS_CLIENT_SECRET}, ) user_profile = tas.get_user(username=request.user.username) @@ -78,13 +76,13 @@ def get_profile_data(request): demographics = model_to_dict(django_user.profile) except ObjectDoesNotExist as e: demographics = {} - logger.info('exception e:{} {}'.format(type(e), e)) + logger.info("exception e:{} {}".format(type(e), e)) demographics.update(user_profile) context = { - 'demographics': demographics, - 'history': history, - 'licenses': _manage_licenses(request), - 'integrations': _manage_integrations(request), + "demographics": demographics, + "history": history, + "licenses": _manage_licenses(request), + "integrations": _manage_integrations(request), } return JsonResponse(context) @@ -92,14 +90,15 @@ def get_profile_data(request): def _manage_licenses(request): from portal.apps.licenses.models import get_license_info + licenses, license_models = get_license_info() - licenses.sort(key=lambda x: x['license_type']) + licenses.sort(key=lambda x: x["license_type"]) license_models.sort(key=lambda x: x.license_type) for license, m in zip(licenses, license_models): if m.objects.filter(user=request.user).exists(): - license['current_user_license'] = True - license['template_html'] = render_to_string(license['details_html']) + license["current_user_license"] = True + license["template_html"] = render_to_string(license["details_html"]) return licenses diff --git a/server/portal/apps/auth/api/urls.py b/server/portal/apps/auth/api/urls.py index 7fccd9b5a2..8d534897f8 100644 --- a/server/portal/apps/auth/api/urls.py +++ b/server/portal/apps/auth/api/urls.py @@ -2,7 +2,7 @@ from portal.apps.auth.api.views import TapisToken -app_name = 'auth_api' +app_name = "auth_api" urlpatterns = [ - path('tapis/', TapisToken.as_view(), name='tapis_token'), + path("tapis/", TapisToken.as_view(), name="tapis_token"), ] diff --git a/server/portal/apps/auth/api/views.py b/server/portal/apps/auth/api/views.py index 08b93c779a..9e949e85da 100644 --- a/server/portal/apps/auth/api/views.py +++ b/server/portal/apps/auth/api/views.py @@ -19,9 +19,7 @@ def get(self, request): # By accessing client(), we ensure that there is a non-expired access_token which can be immediately used client = request.user.tapis_oauth.client - session_key_hash = sha256( - (request.session.session_key or "").encode() - ).hexdigest() + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() return JsonResponse( { diff --git a/server/portal/apps/auth/apps.py b/server/portal/apps/auth/apps.py index 82d33623ba..1a173b0ff3 100644 --- a/server/portal/apps/auth/apps.py +++ b/server/portal/apps/auth/apps.py @@ -2,7 +2,7 @@ class AuthConfig(AppConfig): - name = 'portal.apps.auth' - label = 'portal_auth' - app_label = 'portal_auth' - verbose_name = 'Portal Authentication' + name = "portal.apps.auth" + label = "portal_auth" + app_label = "portal_auth" + verbose_name = "Portal Authentication" diff --git a/server/portal/apps/auth/backends.py b/server/portal/apps/auth/backends.py index 75dd89a7ed..c6a7c59d6e 100644 --- a/server/portal/apps/auth/backends.py +++ b/server/portal/apps/auth/backends.py @@ -1,4 +1,5 @@ """Auth backends""" + import logging import requests from django.conf import settings @@ -12,21 +13,21 @@ class TapisOAuthBackend(ModelBackend): - def authenticate(self, *args, **kwargs): user = None - if 'backend' in kwargs and kwargs['backend'] == 'tapis': - token = kwargs['token'] + if "backend" in kwargs and kwargs["backend"] == "tapis": + token = kwargs["token"] - logger.info('Attempting login via Tapis with token "%s"' % - token[:8].ljust(len(token), '-')) + logger.info('Attempting login via Tapis with token "%s"' % token[:8].ljust(len(token), "-")) - response = requests.get(f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/userinfo", headers={"X-Tapis-Token": token}) + response = requests.get( + f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/userinfo", headers={"X-Tapis-Token": token} + ) json_result = response.json() - if 'status' in json_result and json_result['status'] == 'success': - tapis_user = json_result['result'] - username = tapis_user['username'] + if "status" in json_result and json_result["status"] == "success": + tapis_user = json_result["result"] + username = tapis_user["username"] UserModel = get_user_model() defaults = {} @@ -41,21 +42,18 @@ def authenticate(self, *args, **kwargs): } profile_defaults = { "institution": user_data.get("institution"), - "orcid_id": user_data.get("orcidId") + "orcid_id": user_data.get("orcidId"), } except Exception: - logger.exception( - "Error retrieving TAS user profile data for user: %s", username - ) + logger.exception("Error retrieving TAS user profile data for user: %s", username) user, created = UserModel.objects.update_or_create(username=username, defaults=defaults) if created: logger.info('Created local user record for "%s" from TAS Profile' % username) - PortalProfile.objects.update_or_create(user=user, - defaults=profile_defaults) + PortalProfile.objects.update_or_create(user=user, defaults=profile_defaults) logger.info('Login successful for user "%s"' % username) else: - logger.info('Tapis Authentication failed: %s' % json_result) + logger.info("Tapis Authentication failed: %s" % json_result) return user diff --git a/server/portal/apps/auth/middleware.py b/server/portal/apps/auth/middleware.py index 44de62687c..50dfee561f 100644 --- a/server/portal/apps/auth/middleware.py +++ b/server/portal/apps/auth/middleware.py @@ -52,16 +52,10 @@ def process_request(self, request): if not tapis_oauth.expired: return - logger.info( - f"Tapis OAuth token expired for user {request.user.username}. Refreshing token" - ) + logger.info(f"Tapis OAuth token expired for user {request.user.username}. Refreshing token") with transaction.atomic(): # Get a lock on this user's token row in db. - latest_token = ( - TapisOAuthToken.objects.select_for_update() - .filter(user=request.user) - .first() - ) + latest_token = TapisOAuthToken.objects.select_for_update().filter(user=request.user).first() if latest_token.expired: try: logger.info("Refreshing Tapis OAuth token") @@ -75,7 +69,5 @@ def process_request(self, request): return HttpResponseRedirect(reverse("login")) else: - logger.info( - "Token updated by another request. Refreshing token from DB." - ) + logger.info("Token updated by another request. Refreshing token from DB.") tapis_oauth.refresh_from_db() diff --git a/server/portal/apps/auth/migrations/0001_initial.py b/server/portal/apps/auth/migrations/0001_initial.py index 47496d2061..cbcbd8c113 100644 --- a/server/portal/apps/auth/migrations/0001_initial.py +++ b/server/portal/apps/auth/migrations/0001_initial.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -15,16 +14,23 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='AgaveOAuthToken', + name="AgaveOAuthToken", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('token_type', models.CharField(max_length=255)), - ('scope', models.CharField(max_length=255)), - ('access_token', models.CharField(max_length=255)), - ('refresh_token', models.CharField(max_length=255)), - ('expires_in', models.BigIntegerField()), - ('created', models.BigIntegerField()), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='agave_oauth', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("token_type", models.CharField(max_length=255)), + ("scope", models.CharField(max_length=255)), + ("access_token", models.CharField(max_length=255)), + ("refresh_token", models.CharField(max_length=255)), + ("expires_in", models.BigIntegerField()), + ("created", models.BigIntegerField()), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="agave_oauth", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py b/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py index 9241d43b31..069a5a5876 100644 --- a/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py +++ b/server/portal/apps/auth/migrations/0002_auto_20220920_2239.py @@ -6,25 +6,31 @@ class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('portal_auth', '0001_initial'), + ("portal_auth", "0001_initial"), ] operations = [ migrations.CreateModel( - name='TapisOAuthToken', + name="TapisOAuthToken", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('access_token', models.CharField(max_length=2048)), - ('refresh_token', models.CharField(max_length=2048)), - ('expires_in', models.BigIntegerField()), - ('created', models.BigIntegerField()), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='tapis_oauth', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("access_token", models.CharField(max_length=2048)), + ("refresh_token", models.CharField(max_length=2048)), + ("expires_in", models.BigIntegerField()), + ("created", models.BigIntegerField()), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="tapis_oauth", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.DeleteModel( - name='AgaveOAuthToken', + name="AgaveOAuthToken", ), ] diff --git a/server/portal/apps/auth/models.py b/server/portal/apps/auth/models.py index 9aca158174..7315bba90b 100644 --- a/server/portal/apps/auth/models.py +++ b/server/portal/apps/auth/models.py @@ -1,5 +1,4 @@ -"""Auth models -""" +"""Auth models""" import logging import time @@ -19,7 +18,8 @@ class TapisOAuthToken(models.Model): Use this class to store login details as well as refresh a token. """ - user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='tapis_oauth', on_delete=models.CASCADE) + + user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name="tapis_oauth", on_delete=models.CASCADE) access_token = models.CharField(max_length=2048) refresh_token = models.CharField(max_length=2048) expires_in = models.BigIntegerField() @@ -60,10 +60,10 @@ def token(self): :rtype: dict """ return { - 'access_token': self.access_token, - 'refresh_token': self.refresh_token, - 'created': self.created, - 'expires_in': self.expires_in + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "created": self.created, + "expires_in": self.expires_in, } @property @@ -79,9 +79,7 @@ def client(self) -> Tapis: :return: Tapis client using refresh token. :rtype: :class:Tapis """ - tenant_id = urlparse(getattr(settings, "TAPIS_TENANT_BASEURL")).hostname.split( - "." - )[0] + tenant_id = urlparse(getattr(settings, "TAPIS_TENANT_BASEURL")).hostname.split(".")[0] client = Tapis( base_url=getattr(settings, "TAPIS_TENANT_BASEURL"), @@ -110,4 +108,4 @@ def refresh_tokens(self): def __str__(self): access_token_masked = self.access_token[-5:] refresh_token_masked = self.refresh_token[-5:] - return f'access_token:{access_token_masked} refresh_token:{refresh_token_masked} expires_in:{self.expires_in} created:{self.created}' + return f"access_token:{access_token_masked} refresh_token:{refresh_token_masked} expires_in:{self.expires_in} created:{self.created}" diff --git a/server/portal/apps/auth/models_unit_test.py b/server/portal/apps/auth/models_unit_test.py index a889fe7db2..6d3b81f2fd 100644 --- a/server/portal/apps/auth/models_unit_test.py +++ b/server/portal/apps/auth/models_unit_test.py @@ -29,21 +29,18 @@ def user_without_client_mock(django_user_model, django_db_reset_sequences): """ user = django_user_model.objects.create_user(username="testuser2", password="password") TapisOAuthToken.objects.create( - user=user, - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user=user, access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447 + ) yield user def test_client_passes_tenant_id(user_without_client_mock, mocker): - mock_tapis = mocker.patch('portal.apps.auth.models.Tapis') + mock_tapis = mocker.patch("portal.apps.auth.models.Tapis") tapis_oauth = TapisOAuthToken.objects.get(user=user_without_client_mock) _ = tapis_oauth.client mock_tapis.assert_called_once_with( base_url=settings.TAPIS_TENANT_BASEURL, - tenant_id='example', + tenant_id="example", client_id=settings.TAPIS_CLIENT_ID, client_key=settings.TAPIS_CLIENT_KEY, access_token=tapis_oauth.access_token, diff --git a/server/portal/apps/auth/unit_test.py b/server/portal/apps/auth/unit_test.py index 73b57b1d64..8b48f0483c 100644 --- a/server/portal/apps/auth/unit_test.py +++ b/server/portal/apps/auth/unit_test.py @@ -1,7 +1,4 @@ -from django.test import ( - TransactionTestCase, - override_settings -) +from django.test import TransactionTestCase, override_settings from django.contrib.auth import get_user_model from mock import patch, MagicMock from portal.apps.auth.backends import TapisOAuthBackend @@ -13,12 +10,12 @@ def test_launch_setup_checks(mocker, regular_user, settings): - mocker.patch('portal.apps.auth.views.new_user_setup_check') - mocker.patch('portal.apps.auth.views.index_allocations') - mock_execute = mocker.patch('portal.apps.auth.views.execute_setup_steps') + mocker.patch("portal.apps.auth.views.new_user_setup_check") + mocker.patch("portal.apps.auth.views.index_allocations") + mock_execute = mocker.patch("portal.apps.auth.views.execute_setup_steps") regular_user.profile.setup_complete = False launch_setup_checks(regular_user) - mock_execute.apply_async.assert_called_with(args=['username']) + mock_execute.apply_async.assert_called_with(args=["username"]) class TestTapisOAuthBackend(TransactionTestCase): @@ -26,20 +23,12 @@ def setUp(self): super(TestTapisOAuthBackend, self).setUp() self.backend = TapisOAuthBackend() self.mock_response = MagicMock(autospec=Response) - self.mock_requests_patcher = patch( - 'portal.apps.auth.backends.requests.get', - return_value=self.mock_response - ) + self.mock_requests_patcher = patch("portal.apps.auth.backends.requests.get", return_value=self.mock_response) self.mock_requests = self.mock_requests_patcher.start() self.mock_user_data_patcher = patch( - 'portal.apps.auth.backends.get_user_data', - return_value={ - 'username': 'testuser', - 'firstName': 'test', - 'lastName': 'user', - 'email': 'new@email.com' - } + "portal.apps.auth.backends.get_user_data", + return_value={"username": "testuser", "firstName": "test", "lastName": "user", "email": "new@email.com"}, ) self.mock_user_data = self.mock_user_data_patcher.start() @@ -53,7 +42,7 @@ def test_bad_backend_params(self): result = self.backend.authenticate() self.assertIsNone(result) # Test TapisOAuthBackend if params do not indicate tapis - result = self.backend.authenticate(backend='not_tapis') + result = self.backend.authenticate(backend="not_tapis") self.assertIsNone(result) def test_bad_response_status(self): @@ -61,22 +50,17 @@ def test_bad_response_status(self): # Mock different return values for the backend response self.mock_response.json.return_value = {} - result = self.backend.authenticate(backend='tapis', token='1234') + result = self.backend.authenticate(backend="tapis", token="1234") self.assertIsNone(result) self.mock_response.json.return_value = {"status": "failure"} - result = self.backend.authenticate(backend='tapis', token='1234') + result = self.backend.authenticate(backend="tapis", token="1234") self.assertIsNone(result) @override_settings(PORTAL_USER_ACCOUNT_SETUP_STEPS=[]) def test_new_user(self): # Test that a new user is created and returned - self.mock_response.json.return_value = { - "status": "success", - "result": { - "username": "testuser" - } - } - result = self.backend.authenticate(backend='tapis', token='1234') + self.mock_response.json.return_value = {"status": "success", "result": {"username": "testuser"}} + result = self.backend.authenticate(backend="tapis", token="1234") self.assertEqual(result.username, "testuser") @override_settings(PORTAL_USER_ACCOUNT_SETUP_STEPS=[]) @@ -86,18 +70,15 @@ def test_update_existing_user(self): # Create a pre-existing user with the same username user = get_user_model().objects.create_user( - username="testuser", - first_name="test", - last_name="user", - email="old@email.com" + username="testuser", first_name="test", last_name="user", email="old@email.com" ) self.mock_response.json.return_value = { "status": "success", "result": { "username": "testuser", - } + }, } - result = self.backend.authenticate(backend='tapis', token='1234') + result = self.backend.authenticate(backend="tapis", token="1234") # Result user object should be the same self.assertEqual(result, user) # Existing user object should be updated diff --git a/server/portal/apps/auth/urls.py b/server/portal/apps/auth/urls.py index 16e92c3c0d..9583a79b64 100644 --- a/server/portal/apps/auth/urls.py +++ b/server/portal/apps/auth/urls.py @@ -2,13 +2,14 @@ .. module:: portal.apps.auth.urls :synopsis: Auth URls """ + from django.urls import re_path from portal.apps.auth import views -app_name = 'portal_auth' +app_name = "portal_auth" urlpatterns = [ - re_path(r'^logged-out/$', views.logged_out, name='logout'), - re_path(r'^session-lifetime/$', views.get_session_lifetime, name='session_lifetime'), - re_path(r'^tapis/$', views.tapis_oauth, name='tapis_oauth'), - re_path(r'^tapis/callback/$', views.tapis_oauth_callback, name='tapis_oauth_callback'), + re_path(r"^logged-out/$", views.logged_out, name="logout"), + re_path(r"^session-lifetime/$", views.get_session_lifetime, name="session_lifetime"), + re_path(r"^tapis/$", views.tapis_oauth, name="tapis_oauth"), + re_path(r"^tapis/callback/$", views.tapis_oauth_callback, name="tapis_oauth_callback"), ] diff --git a/server/portal/apps/auth/views.py b/server/portal/apps/auth/views.py index 5e56076249..aa02ca4e12 100644 --- a/server/portal/apps/auth/views.py +++ b/server/portal/apps/auth/views.py @@ -1,6 +1,7 @@ """ Auth views. """ + import logging import time import requests @@ -15,20 +16,17 @@ from django.utils import timezone from django.contrib.sessions.models import Session from .models import TapisOAuthToken -from portal.apps.onboarding.execute import ( - execute_setup_steps, - new_user_setup_check -) +from portal.apps.onboarding.execute import execute_setup_steps, new_user_setup_check from portal.apps.users.tasks import index_allocations from portal.apps.users.utils import check_user_groups from portal.utils import get_client_ip logger = logging.getLogger(__name__) -METRICS = logging.getLogger(f'metrics.{__name__}') +METRICS = logging.getLogger(f"metrics.{__name__}") def logged_out(request): - return render(request, 'portal/apps/auth/logged_out.html') + return render(request, "portal/apps/auth/logged_out.html") def get_session_seconds_left(request): @@ -44,12 +42,7 @@ def get_session_seconds_left(request): if not session_key: return 0 - expire_date = ( - Session.objects - .filter(session_key=session_key) - .values_list("expire_date", flat=True) - .first() - ) + expire_date = Session.objects.filter(session_key=session_key).values_list("expire_date", flat=True).first() if not expire_date: return 0 @@ -74,23 +67,22 @@ def _get_auth_state(): def tapis_oauth(request): - """First step for Tapis OAuth workflow. - """ + """First step for Tapis OAuth workflow.""" session = request.session - session['auth_state'] = _get_auth_state() - next_page = request.GET.get('next') + session["auth_state"] = _get_auth_state() + next_page = request.GET.get("next") if next_page: - session['next'] = next_page + session["next"] = next_page if request.is_secure(): - protocol = 'https' + protocol = "https" else: - protocol = 'http' + protocol = "http" redirect_uri = f"{protocol}://{request.get_host()}{reverse('portal_auth:tapis_oauth_callback')}" - tenant_base_url = getattr(settings, 'TAPIS_TENANT_BASEURL') - client_id = getattr(settings, 'TAPIS_CLIENT_ID') + tenant_base_url = getattr(settings, "TAPIS_TENANT_BASEURL") + client_id = getattr(settings, "TAPIS_CLIENT_ID") METRICS.debug(f"user:{request.user.username} starting oauth redirect login") # Authorization code request @@ -106,8 +98,7 @@ def tapis_oauth(request): def launch_setup_checks(user): - """Perform any onboarding checks or non-onboarding steps that may spawn celery tasks - """ + """Perform any onboarding checks or non-onboarding steps that may spawn celery tasks""" # Check onboarding settings if settings.IS_TACC_PORTAL: @@ -121,59 +112,64 @@ def launch_setup_checks(user): portal_roles = settings.PORTAL_ELEVATED_ROLES for role, groups_and_users in portal_roles.items(): if role == "is_staff" and not user.is_staff: - if str(user.username) in groups_and_users["usernames"] or check_user_groups(user, groups_and_users["groups"]): + if str(user.username) in groups_and_users["usernames"] or check_user_groups( + user, groups_and_users["groups"] + ): user.is_staff = True user.save() logger.info(f"user {user.username} is set to staff") elif role == "is_superuser" and not user.is_superuser: - if str(user.username) in groups_and_users["usernames"] or check_user_groups(user, groups_and_users["groups"]): + if str(user.username) in groups_and_users["usernames"] or check_user_groups( + user, groups_and_users["groups"] + ): user.is_superuser = True user.save() logger.info(f"user {user.username} is set to superuser") def tapis_oauth_callback(request): - """Tapis OAuth callback handler. - """ + """Tapis OAuth callback handler.""" - state = request.GET.get('state') + state = request.GET.get("state") - if request.session['auth_state'] != state: - msg = ( - 'OAuth Authorization State mismatch!? auth_state=%s ' - 'does not match returned state=%s' % ( - request.session['auth_state'], state - ) + if request.session["auth_state"] != state: + msg = "OAuth Authorization State mismatch!? auth_state=%s does not match returned state=%s" % ( + request.session["auth_state"], + state, ) logger.warning(msg) - return HttpResponseBadRequest('Authorization State Failed') + return HttpResponseBadRequest("Authorization State Failed") - if 'code' in request.GET: + if "code" in request.GET: # obtain a token for the user if request.is_secure(): - protocol = 'https' + protocol = "https" else: - protocol = 'http' + protocol = "http" redirect_uri = f"{protocol}://{request.get_host()}{reverse('portal_auth:tapis_oauth_callback')}" - code = request.GET['code'] + code = request.GET["code"] body = { - 'grant_type': 'authorization_code', - 'code': code, - 'redirect_uri': redirect_uri, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": redirect_uri, } - response = requests.post(f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/tokens", data=body, auth=(settings.TAPIS_CLIENT_ID, settings.TAPIS_CLIENT_KEY)) + response = requests.post( + f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/tokens", + data=body, + auth=(settings.TAPIS_CLIENT_ID, settings.TAPIS_CLIENT_KEY), + ) response_json = response.json() token_data = { - 'created': int(time.time()), - 'access_token': response_json['result']['access_token']['access_token'], - 'refresh_token': response_json['result']['refresh_token']['refresh_token'], - 'expires_in': response_json['result']['access_token']['expires_in'] + "created": int(time.time()), + "access_token": response_json["result"]["access_token"]["access_token"], + "refresh_token": response_json["result"]["refresh_token"]["refresh_token"], + "expires_in": response_json["result"]["access_token"]["expires_in"], } # log user in - user = authenticate(backend='tapis', token=token_data['access_token']) + user = authenticate(backend="tapis", token=token_data["access_token"]) if user: TapisOAuthToken.objects.update_or_create(user=user, defaults={**token_data}) @@ -194,20 +190,19 @@ def tapis_oauth_callback(request): else: messages.error( request, - 'Authentication failed. Please try again. If this problem ' - 'persists please submit a support ticket.' + "Authentication failed. Please try again. If this problem persists please submit a support ticket.", ) - return HttpResponseRedirect(reverse('portal_accounts:logout')) + return HttpResponseRedirect(reverse("portal_accounts:logout")) else: - if 'error' in request.GET: - error = request.GET['error'] - logger.warning('Authorization failed: %s' % error) + if "error" in request.GET: + error = request.GET["error"] + logger.warning("Authorization failed: %s" % error) - return HttpResponseRedirect(reverse('portal_accounts:logout')) + return HttpResponseRedirect(reverse("portal_accounts:logout")) - redirect = getattr(settings, 'LOGIN_REDIRECT_URL', '/') - if 'next' in request.session: - redirect += '?next=' + request.session.pop('next') + redirect = getattr(settings, "LOGIN_REDIRECT_URL", "/") + if "next" in request.session: + redirect += "?next=" + request.session.pop("next") response = HttpResponseRedirect(redirect) return response diff --git a/server/portal/apps/auth/views_unit_test.py b/server/portal/apps/auth/views_unit_test.py index ee60569f85..1fb6d35b70 100644 --- a/server/portal/apps/auth/views_unit_test.py +++ b/server/portal/apps/auth/views_unit_test.py @@ -27,9 +27,7 @@ def test_auth_tapis(client, mocker): def test_tapis_callback(client, mocker, regular_user, tapis_tokens_create_mock): mock_authenticate = mocker.patch("portal.apps.auth.views.authenticate") mock_tapis_token_post = mocker.patch("portal.apps.auth.views.requests.post") - mock_launch_setup_checks = mocker.patch( - "portal.apps.auth.views.launch_setup_checks" - ) + mock_launch_setup_checks = mocker.patch("portal.apps.auth.views.launch_setup_checks") # add auth to session session = client.session @@ -40,9 +38,7 @@ def test_tapis_callback(client, mocker, regular_user, tapis_tokens_create_mock): mock_tapis_token_post.return_value.status_code = 200 mock_authenticate.return_value = regular_user - response = client.get( - f"/auth/tapis/callback/?state={TEST_STATE}&code=83163624a0bc41c4a376e0acb16a62f9" - ) + response = client.get(f"/auth/tapis/callback/?state={TEST_STATE}&code=83163624a0bc41c4a376e0acb16a62f9") assert response.status_code == 302 assert response.url == settings.LOGIN_REDIRECT_URL assert mock_launch_setup_checks.call_count == 1 @@ -81,14 +77,10 @@ def test_session_lifetime_endpoint(client, regular_user): def test_launch_setup_checks(regular_user, mocker): - mock_execute_setup_steps = mocker.patch( - "portal.apps.auth.views.execute_setup_steps" - ) - mocker.patch('portal.apps.auth.views.index_allocations') + mock_execute_setup_steps = mocker.patch("portal.apps.auth.views.execute_setup_steps") + mocker.patch("portal.apps.auth.views.index_allocations") launch_setup_checks(regular_user) - mock_execute_setup_steps.apply_async.assert_called_with( - args=[regular_user.username] - ) + mock_execute_setup_steps.apply_async.assert_called_with(args=[regular_user.username]) def test_launch_setup_checks_already_onboarded(regular_user, mocker): diff --git a/server/portal/apps/datafiles/apps.py b/server/portal/apps/datafiles/apps.py index e1c580c061..6c12f6a129 100644 --- a/server/portal/apps/datafiles/apps.py +++ b/server/portal/apps/datafiles/apps.py @@ -2,7 +2,7 @@ class DatafilesConfig(AppConfig): - name = 'portal.apps.datafiles' - label = 'datafiles' - verbose_name = 'Datafiles' - app_label = 'datafiles' + name = "portal.apps.datafiles" + label = "datafiles" + verbose_name = "Datafiles" + app_label = "datafiles" diff --git a/server/portal/apps/datafiles/handlers/googledrive_handlers.py b/server/portal/apps/datafiles/handlers/googledrive_handlers.py index a769cbc010..bb4c3858ca 100644 --- a/server/portal/apps/datafiles/handlers/googledrive_handlers.py +++ b/server/portal/apps/datafiles/handlers/googledrive_handlers.py @@ -6,9 +6,9 @@ logger = logging.getLogger(__name__) allowed_actions = { - 'private': ['listing', 'search', 'copy'], - 'public': [], - 'community': [], + "private": ["listing", "search", "copy"], + "public": [], + "community": [], } @@ -19,8 +19,7 @@ def googledrive_get_handler(client, scheme, system, path, operation, **kwargs): return op(client, system, path, **kwargs) -def googledrive_put_handler(client, scheme, system, - path, operation, body=None): +def googledrive_put_handler(client, scheme, system, path, operation, body=None): if operation not in allowed_actions[scheme]: raise PermissionDenied diff --git a/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py b/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py index b5867c9f4f..8016bd7ea0 100644 --- a/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py +++ b/server/portal/apps/datafiles/handlers/googledrive_handlers_unit_test.py @@ -1,45 +1,28 @@ import pytest from django.core.exceptions import PermissionDenied -from portal.apps.datafiles.handlers.googledrive_handlers import \ - googledrive_get_handler, googledrive_put_handler +from portal.apps.datafiles.handlers.googledrive_handlers import googledrive_get_handler, googledrive_put_handler @pytest.fixture def mock_operations(mocker): - yield mocker.patch( - 'portal.apps.datafiles.handlers.googledrive_handlers.operations') + yield mocker.patch("portal.apps.datafiles.handlers.googledrive_handlers.operations") def test_get_handler(mock_googledrive_client, mock_operations): - googledrive_get_handler(mock_googledrive_client, 'private', - 'googledrive', - 'id1', - 'listing') - mock_operations.listing.assert_called_with(mock_googledrive_client, - 'googledrive', 'id1') + googledrive_get_handler(mock_googledrive_client, "private", "googledrive", "id1", "listing") + mock_operations.listing.assert_called_with(mock_googledrive_client, "googledrive", "id1") def test_get_handler_forbidden(mock_googledrive_client, mock_operations): with pytest.raises(PermissionDenied): - googledrive_get_handler(mock_googledrive_client, 'public', - 'googledrive', - 'id1', - 'listing') + googledrive_get_handler(mock_googledrive_client, "public", "googledrive", "id1", "listing") def test_put_handler(mock_googledrive_client, mock_operations): - googledrive_put_handler(mock_googledrive_client, 'private', - 'googledrive', - 'id1', - 'copy', body={'id': '1'}) - mock_operations.copy.assert_called_with(mock_googledrive_client, - 'googledrive', 'id1', - **{'id': '1'}) + googledrive_put_handler(mock_googledrive_client, "private", "googledrive", "id1", "copy", body={"id": "1"}) + mock_operations.copy.assert_called_with(mock_googledrive_client, "googledrive", "id1", **{"id": "1"}) def test_put_handler_forbidden(mock_googledrive_client, mock_operations): with pytest.raises(PermissionDenied): - googledrive_put_handler(mock_googledrive_client, 'public', - 'googledrive', - 'id1', - 'copy') + googledrive_put_handler(mock_googledrive_client, "public", "googledrive", "id1", "copy") diff --git a/server/portal/apps/datafiles/handlers/tapis_handlers.py b/server/portal/apps/datafiles/handlers/tapis_handlers.py index a1506b0c4c..4d2627cce5 100644 --- a/server/portal/apps/datafiles/handlers/tapis_handlers.py +++ b/server/portal/apps/datafiles/handlers/tapis_handlers.py @@ -6,12 +6,38 @@ logger = logging.getLogger(__name__) allowed_actions = { - 'private': ['listing', 'search', 'copy', 'download', 'mkdir', 'detail', - 'move', 'rename', 'trash', 'preview', 'upload', 'makepublic', 'delete'], - 'public': ['listing', 'search', 'copy', 'download', 'preview', 'detail'], - 'community': ['listing', 'search', 'copy', 'download', 'preview', 'detail'], - 'projects': ['listing', 'search', 'copy', 'download', 'mkdir', 'detail', - 'move', 'rename', 'trash', 'preview', 'upload', 'makepublic', 'upload_file_metadata'] + "private": [ + "listing", + "search", + "copy", + "download", + "mkdir", + "detail", + "move", + "rename", + "trash", + "preview", + "upload", + "makepublic", + "delete", + ], + "public": ["listing", "search", "copy", "download", "preview", "detail"], + "community": ["listing", "search", "copy", "download", "preview", "detail"], + "projects": [ + "listing", + "search", + "copy", + "download", + "mkdir", + "detail", + "move", + "rename", + "trash", + "preview", + "upload", + "makepublic", + "upload_file_metadata", + ], } @@ -20,13 +46,12 @@ def tapis_get_handler(client, scheme, system, path, operation, tapis_tracking_id raise PermissionDenied op = getattr(operations, operation) # Exclude .Trash directory from Public and Community Data listing and search - if scheme in ('public', 'community'): - kwargs['hideTrash'] = True + if scheme in ("public", "community"): + kwargs["hideTrash"] = True return op(client, system, path, tapis_tracking_id=tapis_tracking_id, scheme=scheme, **kwargs) -def tapis_post_handler(client, scheme, system, - path, operation, body=None, tapis_tracking_id=None): +def tapis_post_handler(client, scheme, system, path, operation, body=None, tapis_tracking_id=None): if operation not in allowed_actions[scheme]: raise PermissionDenied("") @@ -34,8 +59,7 @@ def tapis_post_handler(client, scheme, system, return op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body) -def tapis_put_handler(client, scheme, system, - path, operation, body=None, tapis_tracking_id=None): +def tapis_put_handler(client, scheme, system, path, operation, body=None, tapis_tracking_id=None): if operation not in allowed_actions[scheme]: raise PermissionDenied diff --git a/server/portal/apps/datafiles/migrations/0001_initial.py b/server/portal/apps/datafiles/migrations/0001_initial.py index ac08a9bfd9..301b4a225f 100644 --- a/server/portal/apps/datafiles/migrations/0001_initial.py +++ b/server/portal/apps/datafiles/migrations/0001_initial.py @@ -4,19 +4,17 @@ class Migration(migrations.Migration): - initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Link', + name="Link", fields=[ - ('agave_uri', models.TextField(primary_key=True, serialize=False)), - ('postit_url', models.TextField()), - ('updated', models.DateTimeField(auto_now=True)), + ("agave_uri", models.TextField(primary_key=True, serialize=False)), + ("postit_url", models.TextField()), + ("updated", models.DateTimeField(auto_now=True)), ], ), ] diff --git a/server/portal/apps/datafiles/migrations/0002_auto_20230317_2209.py b/server/portal/apps/datafiles/migrations/0002_auto_20230317_2209.py index 9c9950ec4e..135c99cfc9 100644 --- a/server/portal/apps/datafiles/migrations/0002_auto_20230317_2209.py +++ b/server/portal/apps/datafiles/migrations/0002_auto_20230317_2209.py @@ -4,20 +4,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('datafiles', '0001_initial'), + ("datafiles", "0001_initial"), ] operations = [ migrations.RenameField( - model_name='link', - old_name='agave_uri', - new_name='tapis_uri', + model_name="link", + old_name="agave_uri", + new_name="tapis_uri", ), migrations.AddField( - model_name='link', - name='expiration', + model_name="link", + name="expiration", field=models.DateTimeField(null=True), ), ] diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index d35f2ec64f..a393bfd974 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -2,6 +2,7 @@ .. :module:: apps.accounts.managers.models :synopsis: Account's models """ + from django.db import models @@ -12,12 +13,12 @@ class Link(models.Model): expiration = models.DateTimeField(null=True) def get_uuid(self): - return self.postit_url.split('/')[-1] + return self.postit_url.split("/")[-1] def to_dict(self): return { - 'tapis_uri': self.agave_uri, - 'postit_url': self.postit_url, - 'updated': str(self.updated), - 'expiration': str(self.expiration) + "tapis_uri": self.agave_uri, + "postit_url": self.postit_url, + "updated": str(self.updated), + "expiration": str(self.expiration), } diff --git a/server/portal/apps/datafiles/models_unit_test.py b/server/portal/apps/datafiles/models_unit_test.py index 5704522245..9946cd56c2 100644 --- a/server/portal/apps/datafiles/models_unit_test.py +++ b/server/portal/apps/datafiles/models_unit_test.py @@ -4,8 +4,5 @@ @pytest.mark.django_db def test_link_uuid(): - link = Link.objects.create( - tapis_uri="mock.system/path", - postit_url="https://tenant/postits/v2/listing/uuid" - ) + link = Link.objects.create(tapis_uri="mock.system/path", postit_url="https://tenant/postits/v2/listing/uuid") assert link.get_uuid() == "uuid" diff --git a/server/portal/apps/datafiles/urls.py b/server/portal/apps/datafiles/urls.py index 6c76d20bb0..68b3c844c1 100644 --- a/server/portal/apps/datafiles/urls.py +++ b/server/portal/apps/datafiles/urls.py @@ -1,25 +1,22 @@ from django.urls import path -from portal.apps.datafiles.views import (TapisFilesView, - GoogleDriveFilesView, - TransferFilesView, - LinkView, - SystemListingView, - SystemDefinitionView) +from portal.apps.datafiles.views import ( + TapisFilesView, + GoogleDriveFilesView, + TransferFilesView, + LinkView, + SystemListingView, + SystemDefinitionView, +) -app_name = 'users' +app_name = "users" urlpatterns = [ - path('systems/list/', SystemListingView.as_view()), - path('transfer//', TransferFilesView.as_view()), - path('systems/definition//', SystemDefinitionView.as_view()), - path('tapis////', - TapisFilesView.as_view()), - path('tapis/////', - TapisFilesView.as_view()), - path('googledrive////', - GoogleDriveFilesView.as_view()), - path('googledrive/////', - GoogleDriveFilesView.as_view()), - path('link///', - LinkView.as_view()) + path("systems/list/", SystemListingView.as_view()), + path("transfer//", TransferFilesView.as_view()), + path("systems/definition//", SystemDefinitionView.as_view()), + path("tapis////", TapisFilesView.as_view()), + path("tapis/////", TapisFilesView.as_view()), + path("googledrive////", GoogleDriveFilesView.as_view()), + path("googledrive/////", GoogleDriveFilesView.as_view()), + path("link///", LinkView.as_view()), ] diff --git a/server/portal/apps/datafiles/utils.py b/server/portal/apps/datafiles/utils.py index 3739d3ad50..e29dec10cf 100644 --- a/server/portal/apps/datafiles/utils.py +++ b/server/portal/apps/datafiles/utils.py @@ -56,9 +56,7 @@ def evaluate_datafiles_storage_system( if "homeDir" in system: home_dir_vars = {"username": tapis.user.username} if "{tasdir}" in system["homeDir"]: - home_dir_vars["tasdir"] = get_user_data(tapis.user.username)[ - "homeDirectory" - ] + home_dir_vars["tasdir"] = get_user_data(tapis.user.username)["homeDirectory"] evaluated_system = { **system, @@ -89,21 +87,15 @@ def evaluate_datafiles_storage_system( elif system["scheme"] == "projects": # For projects systems, determine resource provider based on projects host evaluation projects_host = settings.PORTAL_PROJECTS_ROOT_HOST - evaluated_system["resourceProvider"] = _get_resource_provider_from_host( - projects_host - ) + evaluated_system["resourceProvider"] = _get_resource_provider_from_host(projects_host) else: system_def = tapis.client.systems.getSystem(systemId=system["system"]) - evaluated_system["resourceProvider"] = _get_resource_provider_from_system( - system_def - ) + evaluated_system["resourceProvider"] = _get_resource_provider_from_system(system_def) return evaluated_system -def evaluate_datafiles_storage_systems( - tapis: TapisOAuthToken, systems: list, default_host_eval: str = None -) -> list: +def evaluate_datafiles_storage_systems(tapis: TapisOAuthToken, systems: list, default_host_eval: str = None) -> list: """Evaluate storage systems homeDir or hostEval for user Args: @@ -118,9 +110,7 @@ def evaluate_datafiles_storage_systems( evaluated_systems = [] for system in systems: try: - evaluated_systems.append( - evaluate_datafiles_storage_system(tapis, system, default_host_eval) - ) + evaluated_systems.append(evaluate_datafiles_storage_system(tapis, system, default_host_eval)) except (BaseTapyException, KeyError, AttributeError): logger.exception( "Error evaluating storage system %s for user %s", @@ -139,9 +129,7 @@ def get_user_storage_systems(tapis: TapisOAuthToken) -> list: list: List of evaluated storage system definitions """ logger.info("Getting user storage systems for user: %s", tapis.user.username) - systems = tapis.client.systems.getSystems( - listType="ALL", limit="-1", select="id,notes,host", orderBy="id" - ) + systems = tapis.client.systems.getSystems(listType="ALL", limit="-1", select="id,notes,host", orderBy="id") available_systems = [ { @@ -156,9 +144,7 @@ def get_user_storage_systems(tapis: TapisOAuthToken) -> list: for system in systems ] - return evaluate_datafiles_storage_systems( - tapis, available_systems, default_host_eval="HOME" - ) + return evaluate_datafiles_storage_systems(tapis, available_systems, default_host_eval="HOME") def _get_resource_provider_from_system(system: TapisResult) -> str: diff --git a/server/portal/apps/datafiles/views.py b/server/portal/apps/datafiles/views.py index 54f6620614..76109f16ed 100644 --- a/server/portal/apps/datafiles/views.py +++ b/server/portal/apps/datafiles/views.py @@ -9,12 +9,8 @@ from portal.views.base import BaseApiView from portal.utils import check_group_membership, get_client_ip from portal.libs.agave.utils import service_account -from portal.apps.datafiles.handlers.tapis_handlers import (tapis_get_handler, - tapis_put_handler, - tapis_post_handler) -from portal.apps.datafiles.handlers.googledrive_handlers import \ - (googledrive_get_handler, - googledrive_put_handler) +from portal.apps.datafiles.handlers.tapis_handlers import tapis_get_handler, tapis_put_handler, tapis_post_handler +from portal.apps.datafiles.handlers.googledrive_handlers import googledrive_get_handler, googledrive_put_handler from portal.libs.transfer.operations import transfer, transfer_folder from portal.libs.agave.serializers import BaseTapisResultSerializer from portal.exceptions.api import ApiException @@ -22,9 +18,7 @@ from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied from django.utils.decorators import method_decorator -from portal.apps.workspace.api.utils import ( - push_keys_required_if_not_credentials_ensured -) +from portal.apps.workspace.api.utils import push_keys_required_if_not_credentials_ensured from .utils import notify, NOTIFY_ACTIONS import dateutil.parser from portal.utils.decorators import retry @@ -44,9 +38,9 @@ def is_project_system(system): project_prefixes = tuple( prefix for prefix in ( - getattr(settings, 'PORTAL_PROJECTS_SYSTEM_PREFIX', None), - getattr(settings, 'PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX', None), - getattr(settings, 'PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX', None), + getattr(settings, "PORTAL_PROJECTS_SYSTEM_PREFIX", None), + getattr(settings, "PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX", None), + getattr(settings, "PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX", None), ) if prefix ) @@ -68,28 +62,32 @@ def get(self, request): response = {} if request.user.is_authenticated: - tapis_oauth = request.user.tapis_oauth if not portal_systems: - response['system_list'] = get_user_storage_systems(tapis_oauth) + response["system_list"] = get_user_storage_systems(tapis_oauth) default_system = None else: - response["system_list"] = evaluate_datafiles_storage_systems( - tapis_oauth, portal_systems - ) + response["system_list"] = evaluate_datafiles_storage_systems(tapis_oauth, portal_systems) - default_system = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM or settings.PORTAL_DATAFILES_STORAGE_SYSTEMS[0] + default_system = ( + settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM or settings.PORTAL_DATAFILES_STORAGE_SYSTEMS[0] + ) if default_system: - default_system_id = default_system.get('system') - system_def = request.user.tapis_oauth.client.systems.getSystem(systemId=default_system_id, select='host') - response['default_host'] = system_def.host - response['default_system_id'] = default_system_id + default_system_id = default_system.get("system") + system_def = request.user.tapis_oauth.client.systems.getSystem( + systemId=default_system_id, select="host" + ) + response["default_host"] = system_def.host + response["default_system_id"] = default_system_id else: - response['system_list'] = [sys for sys in portal_systems if sys['scheme'] == - 'public' or sys['system'] == settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME] + response["system_list"] = [ + sys + for sys in portal_systems + if sys["scheme"] == "public" or sys["system"] == settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + ] return JsonResponse(response) @@ -102,61 +100,66 @@ def get(self, request, systemId): client = get_tapis_client(request.user, systemId) except AttributeError: # Make sure that we only let unauth'd users see public systems - public_sys = next((sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['scheme'] == 'public'), None) - if public_sys and public_sys['system'] == systemId: + public_sys = next( + (sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["scheme"] == "public"), None + ) + if public_sys and public_sys["system"] == systemId: client = service_account() - elif settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX and systemId.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + elif settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX and systemId.startswith( + settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + ): client = service_account() else: - return JsonResponse({'message': 'Unauthorized'}, status=401) + return JsonResponse({"message": "Unauthorized"}, status=401) system_def = client.systems.getSystem(systemId=systemId) - return JsonResponse( - { - "status": 200, - "response": system_def - }, - encoder=BaseTapisResultSerializer - ) + return JsonResponse({"status": 200, "response": system_def}, encoder=BaseTapisResultSerializer) class TapisFilesView(BaseApiView): @retry(UnauthorizedError, tries=3, max_time=15) - def get(self, request, operation=None, scheme=None, system=None, path='/'): + def get(self, request, operation=None, scheme=None, system=None, path="/"): try: client = get_tapis_client(request.user, system) except AttributeError: # Make sure that we only let unauth'd users see public systems - public_sys = next((sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['scheme'] == 'public'), None) - if public_sys and public_sys['system'] == system and path.startswith(public_sys['homeDir'].strip('/')): + public_sys = next( + (sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["scheme"] == "public"), None + ) + if public_sys and public_sys["system"] == system and path.startswith(public_sys["homeDir"].strip("/")): client = service_account() - elif system and settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX and system.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + elif ( + system + and settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + and system.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX) + ): client = service_account() else: - return JsonResponse( - {'message': 'This data requires authentication to view.'}, - status=403) + return JsonResponse({"message": "This data requires authentication to view."}, status=403) try: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': operation, - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) - session_key_hash = sha256((request.session.session_key or '').encode()).hexdigest() + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": operation, + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "tapis", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() response = tapis_get_handler( - client, scheme, system, path, operation, tapis_tracking_id=f"portals.{session_key_hash}", **request.GET.dict()) + client, + scheme, + system, + path, + operation, + tapis_tracking_id=f"portals.{session_key_hash}", + **request.GET.dict(), + ) if operation in NOTIFY_ACTIONS: - notify( - request.user.username, operation, "success", {"response": response} - ) + notify(request.user.username, operation, "success", {"response": response}) except (InternalServerError, UnauthorizedError) as e: error_status = e.response.status_code if operation in NOTIFY_ACTIONS: @@ -166,9 +169,7 @@ def get(self, request, operation=None, scheme=None, system=None, path='/'): # In case of 500 determine cause system_def = client.systems.getSystem(systemId=system) - if settings.IS_TACC_PORTAL and not system_def.notes.get( - "noAllocationRequired" - ): + if settings.IS_TACC_PORTAL and not system_def.notes.get("noAllocationRequired"): # If user is missing a non-corral allocation mangle error to a 403 allocations = get_allocations(request.user.username) if not any( @@ -189,7 +190,7 @@ def get(self, request, operation=None, scheme=None, system=None, path='/'): ) # If the user has valid system credentials, retry the request - session_key_hash = sha256((request.session.session_key or '').encode()).hexdigest() + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() response = tapis_get_handler( client, scheme, @@ -212,8 +213,7 @@ def get(self, request, operation=None, scheme=None, system=None, path='/'): return JsonResponse({"data": response}) - def put(self, request, operation=None, scheme=None, - handler=None, system=None, path='/'): + def put(self, request, operation=None, scheme=None, handler=None, system=None, path="/"): body = json.loads(request.body) try: client = get_tapis_client(request.user, system) @@ -221,31 +221,34 @@ def put(self, request, operation=None, scheme=None, return HttpResponseForbidden("This data requires authentication to view.") try: - METRICS.info('Data Depot', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': operation, - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'scheme': scheme, - 'system': system, - 'path': path, - 'body': body, - } - }) - session_key_hash = sha256((request.session.session_key or '').encode()).hexdigest() - response = tapis_put_handler(client, scheme, system, path, operation, body, tapis_tracking_id=f"portals.{session_key_hash}") + METRICS.info( + "Data Depot", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": operation, + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "api": "tapis", + "scheme": scheme, + "system": system, + "path": path, + "body": body, + }, + }, + ) + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() + response = tapis_put_handler( + client, scheme, system, path, operation, body, tapis_tracking_id=f"portals.{session_key_hash}" + ) except Exception as exc: - operation in NOTIFY_ACTIONS and notify(request.user.username, operation, 'error', {}) + operation in NOTIFY_ACTIONS and notify(request.user.username, operation, "error", {}) raise exc return JsonResponse({"data": response}) - def post(self, request, operation=None, scheme=None, - handler=None, system=None, path='/'): + def post(self, request, operation=None, scheme=None, handler=None, system=None, path="/"): metadata_json = request.POST.get("metadata") metadata = json.loads(metadata_json) if metadata_json else None @@ -277,46 +280,48 @@ def post(self, request, operation=None, scheme=None, }, }, ) - session_key_hash = sha256((request.session.session_key or '').encode()).hexdigest() - response = tapis_post_handler(client, scheme, system, path, operation, { - **body, 'metadata': metadata}, tapis_tracking_id=f"portals.{session_key_hash}") + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() + response = tapis_post_handler( + client, + scheme, + system, + path, + operation, + {**body, "metadata": metadata}, + tapis_tracking_id=f"portals.{session_key_hash}", + ) except Exception as exc: - operation in NOTIFY_ACTIONS and notify(request.user.username, operation, 'error', {}) + operation in NOTIFY_ACTIONS and notify(request.user.username, operation, "error", {}) raise exc return JsonResponse({"data": response}) class GoogleDriveFilesView(BaseApiView): - def get(self, request, operation=None, scheme=None, system=None, - path='root'): + def get(self, request, operation=None, scheme=None, system=None, path="root"): try: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': operation, - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'googledrive', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": operation, + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "googledrive", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) client = request.user.googledrive_user_token.client except AttributeError: raise ApiException("Login Required", status=400) try: - response = googledrive_get_handler( - client, scheme, system, path, operation, **request.GET.dict()) + response = googledrive_get_handler(client, scheme, system, path, operation, **request.GET.dict()) except HTTPError as e: raise e - return JsonResponse({'data': response}) + return JsonResponse({"data": response}) - def put(self, request, operation=None, scheme=None, - handler=None, system=None, path='root'): + def put(self, request, operation=None, scheme=None, handler=None, system=None, path="root"): body = json.loads(request.body) try: @@ -325,10 +330,9 @@ def put(self, request, operation=None, scheme=None, return HttpResponseForbidden try: - response = googledrive_put_handler(client, scheme, system, path, - operation, body=body) + response = googledrive_put_handler(client, scheme, system, path, operation, body=body) except Exception as exc: - operation in NOTIFY_ACTIONS and notify(request.user.username, operation, 'error', {}) + operation in NOTIFY_ACTIONS and notify(request.user.username, operation, "error", {}) raise exc return JsonResponse({"data": response}) @@ -336,13 +340,13 @@ def put(self, request, operation=None, scheme=None, def get_client(user, api, system=None): client_mappings = { - 'tapis': 'tapis_oauth', - 'shared': 'tapis_oauth', - 'googledrive': 'googledrive_user_token', - 'box': 'box_user_token', - 'dropbox': 'dropbox_user_token' + "tapis": "tapis_oauth", + "shared": "tapis_oauth", + "googledrive": "googledrive_user_token", + "box": "box_user_token", + "dropbox": "dropbox_user_token", } - if api in ('tapis', 'shared') and is_project_system(system) and check_project_admin_group(user): + if api in ("tapis", "shared") and is_project_system(system) and check_project_admin_group(user): return service_account() return getattr(user, client_mappings[api]).client @@ -351,34 +355,34 @@ class TransferFilesView(BaseApiView): def put(self, request, filetype): body = json.loads(request.body) - src_client = get_client(request.user, body['src_api'], body.get('src_system')) - dest_client = get_client(request.user, body['dest_api'], body.get('dest_system')) - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': 'transfer', - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'body': body - } - }) + src_client = get_client(request.user, body["src_api"], body.get("src_system")) + dest_client = get_client(request.user, body["dest_api"], body.get("dest_system")) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "transfer", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"body": body}, + }, + ) try: - if filetype == 'dir': + if filetype == "dir": transfer_folder(src_client, dest_client, **body) else: transfer(src_client, dest_client, **body) - return JsonResponse({'success': True}) + return JsonResponse({"success": True}) except Exception as exc: logger.info(exc) - notify(request.user.username, 'copy', 'error', {}) + notify(request.user.username, "copy", "error", {}) raise exc -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class LinkView(BaseApiView): def create_postit(self, request, scheme, system, path): @@ -391,7 +395,7 @@ def create_postit(self, request, scheme, system, path): Link.objects.create( tapis_uri=f"{system}/{path}", postit_url=postit_redeem_url, - expiration=dateutil.parser.parse(postit.expiration) if postit.expiration else None + expiration=dateutil.parser.parse(postit.expiration) if postit.expiration else None, ) return {"data": postit_redeem_url, "expiration": postit.expiration} @@ -406,22 +410,19 @@ def delete_link(self, request, system, link): return "OK" def get(self, request, scheme, system, path): - """Given a file, returns a link for a file - """ + """Given a file, returns a link for a file""" try: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': 'retrieve-postit', - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "retrieve-postit", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "tapis", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) link = Link.objects.get(tapis_uri=f"{system}/{path}") except Link.DoesNotExist: return JsonResponse({"data": None, "expiration": None}) @@ -429,22 +430,19 @@ def get(self, request, scheme, system, path): return JsonResponse({"data": link.postit_url, "expiration": link.expiration}) def delete(self, request, scheme, system, path): - """Delete an existing link for a file - """ + """Delete an existing link for a file""" try: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': 'delete-postit', - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "delete-postit", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "tapis", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) link = Link.objects.get(tapis_uri=f"{system}/{path}") except Link.DoesNotExist: raise ApiException("Post-it does not exist") @@ -452,50 +450,44 @@ def delete(self, request, scheme, system, path): return JsonResponse({"data": response}) def post(self, request, scheme, system, path): - """Generates a new link for a file - """ + """Generates a new link for a file""" try: Link.objects.get(tapis_uri=f"{system}/{path}") except Link.DoesNotExist: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': 'create-postit', - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "create-postit", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "tapis", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) # Link doesn't exist - proceed with creating one postit = self.create_postit(request, scheme, system, path) - return JsonResponse({"data": postit['data'], "expiration": postit['expiration']}) + return JsonResponse({"data": postit["data"], "expiration": postit["expiration"]}) # Link for this file already exists, raise an exception raise ApiException("Link for this file already exists") def put(self, request, scheme, system, path): - """Replace an existing link for a file - """ + """Replace an existing link for a file""" try: - METRICS.info('Data Files', - extra={ - 'user': request.user.username, - 'sessionId': getattr(request.session, 'session_key', ''), - 'operation': 'replace-postit', - 'agent': request.META.get('HTTP_USER_AGENT'), - 'ip': get_client_ip(request), - 'info': { - 'api': 'tapis', - 'systemId': system, - 'filePath': path, - 'query': request.GET.dict()} - }) + METRICS.info( + "Data Files", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "replace-postit", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"api": "tapis", "systemId": system, "filePath": path, "query": request.GET.dict()}, + }, + ) link = Link.objects.get(tapis_uri=f"{system}/{path}") self.delete_link(request, system, link) except Link.DoesNotExist: raise ApiException("Could not find pre-existing link") postit = self.create_postit(request, scheme, system, path) - return JsonResponse({"data": postit['data'], "expiration": postit['expiration']}) + return JsonResponse({"data": postit["data"], "expiration": postit["expiration"]}) diff --git a/server/portal/apps/datafiles/views_unit_test.py b/server/portal/apps/datafiles/views_unit_test.py index c3958cacba..f2603c414a 100644 --- a/server/portal/apps/datafiles/views_unit_test.py +++ b/server/portal/apps/datafiles/views_unit_test.py @@ -11,23 +11,21 @@ from portal.apps.datafiles.models import Link from portal.apps.datafiles.views import get_tapis_client + pytestmark = pytest.mark.django_db @pytest.fixture def postits_create(mock_tapis_client): - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl="https://tenant/uuid", - expiration=None - ) + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl="https://tenant/uuid", expiration=None) yield mock_tapis_client.files.createPostIt @pytest.fixture def get_user_data(mocker): - mock = mocker.patch('portal.apps.datafiles.utils.get_user_data') - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_user.json')) as f: + mock = mocker.patch("portal.apps.datafiles.utils.get_user_data") + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_user.json")) as f: tas_user = json.load(f) mock.return_value = tas_user yield mock @@ -36,11 +34,11 @@ def get_user_data(mocker): def test_get_tapis_client_uses_service_account_for_project_admin(authenticated_user, mocker): group = Group.objects.create(name=settings.PROJECT_ADMIN_GROUP) authenticated_user.groups.add(group) - mock_service_account = mocker.patch('portal.apps.datafiles.views.service_account') + mock_service_account = mocker.patch("portal.apps.datafiles.views.service_account") client = get_tapis_client( authenticated_user, - f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123', + f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123", ) assert client == mock_service_account.return_value @@ -48,119 +46,85 @@ def test_get_tapis_client_uses_service_account_for_project_admin(authenticated_u def test_get_no_allocation(client, authenticated_user, mocker, monkeypatch, mock_tapis_client): - mock_tapis_get = mocker.patch('portal.apps.datafiles.views.tapis_get_handler') + mock_tapis_get = mocker.patch("portal.apps.datafiles.views.tapis_get_handler") mock_error = InternalServerError() - monkeypatch.setattr( - mock_error, 'response', MagicMock( - json=MagicMock(return_value={}), - status_code=500 - ) - ) + monkeypatch.setattr(mock_error, "response", MagicMock(json=MagicMock(return_value={}), status_code=500)) mock_tapis_get.side_effect = mock_error - mock_get_allocations = mocker.patch('portal.apps.datafiles.views.get_allocations') - mock_get_allocations.return_value = { - 'hosts': {} - } + mock_get_allocations = mocker.patch("portal.apps.datafiles.views.get_allocations") + mock_get_allocations.return_value = {"hosts": {}} - mock_tapis_client.systems.getSystem.return_value = TapisResult(host='frontera.tacc.utexas.edu', notes={}) + mock_tapis_client.systems.getSystem.return_value = TapisResult(host="frontera.tacc.utexas.edu", notes={}) - response = client.get('/api/datafiles/tapis/listing/private/frontera.home.username/') + response = client.get("/api/datafiles/tapis/listing/private/frontera.home.username/") assert response.status_code == 403 def test_ignore_missing_corral(client, authenticated_user, mocker, monkeypatch, mock_tapis_client): - mock_tapis_get = mocker.patch('portal.apps.datafiles.views.tapis_get_handler') + mock_tapis_get = mocker.patch("portal.apps.datafiles.views.tapis_get_handler") mock_error = InternalServerError() - monkeypatch.setattr( - mock_error, 'response', MagicMock( - json=MagicMock(return_value={}), - status_code=500 - ) - ) + monkeypatch.setattr(mock_error, "response", MagicMock(json=MagicMock(return_value={}), status_code=500)) mock_tapis_get.side_effect = mock_error - mock_get_allocations = mocker.patch('portal.apps.datafiles.views.get_allocations') - mock_get_allocations.return_value = { - 'hosts': {} - } + mock_get_allocations = mocker.patch("portal.apps.datafiles.views.get_allocations") + mock_get_allocations.return_value = {"hosts": {}} - mock_tapis_client.systems.getSystem.return_value = TapisResult(host='data.tacc.utexas.edu') + mock_tapis_client.systems.getSystem.return_value = TapisResult(host="data.tacc.utexas.edu") - response = client.get('/api/datafiles/tapis/listing/private/corral.home.username/') + response = client.get("/api/datafiles/tapis/listing/private/corral.home.username/") assert response.status_code == 500 def test_get_requires_push_keys(client, authenticated_user, mocker, monkeypatch, mock_tapis_client): - mock_tapis_get = mocker.patch('portal.apps.datafiles.views.tapis_get_handler') + mock_tapis_get = mocker.patch("portal.apps.datafiles.views.tapis_get_handler") mock_tapis_client.systems.checkUserCredential.side_effect = UnauthorizedError() mock_tapis_client.files.listFiles.side_effect = UnauthorizedError() mock_error = InternalServerError() - monkeypatch.setattr( - mock_error, 'response', MagicMock( - json=MagicMock(return_value={}), - status_code=500 - ) - ) + monkeypatch.setattr(mock_error, "response", MagicMock(json=MagicMock(return_value={}), status_code=500)) mock_tapis_get.side_effect = mock_error - mock_get_allocations = mocker.patch('portal.apps.datafiles.views.get_allocations') - mock_get_allocations.return_value = { - 'hosts': {'frontera.tacc.utexas.edu': []} - } + mock_get_allocations = mocker.patch("portal.apps.datafiles.views.get_allocations") + mock_get_allocations.return_value = {"hosts": {"frontera.tacc.utexas.edu": []}} system = { - 'host': 'frontera.tacc.utexas.edu', - 'defaultAuthnMethod': 'PKI_KEYS', + "host": "frontera.tacc.utexas.edu", + "defaultAuthnMethod": "PKI_KEYS", "effectiveUserId": authenticated_user.username, - "notes": {} + "notes": {}, } mock_tapis_client.systems.getSystem.return_value = TapisResult(**system) - response = client.get('/api/datafiles/tapis/listing/private/frontera.home.username/') + response = client.get("/api/datafiles/tapis/listing/private/frontera.home.username/") assert response.status_code == 500 - assert response.json() == {'system': system} + assert response.json() == {"system": system} def test_get_link(client, authenticated_user): - Link.objects.create( - tapis_uri="system/path", - postit_url="https://postit" - ) + Link.objects.create(tapis_uri="system/path", postit_url="https://postit") response = client.get("/api/datafiles/link/tapis/system/path") result = json.loads(response.content) - assert result['data'] == "https://postit" + assert result["data"] == "https://postit" def test_link_not_found(client, authenticated_user): response = client.get("/api/datafiles/link/tapis/system/notfound") result = json.loads(response.content) - assert result['data'] is None + assert result["data"] is None def test_link_post(postits_create, authenticated_user, client): result = client.post("/api/datafiles/link/tapis/system/path") assert json.loads(result.content)["data"] == "https://tenant/uuid" assert Link.objects.all()[0].get_uuid() == "uuid" - postits_create.assert_called_with( - systemId="system", - path="path", - allowedUses=-1, - validSeconds=31536000 - ) + postits_create.assert_called_with(systemId="system", path="path", allowedUses=-1, validSeconds=31536000) def test_link_post_already_exists(postits_create, authenticated_user, client): result = client.post("/api/datafiles/link/tapis/system/path") assert json.loads(result.content)["data"] == "https://tenant/uuid" assert Link.objects.all()[0].get_uuid() == "uuid" - postits_create.assert_called_with( - systemId="system", - path="path", - allowedUses=-1, - validSeconds=31536000 - ) + postits_create.assert_called_with(systemId="system", path="path", allowedUses=-1, validSeconds=31536000) result = client.post("/api/datafiles/link/tapis/system/path") assert result.status_code == 400 assert result.json() == {"message": "Link for this file already exists"} @@ -184,10 +148,7 @@ def test_link_delete_dne(authenticated_user, mock_tapis_client, client): def test_link_put(postits_create, authenticated_user, mock_tapis_client, client): mock_tapis_client.files.deletePostIt.return_value = "OK" - Link.objects.create( - tapis_uri="system/path", - postit_url="https://tenant/olduuid" - ) + Link.objects.create(tapis_uri="system/path", postit_url="https://tenant/olduuid") result = client.put("/api/datafiles/link/tapis/system/path") assert json.loads(result.content)["data"] == "https://tenant/uuid" assert Link.objects.all()[0].get_uuid() == "uuid" @@ -217,13 +178,14 @@ def test_get_system_forbidden(client, regular_user, mock_tapis_client, agave_sto @pytest.fixture def logging_metric_mock(mocker): - logger = logging.getLogger('metrics.{}'.format("portal.apps.datafiles.views")) - yield mocker.patch.object(logger, 'info') + logger = logging.getLogger("metrics.{}".format("portal.apps.datafiles.views")) + yield mocker.patch.object(logger, "info") -@patch('portal.libs.agave.operations.tapis_listing_indexer') -def test_tapis_file_view_get_is_logged_for_metrics(mock_indexer, client, authenticated_user, mock_tapis_client, - tapis_file_listing_mock, logging_metric_mock): +@patch("portal.libs.agave.operations.tapis_listing_indexer") +def test_tapis_file_view_get_is_logged_for_metrics( + mock_indexer, client, authenticated_user, mock_tapis_client, tapis_file_listing_mock, logging_metric_mock +): tapis_listing_result = [TapisResult(**f) for f in tapis_file_listing_mock] mock_tapis_client.files.listFiles.return_value = tapis_listing_result response = client.get("/api/datafiles/tapis/listing/private/frontera.home.username/test.txt/?length=1234") @@ -232,23 +194,22 @@ def test_tapis_file_view_get_is_logged_for_metrics(mock_indexer, client, authent "data": { "listing": [ { - 'uuid': None, - 'system': 'frontera.home.username', - 'type': 'dir' if f.type == 'dir' else 'file', - 'format': 'folder' if f.type == 'dir' else 'raw', - 'mimeType': f.mimeType, - 'path': f.path, - 'name': f.name, - 'length': f.size, - 'lastModified': f.lastModified, - '_links': { - 'self': {'href': f.url} - }, - 'metadata': None - } for f in tapis_listing_result + "uuid": None, + "system": "frontera.home.username", + "type": "dir" if f.type == "dir" else "file", + "format": "folder" if f.type == "dir" else "raw", + "mimeType": f.mimeType, + "path": f.path, + "name": f.name, + "length": f.size, + "lastModified": f.lastModified, + "_links": {"self": {"href": f.url}}, + "metadata": None, + } + for f in tapis_listing_result ], "folder_metadata": None, - "reachedEnd": True + "reachedEnd": True, } } @@ -256,7 +217,7 @@ def test_tapis_file_view_get_is_logged_for_metrics(mock_indexer, client, authent logging_metric_mock.assert_called() -@patch('portal.libs.agave.operations.tapis_indexer') +@patch("portal.libs.agave.operations.tapis_indexer") @patch( "django.conf.settings.PORTAL_DATAFILES_STORAGE_SYSTEMS", [{"scheme": "public", "system": "public.system", "homeDir": "/public/home/"}], @@ -268,43 +229,46 @@ def test_tapis_file_view_get_unauthorized( mock_user = MagicMock() mock_user.tapis_oauth = 0 - with patch('django.contrib.auth.get_user', return_value=mock_user): + with patch("django.contrib.auth.get_user", return_value=mock_user): response = client.get("/api/datafiles/tapis/listing/private/frontera.home.username/test.txt/?length=1234") assert response.status_code == 403 - assert response.json() == {'message': 'This data requires authentication to view.'} + assert response.json() == {"message": "This data requires authentication to view."} -@patch('portal.libs.agave.operations.tapis_indexer') -def test_tapis_file_view_put_is_logged_for_metrics(mock_indexer, client, authenticated_user, mock_tapis_client, - tapis_file_listing_mock, logging_metric_mock): - mock_response = {'status': 'success'} +@patch("portal.libs.agave.operations.tapis_indexer") +def test_tapis_file_view_put_is_logged_for_metrics( + mock_indexer, client, authenticated_user, mock_tapis_client, tapis_file_listing_mock, logging_metric_mock +): + mock_response = {"status": "success"} mock_tapis_client.files.moveCopy.return_value = mock_response body = {"dest_path": "/testfol", "dest_system": "frontera.home.username"} - response = client.put("/api/datafiles/tapis/move/private/frontera.home.username/test.txt/", - content_type="application/json", - data=body) + response = client.put( + "/api/datafiles/tapis/move/private/frontera.home.username/test.txt/", content_type="application/json", data=body + ) assert response.status_code == 200 # Ensure metric-related logging is being performed logging_metric_mock.assert_called() -@patch('portal.libs.agave.operations.tapis_indexer') -@patch('portal.apps.datafiles.views.tapis_put_handler') -def test_tapis_file_view_put_is_logged_for_metrics_exception(mock_put_handler, mock_indexer, client, authenticated_user, mock_tapis_client): +@patch("portal.libs.agave.operations.tapis_indexer") +@patch("portal.apps.datafiles.views.tapis_put_handler") +def test_tapis_file_view_put_is_logged_for_metrics_exception( + mock_put_handler, mock_indexer, client, authenticated_user, mock_tapis_client +): mock_put_handler.side_effect = Exception("Exception in Metrics info or Tapis Put Handler views.py:142") body = {"dest_path": "/testfol", "dest_system": "frontera.home.username"} - response = client.put("/api/datafiles/tapis/move/private/frontera.home.username/test.txt/", - content_type="application/json", - data=body) + response = client.put( + "/api/datafiles/tapis/move/private/frontera.home.username/test.txt/", content_type="application/json", data=body + ) assert response.status_code == 500 -@patch('portal.libs.agave.operations.tapis_indexer') +@patch("portal.libs.agave.operations.tapis_indexer") def test_tapis_file_view_put_is_unauthorized(mock_indexer, client): mock_user = MagicMock() mock_user.tapis_oauth = 0 - with patch('django.contrib.auth.get_user', return_value=mock_user): + with patch("django.contrib.auth.get_user", return_value=mock_user): body = {"dest_path": "/testfol", "dest_system": "frontera.home.username"} response = client.put( "/api/datafiles/tapis/move/private/frontera.home.username/test.txt/", @@ -315,17 +279,26 @@ def test_tapis_file_view_put_is_unauthorized(mock_indexer, client): assert response.content == b"This data requires authentication to view." -@patch('portal.libs.agave.operations.tapis_indexer') -@patch('portal.libs.agave.operations.httpx') -def test_tapis_file_view_post_is_logged_for_metrics(mock_httpx, mock_indexer, client, authenticated_user, mock_tapis_client, - logging_metric_mock, - tapis_file_mock, requests_mock, text_file_fixture): +@patch("portal.libs.agave.operations.tapis_indexer") +@patch("portal.libs.agave.operations.httpx") +def test_tapis_file_view_post_is_logged_for_metrics( + mock_httpx, + mock_indexer, + client, + authenticated_user, + mock_tapis_client, + logging_metric_mock, + tapis_file_mock, + requests_mock, + text_file_fixture, +): mock_tapis_client.files.insert.return_value = tapis_file_mock mock_httpx.post.return_value.json.return_value = {"result": "OK"} - response = client.post("/api/datafiles/tapis/upload/private/frontera.home.username/", - data={"uploaded_file": text_file_fixture}) + response = client.post( + "/api/datafiles/tapis/upload/private/frontera.home.username/", data={"uploaded_file": text_file_fixture} + ) assert response.status_code == 200 # assert response.json() == {"data": tapis_file_mock} @@ -334,25 +307,37 @@ def test_tapis_file_view_post_is_logged_for_metrics(mock_httpx, mock_indexer, cl logging_metric_mock.assert_called() -@patch('portal.libs.agave.operations.tapis_indexer') +@patch("portal.libs.agave.operations.tapis_indexer") def test_tapis_file_view_post_is_unauthorized(mock_indexer, text_file_fixture, client): mock_user = MagicMock() mock_user.tapis_oauth = 0 - with patch('django.contrib.auth.get_user', return_value=mock_user): - response = client.post("/api/datafiles/tapis/upload/private/frontera.home.username/", data={"uploaded_file": text_file_fixture}) + with patch("django.contrib.auth.get_user", return_value=mock_user): + response = client.post( + "/api/datafiles/tapis/upload/private/frontera.home.username/", data={"uploaded_file": text_file_fixture} + ) assert response.status_code == 403 assert response.content == b"This data requires authentication to upload." -@patch('portal.libs.agave.operations.tapis_indexer') -@patch('portal.apps.datafiles.views.tapis_post_handler') -def test_tapis_file_view_post_is_logged_for_metrics_exception(mock_post_handler, mock_indexer, client, authenticated_user, mock_tapis_client, - logging_metric_mock, tapis_file_mock, requests_mock, text_file_fixture): +@patch("portal.libs.agave.operations.tapis_indexer") +@patch("portal.apps.datafiles.views.tapis_post_handler") +def test_tapis_file_view_post_is_logged_for_metrics_exception( + mock_post_handler, + mock_indexer, + client, + authenticated_user, + mock_tapis_client, + logging_metric_mock, + tapis_file_mock, + requests_mock, + text_file_fixture, +): mock_post_handler.side_effect = Exception("Exception in Metrics info or Tapis Put Handler views.py:175") mock_tapis_client.files.insert.return_value = tapis_file_mock - response = client.post("/api/datafiles/tapis/upload/private/frontera.home.username/", - data={"uploaded_file": text_file_fixture}) + response = client.post( + "/api/datafiles/tapis/upload/private/frontera.home.username/", data={"uploaded_file": text_file_fixture} + ) assert response.status_code == 500 @@ -360,151 +345,182 @@ def test_tapis_file_view_post_is_logged_for_metrics_exception(mock_post_handler, POSTIT_HREF = "https://tapis.example/postit/something" -@pytest.mark.parametrize("EXTENSION,TYPE", [("PNG", "image", ), ("JPG", "image"), ("jpeg", "image"), - ("doc", "ms-office"), ("docx", "ms-office"), - ("pdf", "object")]) -def test_tapis_file_view_preview_supported_non_text_files(client, authenticated_user, mock_tapis_client, - agave_file_listing_mock, EXTENSION, TYPE): +@pytest.mark.parametrize( + "EXTENSION,TYPE", + [ + ( + "PNG", + "image", + ), + ("JPG", "image"), + ("jpeg", "image"), + ("doc", "ms-office"), + ("docx", "ms-office"), + ("pdf", "object"), + ], +) +def test_tapis_file_view_preview_supported_non_text_files( + client, authenticated_user, mock_tapis_client, agave_file_listing_mock, EXTENSION, TYPE +): mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl=POSTIT_HREF, - expiration=None + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) + response = client.put( + "/api/datafiles/tapis/preview/private/frontera.home.username/test_text.{}/".format(EXTENSION), + content_type="application/json", + data={"href": "https//tapis.example/href"}, ) - response = client.put("/api/datafiles/tapis/preview/private/frontera.home.username/test_text.{}/".format(EXTENSION), - content_type="application/json", - data={"href": "https//tapis.example/href"}) - href = POSTIT_HREF if TYPE != "ms-office" \ + href = ( + POSTIT_HREF + if TYPE != "ms-office" else "https://view.officeapps.live.com/op/view.aspx?src={}".format(POSTIT_HREF) + ) assert response.status_code == 200 - assert response.json() == {"data": {"href": href, "fileType": TYPE, 'content': None, 'error': None}} + assert response.json() == {"data": {"href": href, "fileType": TYPE, "content": None, "error": None}} -def test_tapis_file_view_preview_text_file(client, authenticated_user, mock_tapis_client, agave_file_listing_mock, - requests_mock): +def test_tapis_file_view_preview_text_file( + client, authenticated_user, mock_tapis_client, agave_file_listing_mock, requests_mock +): mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl=POSTIT_HREF, - expiration=None - ) + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) requests_mock.get(POSTIT_HREF, text="file content") - response = client.put("/api/datafiles/tapis/preview/private/frontera.home.username/test_text.txt/", - content_type="application/json", - data={"href": "https//tapis.example/href"}) + response = client.put( + "/api/datafiles/tapis/preview/private/frontera.home.username/test_text.txt/", + content_type="application/json", + data={"href": "https//tapis.example/href"}, + ) assert response.status_code == 200 - assert response.json() == {"data": {"href": POSTIT_HREF, "fileType": "text", "content": "file content", "error": None}} + assert response.json() == { + "data": {"href": POSTIT_HREF, "fileType": "text", "content": "file content", "error": None} + } -def test_tapis_file_view_preview_other_text_file(client, authenticated_user, mock_tapis_client, agave_file_listing_mock, - requests_mock): +def test_tapis_file_view_preview_other_text_file( + client, authenticated_user, mock_tapis_client, agave_file_listing_mock, requests_mock +): mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl=POSTIT_HREF, - expiration=None - ) + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) requests_mock.get(POSTIT_HREF, text="file content") - response = client.put("/api/datafiles/tapis/preview/private/frontera.home.username/some_other_txt_file_like_log.applog/", - content_type="application/json", - data={"href": "https//tapis.example/href"}) + response = client.put( + "/api/datafiles/tapis/preview/private/frontera.home.username/some_other_txt_file_like_log.applog/", + content_type="application/json", + data={"href": "https//tapis.example/href"}, + ) assert response.status_code == 200 - assert response.json() == {"data": {"href": POSTIT_HREF, "fileType": "other", "content": "file content", "error": None}} + assert response.json() == { + "data": {"href": POSTIT_HREF, "fileType": "other", "content": "file content", "error": None} + } -def test_tapis_file_view_preview_unsupported_file(client, authenticated_user, mock_tapis_client, agave_file_listing_mock, - requests_mock): +def test_tapis_file_view_preview_unsupported_file( + client, authenticated_user, mock_tapis_client, agave_file_listing_mock, requests_mock +): mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl=POSTIT_HREF, - expiration=None - ) + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) requests_mock.get(POSTIT_HREF, text="file content") - response = client.put("/api/datafiles/tapis/preview/private/frontera.home.username/test.html/", - content_type="application/json", - data={"href": "https//tapis.example/href"}) + response = client.put( + "/api/datafiles/tapis/preview/private/frontera.home.username/test.html/", + content_type="application/json", + data={"href": "https//tapis.example/href"}, + ) assert response.status_code == 200 - assert response.json() == {"data": {"href": POSTIT_HREF, "fileType": None, "content": None, "error": "This file type must be previewed in a new window."}} + assert response.json() == { + "data": { + "href": POSTIT_HREF, + "fileType": None, + "content": None, + "error": "This file type must be previewed in a new window.", + } + } -def test_tapis_file_view_preview_large_file(client, authenticated_user, mock_tapis_client, agave_file_listing_mock, - requests_mock): +def test_tapis_file_view_preview_large_file( + client, authenticated_user, mock_tapis_client, agave_file_listing_mock, requests_mock +): agave_file_listing_mock[0]["size"] = 5000000 mock_tapis_client.files.listFiles.return_value = [TapisResult(**f) for f in agave_file_listing_mock] - mock_tapis_client.files.createPostIt.return_value = TapisResult( - redeemUrl=POSTIT_HREF, - expiration=None - ) + mock_tapis_client.files.createPostIt.return_value = TapisResult(redeemUrl=POSTIT_HREF, expiration=None) requests_mock.get(POSTIT_HREF, text="file content") - response = client.put("/api/datafiles/tapis/preview/private/frontera.home.username/test.log/", - content_type="application/json", - data={"href": "https//tapis.example/href"}) + response = client.put( + "/api/datafiles/tapis/preview/private/frontera.home.username/test.log/", + content_type="application/json", + data={"href": "https//tapis.example/href"}, + ) assert response.status_code == 200 - assert response.json() == {"data": {"href": POSTIT_HREF, "fileType": 'other', "content": None, "error": "File too large to preview in this window."}} + assert response.json() == { + "data": { + "href": POSTIT_HREF, + "fileType": "other", + "content": None, + "error": "File too large to preview in this window.", + } + } def test_systems_list(client, authenticated_user, mock_tapis_client, agave_storage_system_mock, get_user_data): mock_tapis_client.systems.getSystem.return_value = TapisResult(**agave_storage_system_mock) - response = client.get('/api/datafiles/systems/list/') + response = client.get("/api/datafiles/systems/list/") assert response.json() == { "default_host": "cloud.data.tacc.utexas.edu", "default_system_id": "cloud.data", "system_list": [ { - 'name': 'My Data (Work)', - 'system': 'cloud.data', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/home/username', - 'icon': None, - 'default': True, - 'resourceProvider': 'TACC', - + "name": "My Data (Work)", + "system": "cloud.data", + "scheme": "private", + "api": "tapis", + "homeDir": "/home/username", + "icon": None, + "default": True, + "resourceProvider": "TACC", }, { - 'name': 'My Data (Frontera)', - 'system': 'frontera', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/home1/01234/username', - 'icon': None, - 'resourceProvider': 'TACC', + "name": "My Data (Frontera)", + "system": "frontera", + "scheme": "private", + "api": "tapis", + "homeDir": "/home1/01234/username", + "icon": None, + "resourceProvider": "TACC", }, { - 'name': 'Community Data', - 'system': 'cloud.data', - 'scheme': 'community', - 'api': 'tapis', - 'homeDir': '/path/to/community', - 'icon': None, - 'siteSearchPriority': 1, - 'resourceProvider': 'TACC', + "name": "Community Data", + "system": "cloud.data", + "scheme": "community", + "api": "tapis", + "homeDir": "/path/to/community", + "icon": None, + "siteSearchPriority": 1, + "resourceProvider": "TACC", }, { - 'name': 'Public Data', - 'system': 'cloud.data', - 'scheme': 'public', - 'api': 'tapis', - 'homeDir': '/path/to/public', - 'icon': 'publications', - 'siteSearchPriority': 0, - 'resourceProvider': 'TACC', + "name": "Public Data", + "system": "cloud.data", + "scheme": "public", + "api": "tapis", + "homeDir": "/path/to/public", + "icon": "publications", + "siteSearchPriority": 0, + "resourceProvider": "TACC", }, { - 'name': 'Shared Workspaces', - 'scheme': 'projects', - 'api': 'tapis', - 'icon': 'publications', - 'resourceProvider': 'TACC', + "name": "Shared Workspaces", + "scheme": "projects", + "api": "tapis", + "icon": "publications", + "resourceProvider": "TACC", }, { - 'name': 'Google Drive', - 'system': 'googledrive', - 'scheme': 'private', - 'api': 'googledrive', - 'icon': None, - 'integration': 'portal.apps.googledrive_integration', - 'resourceProvider': 'Other', - } - ] + "name": "Google Drive", + "system": "googledrive", + "scheme": "private", + "api": "googledrive", + "icon": None, + "integration": "portal.apps.googledrive_integration", + "resourceProvider": "Other", + }, + ], } diff --git a/server/portal/apps/forms/urls.py b/server/portal/apps/forms/urls.py index 2c0b16cfa1..fec0a2b950 100644 --- a/server/portal/apps/forms/urls.py +++ b/server/portal/apps/forms/urls.py @@ -2,10 +2,11 @@ .. module:: portal.apps.forms.urls :synopsis: Forms URLs """ + from django.urls import re_path from portal.apps.forms.views import FormsView -app_name = 'workbench' +app_name = "workbench" urlpatterns = [ - re_path('', FormsView.as_view(), name='form'), + re_path("", FormsView.as_view(), name="form"), ] diff --git a/server/portal/apps/forms/views.py b/server/portal/apps/forms/views.py index 1ad3249c9c..5c28fb3e37 100644 --- a/server/portal/apps/forms/views.py +++ b/server/portal/apps/forms/views.py @@ -4,9 +4,8 @@ class FormsView(BaseApiView): - def get(self, request): - form_name = request.GET.get('form_name') + form_name = request.GET.get("form_name") form = settings.FORMS.get(form_name) return JsonResponse({"response": form}) diff --git a/server/portal/apps/googledrive_integration/apps.py b/server/portal/apps/googledrive_integration/apps.py index 90799660d8..d92f49d4fd 100644 --- a/server/portal/apps/googledrive_integration/apps.py +++ b/server/portal/apps/googledrive_integration/apps.py @@ -2,6 +2,6 @@ class GoogleDriveConfig(AppConfig): - name = 'portal.apps.googledrive_integration' - label = 'googledrive' - verbose_name = 'Google Drive Integration' + name = "portal.apps.googledrive_integration" + label = "googledrive" + verbose_name = "Google Drive Integration" diff --git a/server/portal/apps/googledrive_integration/integrations.py b/server/portal/apps/googledrive_integration/integrations.py index b85c15bf25..9e125170ba 100644 --- a/server/portal/apps/googledrive_integration/integrations.py +++ b/server/portal/apps/googledrive_integration/integrations.py @@ -6,23 +6,25 @@ def provide_integrations(request): activated = False - error = '' + error = "" try: request.user.googledrive_user_token activated = True except GoogleDriveUserToken.DoesNotExist: - if cache.get('{0}_googledrive_error'.format(request.session.session_key), False): - error = cache.get('{0}_googledrive_error'.format(request.session.session_key)) + if cache.get("{0}_googledrive_error".format(request.session.session_key), False): + error = cache.get("{0}_googledrive_error".format(request.session.session_key)) pass - integration = { - 'label': 'Google Drive', - 'description': 'Access files from your Google Drive account in {}.'.format(settings.PORTAL_NAMESPACE), - 'activated': activated, - 'error': error, - 'disconnect': reverse('googledrive_integration:disconnect'), - 'connect': reverse('googledrive_integration:initialize') - }, + integration = ( + { + "label": "Google Drive", + "description": "Access files from your Google Drive account in {}.".format(settings.PORTAL_NAMESPACE), + "activated": activated, + "error": error, + "disconnect": reverse("googledrive_integration:disconnect"), + "connect": reverse("googledrive_integration:initialize"), + }, + ) return integration diff --git a/server/portal/apps/googledrive_integration/migrations/0001_initial.py b/server/portal/apps/googledrive_integration/migrations/0001_initial.py index 6b41cb68d3..63817bea10 100644 --- a/server/portal/apps/googledrive_integration/migrations/0001_initial.py +++ b/server/portal/apps/googledrive_integration/migrations/0001_initial.py @@ -8,7 +8,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -17,11 +16,18 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='GoogleDriveUserToken', + name="GoogleDriveUserToken", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('credentials', portal.apps.googledrive_integration.models.CredentialsField(null=True)), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='googledrive_user_token', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("credentials", portal.apps.googledrive_integration.models.CredentialsField(null=True)), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="googledrive_user_token", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/server/portal/apps/googledrive_integration/models.py b/server/portal/apps/googledrive_integration/models.py index 08165f2f5a..dbbe6927d3 100644 --- a/server/portal/apps/googledrive_integration/models.py +++ b/server/portal/apps/googledrive_integration/models.py @@ -18,12 +18,12 @@ class CredentialsField(models.Field): """Django ORM field for storing OAuth2 Credentials.""" def __init__(self, *args, **kwargs): - if 'null' not in kwargs: - kwargs['null'] = True + if "null" not in kwargs: + kwargs["null"] = True super(CredentialsField, self).__init__(*args, **kwargs) def get_internal_type(self): - return 'BinaryField' + return "BinaryField" def from_db_value(self, value, expression, connection): """Overrides ``models.Field`` method. This converts the value @@ -40,11 +40,9 @@ def to_python(self, value): return value else: try: - return jsonpickle.decode( - base64.b64decode(encoding.smart_bytes(value)).decode()) + return jsonpickle.decode(base64.b64decode(encoding.smart_bytes(value)).decode()) except ValueError: - return pickle.loads( - base64.b64decode(encoding.smart_bytes(value))) + return pickle.loads(base64.b64decode(encoding.smart_bytes(value))) def get_prep_value(self, value): """Overrides ``models.Field`` method. This is used to convert @@ -54,8 +52,7 @@ def get_prep_value(self, value): if value is None: return None else: - return encoding.smart_str( - base64.b64encode(jsonpickle.encode(value).encode())) + return encoding.smart_str(base64.b64encode(jsonpickle.encode(value).encode())) def value_to_string(self, obj): """Convert the field value from the provided model to a string. @@ -76,8 +73,10 @@ class GoogleDriveUserToken(models.Model): """ Represents an OAuth Token for a Google Drive user """ + user = models.OneToOneField( - settings.AUTH_USER_MODEL, related_name='googledrive_user_token', on_delete=models.CASCADE) + settings.AUTH_USER_MODEL, related_name="googledrive_user_token", on_delete=models.CASCADE + ) credentials = CredentialsField() @property @@ -85,6 +84,5 @@ def client(self): if not self.credentials.valid: request = Request() self.credentials.refresh(request) - drive = discovery.build( - 'drive', 'v3', credentials=self.credentials, cache_discovery=False) + drive = discovery.build("drive", "v3", credentials=self.credentials, cache_discovery=False) return drive diff --git a/server/portal/apps/googledrive_integration/tasks.py b/server/portal/apps/googledrive_integration/tasks.py index 303cc2496f..2f26d65238 100644 --- a/server/portal/apps/googledrive_integration/tasks.py +++ b/server/portal/apps/googledrive_integration/tasks.py @@ -19,7 +19,7 @@ def check_connection(username): """ user = get_user_model().objects.get(username=username) drive = user.googledrive_user_token.client - request = drive.about().get(fields='user') + request = drive.about().get(fields="user") response = request.execute() - googledrive_user = response['user'] + googledrive_user = response["user"] return googledrive_user diff --git a/server/portal/apps/googledrive_integration/unit_test.py b/server/portal/apps/googledrive_integration/unit_test.py index ec8d06037e..ff61dbdca2 100644 --- a/server/portal/apps/googledrive_integration/unit_test.py +++ b/server/portal/apps/googledrive_integration/unit_test.py @@ -3,6 +3,7 @@ .. :module:: portal.apps.googledrive_integration.unit_test :synopsis: Google Drive integration app unit tests. """ + # from django.core.urlresolvers import reverse from portal.apps.googledrive_integration.models import GoogleDriveUserToken from mock import MagicMock @@ -11,78 +12,86 @@ import logging -logger = logging.getLogger('portal.apps.googledrive_integration.views') +logger = logging.getLogger("portal.apps.googledrive_integration.views") @pytest.fixture def mock_flow(mocker): - mock_flow = mocker.patch('portal.apps.googledrive_integration.views.google_auth_oauthlib.flow') - mock_flow.Flow.from_client_config.return_value.authorization_url.return_value = ('test_auth_url', 'test_state') - mock_flow.Flow.from_client_config.return_value.credentials = Credentials(token='asdf', refresh_token='1234') + mock_flow = mocker.patch("portal.apps.googledrive_integration.views.google_auth_oauthlib.flow") + mock_flow.Flow.from_client_config.return_value.authorization_url.return_value = ("test_auth_url", "test_state") + mock_flow.Flow.from_client_config.return_value.credentials = Credentials(token="asdf", refresh_token="1234") yield mock_flow @pytest.fixture def mock_request(mocker): - mock_request = mocker.patch('portal.apps.googledrive_integration.views.requests') + mock_request = mocker.patch("portal.apps.googledrive_integration.views.requests") mock_request.post.return_value = MagicMock(status_code=200) yield mock_request def test_initialize(django_user_model, client, mock_flow): - user = django_user_model.objects.create_user(username='testuser', password='testpassword') + user = django_user_model.objects.create_user(username="testuser", password="testpassword") client.force_login(user) - response = client.get('/accounts/applications/googledrive/initialize/') - - mock_flow.Flow.from_client_config.assert_called_with({'web': { - "client_id": 'test', - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://accounts.google.com/o/oauth2/token", - "client_secret": 'test' - }}, scopes=['https://www.googleapis.com/auth/drive']) - - mock_flow.Flow.from_client_config.return_value.authorization_url.assert_called_with(access_type='offline') - - assert mock_flow.Flow.from_client_config.return_value.redirect_uri == 'https://testserver/accounts/applications/googledrive/oauth2/' + response = client.get("/accounts/applications/googledrive/initialize/") + + mock_flow.Flow.from_client_config.assert_called_with( + { + "web": { + "client_id": "test", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "client_secret": "test", + } + }, + scopes=["https://www.googleapis.com/auth/drive"], + ) + + mock_flow.Flow.from_client_config.return_value.authorization_url.assert_called_with(access_type="offline") + + assert ( + mock_flow.Flow.from_client_config.return_value.redirect_uri + == "https://testserver/accounts/applications/googledrive/oauth2/" + ) assert response.status_code == 302 - assert response['location'] == 'test_auth_url' + assert response["location"] == "test_auth_url" def test_redirect(django_user_model, client, mock_flow): - user = django_user_model.objects.create_user(username='testuser', password='testpassword') + user = django_user_model.objects.create_user(username="testuser", password="testpassword") client.force_login(user) session = client.session - session['googledrive'] = {'state': '12345'} + session["googledrive"] = {"state": "12345"} session.save() - response = client.get('/accounts/applications/googledrive/oauth2/', {'state': '12345'}) + response = client.get("/accounts/applications/googledrive/oauth2/", {"state": "12345"}) - user = django_user_model.objects.get(username='testuser') - assert user.googledrive_user_token.credentials.token == 'asdf' - assert user.googledrive_user_token.credentials.refresh_token == '1234' + user = django_user_model.objects.get(username="testuser") + assert user.googledrive_user_token.credentials.token == "asdf" + assert user.googledrive_user_token.credentials.refresh_token == "1234" assert response.status_code == 302 - assert response['location'] == '/accounts/profile' + assert response["location"] == "/accounts/profile" def test_disconnect(django_user_model, client, mock_flow, mock_request): # Create a user and associated Google Drive credentials - user = django_user_model.objects.create_user(username='testuser', password='testpassword') - credentials = Credentials(token='asdf', refresh_token='1234') - GoogleDriveUserToken.objects.create( - user=user, - credentials=credentials) - assert user.googledrive_user_token.credentials.token == 'asdf' + user = django_user_model.objects.create_user(username="testuser", password="testpassword") + credentials = Credentials(token="asdf", refresh_token="1234") + GoogleDriveUserToken.objects.create(user=user, credentials=credentials) + assert user.googledrive_user_token.credentials.token == "asdf" # Disconnect and assert that the client is deleted. client.force_login(user) - response = client.get('/accounts/applications/googledrive/disconnect/') - user = django_user_model.objects.get(username='testuser') + response = client.get("/accounts/applications/googledrive/disconnect/") + user = django_user_model.objects.get(username="testuser") - mock_request.post.assert_called_with('https://accounts.google.com/o/oauth2/revoke', - params={'token': 'asdf'}, - headers={'content-type': 'application/x-www-form-urlencoded'}) + mock_request.post.assert_called_with( + "https://accounts.google.com/o/oauth2/revoke", + params={"token": "asdf"}, + headers={"content-type": "application/x-www-form-urlencoded"}, + ) with pytest.raises(GoogleDriveUserToken.DoesNotExist): user.googledrive_user_token.credentials.token assert response.status_code == 302 - assert response['location'] == '/accounts/profile' + assert response["location"] == "/accounts/profile" diff --git a/server/portal/apps/googledrive_integration/urls.py b/server/portal/apps/googledrive_integration/urls.py index ee64c067ad..24c66e0400 100644 --- a/server/portal/apps/googledrive_integration/urls.py +++ b/server/portal/apps/googledrive_integration/urls.py @@ -1,9 +1,10 @@ from portal.apps.googledrive_integration import views from django.urls import path -app_name = 'googledrive_integration' + +app_name = "googledrive_integration" urlpatterns = [ - path('', views.IndexView.as_view(), name='privacy'), - path('initialize/', views.initialize_token, name='initialize'), - path('oauth2/', views.oauth2_callback, name='oauth2_callback'), - path('disconnect/', views.disconnect, name='disconnect') + path("", views.IndexView.as_view(), name="privacy"), + path("initialize/", views.initialize_token, name="initialize"), + path("oauth2/", views.oauth2_callback, name="oauth2_callback"), + path("disconnect/", views.disconnect, name="disconnect"), ] diff --git a/server/portal/apps/googledrive_integration/views.py b/server/portal/apps/googledrive_integration/views.py index 8c02c9ca9f..065edcf218 100644 --- a/server/portal/apps/googledrive_integration/views.py +++ b/server/portal/apps/googledrive_integration/views.py @@ -10,6 +10,7 @@ from django.core.cache import cache import logging + logger = logging.getLogger(__name__) @@ -17,13 +18,15 @@ class IndexView(TemplateView): """ Main workbench view. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - context['setup_complete'] = False if self.request.user.is_anonymous \ - else self.request.user.profile.setup_complete - context['DEBUG'] = settings.DEBUG + context["setup_complete"] = ( + False if self.request.user.is_anonymous else self.request.user.profile.setup_complete + ) + context["DEBUG"] = settings.DEBUG return context def dispatch(self, request, *args, **kwargs): @@ -31,64 +34,66 @@ def dispatch(self, request, *args, **kwargs): def get_client_config(): - if 'google-drive' not in settings.EXTERNAL_RESOURCE_SECRETS: + if "google-drive" not in settings.EXTERNAL_RESOURCE_SECRETS: raise Exception("Google Drive not configured") - googledrive_secrets = settings.EXTERNAL_RESOURCE_SECRETS['google-drive'] - CLIENT_CONFIG = {'web': { - "client_id": googledrive_secrets['client_id'], - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://accounts.google.com/o/oauth2/token", - "client_secret": googledrive_secrets['client_secret'] - }} + googledrive_secrets = settings.EXTERNAL_RESOURCE_SECRETS["google-drive"] + CLIENT_CONFIG = { + "web": { + "client_id": googledrive_secrets["client_id"], + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "client_secret": googledrive_secrets["client_secret"], + } + } return CLIENT_CONFIG @csrf_exempt @login_required def initialize_token(request): - redirect_uri = 'https://{}{}'.format(request.get_host(), - reverse('googledrive_integration:oauth2_callback')) + redirect_uri = "https://{}{}".format(request.get_host(), reverse("googledrive_integration:oauth2_callback")) flow = google_auth_oauthlib.flow.Flow.from_client_config( - get_client_config(), - scopes=['https://www.googleapis.com/auth/drive']) + get_client_config(), scopes=["https://www.googleapis.com/auth/drive"] + ) flow.redirect_uri = request.build_absolute_uri(redirect_uri) # flow.redirect_uri = 'https://cep.dev/accounts/applications/googledrive/oauth2/' - auth_url, state = flow.authorization_url(access_type='offline') + auth_url, state = flow.authorization_url(access_type="offline") - request.session['googledrive'] = { - 'state': state - } + request.session["googledrive"] = {"state": state} return HttpResponseRedirect(auth_url) @csrf_exempt @login_required def oauth2_callback(request): - error = 'SETUP_ERROR' + error = "SETUP_ERROR" error_timeout = 5 - state = request.GET.get('state') - if 'googledrive' in request.session: - googledrive = request.session['googledrive'] + state = request.GET.get("state") + if "googledrive" in request.session: + googledrive = request.session["googledrive"] else: - logger.error('Could not retrieve googledrive from session') - cache.set('{0}_googledrive_error'.format(request.session.session_key), error, error_timeout) - return HttpResponseRedirect('/accounts/profile') + logger.error("Could not retrieve googledrive from session") + cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) + return HttpResponseRedirect("/accounts/profile") - if not (state == googledrive['state']): - logger.error('Could not retrieve state from googledrive stored var') - cache.set('{0}_googledrive_error'.format(request.session.session_key), error, error_timeout) - return HttpResponseRedirect('/accounts/profile') + if not (state == googledrive["state"]): + logger.error("Could not retrieve state from googledrive stored var") + cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) + return HttpResponseRedirect("/accounts/profile") try: - redirect_uri = reverse('googledrive_integration:oauth2_callback') + redirect_uri = reverse("googledrive_integration:oauth2_callback") flow = google_auth_oauthlib.flow.Flow.from_client_config( get_client_config(), - scopes=['https://www.googleapis.com/auth/drive', ], - state=state) - flow.redirect_uri = 'https://{}{}'.format(request.get_host(), redirect_uri) + scopes=[ + "https://www.googleapis.com/auth/drive", + ], + state=state, + ) + flow.redirect_uri = "https://{}{}".format(request.get_host(), redirect_uri) # Use the authorization server's response to fetch the OAuth 2.0 tokens. - authorization_response = 'https://{}{}'.format(request.get_host(), request.get_full_path()) + authorization_response = "https://{}{}".format(request.get_host(), request.get_full_path()) flow.fetch_token(authorization_response=authorization_response) @@ -97,54 +102,55 @@ def oauth2_callback(request): # Auth flow completed previously, and no refresh_token granted. Need to disconnect to get # another refresh_token. - logger.error('GoogleDriveUserToken refresh_token cannot be null, revoking previous access and restart flow.') - requests.post('https://accounts.google.com/o/oauth2/revoke', - params={'token': credentials.token}, - headers={'content-type': 'application/x-www-form-urlencoded'}) - HttpResponseRedirect(reverse('googledrive_integration:initialize_token')) + logger.error( + "GoogleDriveUserToken refresh_token cannot be null, revoking previous access and restart flow." + ) + requests.post( + "https://accounts.google.com/o/oauth2/revoke", + params={"token": credentials.token}, + headers={"content-type": "application/x-www-form-urlencoded"}, + ) + HttpResponseRedirect(reverse("googledrive_integration:initialize_token")) - GoogleDriveUserToken.objects.update_or_create( - user=request.user, - defaults={'credentials': credentials}) + GoogleDriveUserToken.objects.update_or_create(user=request.user, defaults={"credentials": credentials}) except Exception as e: - logger.exception('Unable to complete Google Drive integration setup: %s' % e) - cache.set('{0}_googledrive_error'.format(request.session.session_key), error, error_timeout) + logger.exception("Unable to complete Google Drive integration setup: %s" % e) + cache.set("{0}_googledrive_error".format(request.session.session_key), error, error_timeout) - return HttpResponseRedirect('/accounts/profile') + return HttpResponseRedirect("/accounts/profile") @login_required def disconnect(request): - logger.info('Disconnect Google Drive requested by user...') + logger.info("Disconnect Google Drive requested by user...") try: googledrive_user_token = GoogleDriveUserToken.objects.get(user=request.user) - revoke = requests.post('https://accounts.google.com/o/oauth2/revoke', - params={'token': googledrive_user_token.credentials.token}, - headers={'content-type': 'application/x-www-form-urlencoded'}) + revoke = requests.post( + "https://accounts.google.com/o/oauth2/revoke", + params={"token": googledrive_user_token.credentials.token}, + headers={"content-type": "application/x-www-form-urlencoded"}, + ) - status_code = getattr(revoke, 'status_code') + status_code = getattr(revoke, "status_code") googledrive_user_token.delete() if status_code == 200: - return HttpResponseRedirect('/accounts/profile') + return HttpResponseRedirect("/accounts/profile") else: - logger.error('Disconnect Google Drive; google drive account revoke error.', - extra={'user': request.user}) - logger.debug('status code:{}'.format(status_code)) + logger.error("Disconnect Google Drive; google drive account revoke error.", extra={"user": request.user}) + logger.debug("status code:{}".format(status_code)) - return HttpResponseRedirect('/accounts/profile') + return HttpResponseRedirect("/accounts/profile") except GoogleDriveUserToken.DoesNotExist: - logger.warn('Disconnect Google Drive; GoogleDriveUserToken does not exist.', - extra={'user': request.user}) + logger.warn("Disconnect Google Drive; GoogleDriveUserToken does not exist.", extra={"user": request.user}) except Exception as e: - logger.error('Disconnect Google Drive; GoogleDriveUserToken delete error.', - extra={'user': request.user}) - logger.exception('google drive delete error: {}'.format(e)) + logger.error("Disconnect Google Drive; GoogleDriveUserToken delete error.", extra={"user": request.user}) + logger.exception("google drive delete error: {}".format(e)) - return HttpResponseRedirect('/accounts/profile') + return HttpResponseRedirect("/accounts/profile") diff --git a/server/portal/apps/jupyter_mounts/api/urls.py b/server/portal/apps/jupyter_mounts/api/urls.py index 190c5dc8de..c5a82d7c5d 100644 --- a/server/portal/apps/jupyter_mounts/api/urls.py +++ b/server/portal/apps/jupyter_mounts/api/urls.py @@ -1,10 +1,10 @@ -"""Jupyter Mounts API Urls -""" +"""Jupyter Mounts API Urls""" + from django.urls import path from portal.apps.jupyter_mounts.api import views -app_name = 'jupyter_mounts' +app_name = "jupyter_mounts" urlpatterns = [ - path('', views.JupyterMountsApiView.as_view(), name='jupyter_mounts_api'), + path("", views.JupyterMountsApiView.as_view(), name="jupyter_mounts_api"), ] diff --git a/server/portal/apps/jupyter_mounts/api/views.py b/server/portal/apps/jupyter_mounts/api/views.py index 8e73e39cfd..ca96d56ff5 100644 --- a/server/portal/apps/jupyter_mounts/api/views.py +++ b/server/portal/apps/jupyter_mounts/api/views.py @@ -14,18 +14,20 @@ logger = logging.getLogger(__name__) -@method_decorator(agave_jwt_login, name='dispatch') -@method_decorator(login_required, name='dispatch') +@method_decorator(agave_jwt_login, name="dispatch") +@method_decorator(login_required, name="dispatch") class JupyterMountsApiView(BaseApiView): """JupyterMountsApiView This API returns a list of mount definitions for JupyterHub """ + def getDatafilesStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: result = [] non_private_systems = [ - sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if sys['api'] == 'tapis' and (sys['scheme'] == 'community' or sys['scheme'] == 'public') + sys + for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS + if sys["api"] == "tapis" and (sys["scheme"] == "community" or sys["scheme"] == "public") ] for system in evaluate_datafiles_storage_systems(tapis_oauth, non_private_systems): try: @@ -33,10 +35,9 @@ def getDatafilesStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: { "path": system.get("homeDir", "/"), "mountPath": "/{namespace}/{name}".format( - namespace=settings.PORTAL_NAMESPACE, - name=system['name'] + namespace=settings.PORTAL_NAMESPACE, name=system["name"] ), - "pems": "ro" + "pems": "ro", } ) except Exception: @@ -46,21 +47,19 @@ def getDatafilesStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: def getLocalStorageSystems(self, tapis_oauth: TapisOAuthToken) -> list: result = [] private_tapis_systems = [ - sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if sys['api'] == 'tapis' and sys['scheme'] == 'private' + sys + for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS + if sys["api"] == "tapis" and sys["scheme"] == "private" ] - for system in evaluate_datafiles_storage_systems( - tapis_oauth, private_tapis_systems - ): + for system in evaluate_datafiles_storage_systems(tapis_oauth, private_tapis_systems): try: result.append( { - "path": system['homeDir'], + "path": system["homeDir"], "mountPath": "/{namespace}/{name}".format( - namespace=settings.PORTAL_NAMESPACE, - name=system['name'] + namespace=settings.PORTAL_NAMESPACE, name=system["name"] ), - "pems": "rw" + "pems": "rw", } ) except Exception: @@ -90,16 +89,18 @@ def getProjectSystems(self, tapis_oauth: TapisOAuthToken) -> list: { "path": project["path"], "mountPath": "/{namespace}/My Projects/{name}".format( - namespace=settings.PORTAL_NAMESPACE, - name=name), - "pems": permissions + namespace=settings.PORTAL_NAMESPACE, name=name + ), + "pems": permissions, } ) return result def get(self, request): tapis_oauth = request.user.tapis_oauth - mounts = self.getDatafilesStorageSystems(tapis_oauth) + \ - self.getLocalStorageSystems(tapis_oauth) + \ - self.getProjectSystems(tapis_oauth) + mounts = ( + self.getDatafilesStorageSystems(tapis_oauth) + + self.getLocalStorageSystems(tapis_oauth) + + self.getProjectSystems(tapis_oauth) + ) return JsonResponse(mounts, safe=False) diff --git a/server/portal/apps/jupyter_mounts/api/views_unit_test.py b/server/portal/apps/jupyter_mounts/api/views_unit_test.py index 7eb9e2d24a..4f73531c28 100644 --- a/server/portal/apps/jupyter_mounts/api/views_unit_test.py +++ b/server/portal/apps/jupyter_mounts/api/views_unit_test.py @@ -6,8 +6,8 @@ @pytest.fixture def get_user_data(mocker): - mock = mocker.patch('portal.apps.datafiles.utils.get_user_data') - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_user.json')) as f: + mock = mocker.patch("portal.apps.datafiles.utils.get_user_data") + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_user.json")) as f: tas_user = json.load(f) mock.return_value = tas_user yield mock @@ -15,68 +15,57 @@ def get_user_data(mocker): @pytest.fixture def mock_projects(mocker): - mock = mocker.patch('portal.apps.jupyter_mounts.api.views.list_projects') - mock.return_value = [{'id': 'cep-project-1', - 'path': '/projects/cep.project-1', - 'name': 'test', 'host': 'cloud.data.tacc.utexas.edu', - 'updated': '2023-05-09T19:12:12.704162Z', - 'owner': {'username': 'jarosenb', - 'first_name': 'Jake', - 'last_name': 'Rosenberg', - 'email': 'jrosenberg@tacc.utexas.edu'}, - 'title': 'test', - 'description': None}, - {'id': 'cep-project-2', - 'path': '/projects/cep.project-2', - 'name': 'CEPV3-DEV-1002', 'host': 'cloud.data.tacc.utexas.edu', - 'updated': '2023-05-09T19:12:12.704162Z', - 'owner': {'username': 'jarosenb', - 'first_name': 'Jake', - 'last_name': 'Rosenberg', - 'email': 'jrosenberg@tacc.utexas.edu'}, - 'title': 'test (cep.project-2)', - 'description': None}] + mock = mocker.patch("portal.apps.jupyter_mounts.api.views.list_projects") + mock.return_value = [ + { + "id": "cep-project-1", + "path": "/projects/cep.project-1", + "name": "test", + "host": "cloud.data.tacc.utexas.edu", + "updated": "2023-05-09T19:12:12.704162Z", + "owner": { + "username": "jarosenb", + "first_name": "Jake", + "last_name": "Rosenberg", + "email": "jrosenberg@tacc.utexas.edu", + }, + "title": "test", + "description": None, + }, + { + "id": "cep-project-2", + "path": "/projects/cep.project-2", + "name": "CEPV3-DEV-1002", + "host": "cloud.data.tacc.utexas.edu", + "updated": "2023-05-09T19:12:12.704162Z", + "owner": { + "username": "jarosenb", + "first_name": "Jake", + "last_name": "Rosenberg", + "email": "jrosenberg@tacc.utexas.edu", + }, + "title": "test (cep.project-2)", + "description": None, + }, + ] yield mock @pytest.fixture def mock_project_pems(mocker): - mock = mocker.patch('portal.apps.jupyter_mounts.api.views.get_workspace_role') + mock = mocker.patch("portal.apps.jupyter_mounts.api.views.get_workspace_role") mock.side_effect = ["OWNER", "GUEST"] yield mock def test_get(authenticated_user, client, mock_projects, mock_project_pems, get_user_data): - result = client.get('/api/jupyter_mounts/') + result = client.get("/api/jupyter_mounts/") expected = [ - { - "path": "/path/to/community", - "mountPath": "/test/Community Data", - "pems": "ro" - }, - { - "path": "/path/to/public", - "mountPath": "/test/Public Data", - "pems": "ro"}, - { - "path": "/home/username", - "mountPath": "/test/My Data (Work)", - "pems": "rw" - }, - { - "path": "/home1/01234/username", - "mountPath": "/test/My Data (Frontera)", - "pems": "rw" - }, - { - "path": "/projects/cep.project-1", - "mountPath": "/test/My Projects/test", - "pems": "rw" - }, - { - "path": "/projects/cep.project-2", - "mountPath": "/test/My Projects/test (cep.project-2)", - "pems": "ro" - } + {"path": "/path/to/community", "mountPath": "/test/Community Data", "pems": "ro"}, + {"path": "/path/to/public", "mountPath": "/test/Public Data", "pems": "ro"}, + {"path": "/home/username", "mountPath": "/test/My Data (Work)", "pems": "rw"}, + {"path": "/home1/01234/username", "mountPath": "/test/My Data (Frontera)", "pems": "rw"}, + {"path": "/projects/cep.project-1", "mountPath": "/test/My Projects/test", "pems": "rw"}, + {"path": "/projects/cep.project-2", "mountPath": "/test/My Projects/test (cep.project-2)", "pems": "ro"}, ] assert json.loads(result.content) == expected diff --git a/server/portal/apps/jupyter_mounts/apps.py b/server/portal/apps/jupyter_mounts/apps.py index 7ae5740331..fc9b4dcb1f 100644 --- a/server/portal/apps/jupyter_mounts/apps.py +++ b/server/portal/apps/jupyter_mounts/apps.py @@ -2,4 +2,4 @@ class JupyterMountsConfig(AppConfig): - name = 'portal.apps.jupyter_mounts' + name = "portal.apps.jupyter_mounts" diff --git a/server/portal/apps/licenses/admin.py b/server/portal/apps/licenses/admin.py index 1fe12a0ece..b59fce4450 100644 --- a/server/portal/apps/licenses/admin.py +++ b/server/portal/apps/licenses/admin.py @@ -4,4 +4,4 @@ @admin.register(models.MATLABLicense) class MATLABLicenseAdmin(admin.ModelAdmin): - readonly_fields = ('license_type', ) + readonly_fields = ("license_type",) diff --git a/server/portal/apps/licenses/apps.py b/server/portal/apps/licenses/apps.py index 71ce6379ba..9f716ecb70 100644 --- a/server/portal/apps/licenses/apps.py +++ b/server/portal/apps/licenses/apps.py @@ -2,6 +2,6 @@ class LicensesAppConfig(AppConfig): - name = 'portal.apps.licenses' - label = 'portal_licenses' - verbose_name = 'Portal Licenses' + name = "portal.apps.licenses" + label = "portal_licenses" + verbose_name = "Portal Licenses" diff --git a/server/portal/apps/licenses/migrations/0001_initial.py b/server/portal/apps/licenses/migrations/0001_initial.py index eac8c21cff..662cc19ffc 100644 --- a/server/portal/apps/licenses/migrations/0001_initial.py +++ b/server/portal/apps/licenses/migrations/0001_initial.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -15,14 +14,26 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='MATLABLicense', + name="MATLABLicense", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('license_file_content', models.TextField(help_text="This should be entire contents of the user's MATLAB license file. Please ensure you paste the license exactly as it is in the license file.")), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='matlablicense', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "license_file_content", + models.TextField( + help_text="This should be entire contents of the user's MATLAB license file. Please ensure you paste the license exactly as it is in the license file." + ), + ), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="matlablicense", + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), ] diff --git a/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py b/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py index 3a8a2431ea..4f49f09144 100644 --- a/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py +++ b/server/portal/apps/licenses/migrations/0002_alter_matlablicense_user.py @@ -6,16 +6,17 @@ class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('portal_licenses', '0001_initial'), + ("portal_licenses", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='matlablicense', - name='user', - field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s', to=settings.AUTH_USER_MODEL), + model_name="matlablicense", + name="user", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, related_name="%(class)s", to=settings.AUTH_USER_MODEL + ), ), ] diff --git a/server/portal/apps/licenses/models.py b/server/portal/apps/licenses/models.py index dadeec50d1..5a2f69fceb 100644 --- a/server/portal/apps/licenses/models.py +++ b/server/portal/apps/licenses/models.py @@ -4,23 +4,21 @@ logger = logging.getLogger(__name__) -LICENSE_TYPES = [ - 'MATLAB' -] +LICENSE_TYPES = ["MATLAB"] def get_license_info(): return [ { - 'license_type': 'MATLAB', - 'class': 'portal.apps.licenses.MATLABLicense', - 'details_html': 'portal/apps/licenses/matlab_details.html', + "license_type": "MATLAB", + "class": "portal.apps.licenses.MATLABLicense", + "details_html": "portal/apps/licenses/matlab_details.html", } ], BaseLicense.__subclasses__() class BaseLicense(models.Model): - user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='%(class)s', on_delete=models.CASCADE) + user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name="%(class)s", on_delete=models.CASCADE) class Meta: abstract = True @@ -29,15 +27,17 @@ def __str__(self): return "%s: %s" % (self.license_type, self.user.username) def license_as_str(self): - self.license_file_content = self.license_file_content.replace('\r\n', '\n') + self.license_file_content = self.license_file_content.replace("\r\n", "\n") return self.license_file_content class MATLABLicense(BaseLicense): - license_file_content = models.TextField(help_text='This should be entire contents of ' - 'the user\'s MATLAB license file. ' - 'Please ensure you paste the ' - 'license exactly as it is in the ' - 'license file.') - - license_type = 'MATLAB' + license_file_content = models.TextField( + help_text="This should be entire contents of " + "the user's MATLAB license file. " + "Please ensure you paste the " + "license exactly as it is in the " + "license file." + ) + + license_type = "MATLAB" diff --git a/server/portal/apps/news/api/urls.py b/server/portal/apps/news/api/urls.py index f5df5104b1..f7a4244046 100644 --- a/server/portal/apps/news/api/urls.py +++ b/server/portal/apps/news/api/urls.py @@ -2,7 +2,7 @@ from portal.apps.news.api.views import UserNewsView -app_name = 'news_api' +app_name = "news_api" urlpatterns = [ - path('', UserNewsView.as_view(), name='list'), + path("", UserNewsView.as_view(), name="list"), ] diff --git a/server/portal/apps/news/api/views.py b/server/portal/apps/news/api/views.py index c79d173a59..6d61a4fc04 100644 --- a/server/portal/apps/news/api/views.py +++ b/server/portal/apps/news/api/views.py @@ -11,31 +11,31 @@ from portal.views.base import BaseApiView -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class UserNewsView(BaseApiView): def get(self, request, *args, **kwargs): user_news = self._get_user_news() - sanitize = request.GET.get('sanitize', 'false').lower() - should_sanitize = sanitize in ['true'] + sanitize = request.GET.get("sanitize", "false").lower() + should_sanitize = sanitize in ["true"] if should_sanitize: for news_item in user_news: - news_item['content'] = self._sanitize_news_content(news_item.get('content', '')) - for update in news_item.get('updates', []): - update['content'] = self._sanitize_news_content(update.get('content', '')) + news_item["content"] = self._sanitize_news_content(news_item.get("content", "")) + for update in news_item.get("updates", []): + update["content"] = self._sanitize_news_content(update.get("content", "")) return JsonResponse({"response": user_news, "status": 200}) def _get_user_news(self): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/announcements'.format(settings.TAS_URL), auth=auth) + r = requests.get("{0}/announcements".format(settings.TAS_URL), auth=auth) resp = r.json() - if resp.get('status') == 'success': - return resp.get('result', []) - raise ApiException('Failed to get announcements', resp.get('message')) + if resp.get("status") == "success": + return resp.get("result", []) + raise ApiException("Failed to get announcements", resp.get("message")) def _sanitize_news_content(self, content): - text_content = strip_tags(content or '') - return unescape(text_content).replace('\xa0', ' ') + text_content = strip_tags(content or "") + return unescape(text_content).replace("\xa0", " ") diff --git a/server/portal/apps/news/api/views_unit_test.py b/server/portal/apps/news/api/views_unit_test.py index 1d9e1519ca..8c1b701ac2 100644 --- a/server/portal/apps/news/api/views_unit_test.py +++ b/server/portal/apps/news/api/views_unit_test.py @@ -1,54 +1,54 @@ from unittest.mock import patch -@patch('portal.apps.news.api.views.requests.get') +@patch("portal.apps.news.api.views.requests.get") def test_api_news_success_without_sanitize(mock_get, client, authenticated_user): mock_get.return_value.json.return_value = { - 'status': 'success', - 'result': [ + "status": "success", + "result": [ { - 'id': 1, - 'content': '

Hello world

', - 'updates': [{'id': 10, 'content': '
Update
'}], + "id": 1, + "content": "

Hello world

", + "updates": [{"id": 10, "content": "
Update
"}], } ], } - response = client.get('/api/news/?sanitize=false') + response = client.get("/api/news/?sanitize=false") assert response.status_code == 200 body = response.json() - assert body['status'] == 200 - assert body['response'][0]['content'] == '

Hello world

' - assert body['response'][0]['updates'][0]['content'] == '
Update
' + assert body["status"] == 200 + assert body["response"][0]["content"] == "

Hello world

" + assert body["response"][0]["updates"][0]["content"] == "
Update
" -@patch('portal.apps.news.api.views.requests.get') +@patch("portal.apps.news.api.views.requests.get") def test_api_news_success_with_sanitize(mock_get, client, authenticated_user): mock_get.return_value.json.return_value = { - 'status': 'success', - 'result': [ + "status": "success", + "result": [ { - 'id': 1, - 'content': '

Hello world

', - 'updates': [{'id': 10, 'content': '
Update
'}], + "id": 1, + "content": "

Hello world

", + "updates": [{"id": 10, "content": "
Update
"}], } ], } - response = client.get('/api/news/?sanitize=true') + response = client.get("/api/news/?sanitize=true") assert response.status_code == 200 body = response.json() - assert body['status'] == 200 - assert body['response'][0]['content'] == 'Hello world' - assert body['response'][0]['updates'][0]['content'] == 'Update' + assert body["status"] == 200 + assert body["response"][0]["content"] == "Hello world" + assert body["response"][0]["updates"][0]["content"] == "Update" -@patch('portal.apps.news.api.views.requests.get') +@patch("portal.apps.news.api.views.requests.get") def test_api_news_failure_from_tas(mock_get, client, authenticated_user): mock_get.return_value.json.return_value = { - 'status': 'error', - 'message': 'bad upstream', + "status": "error", + "message": "bad upstream", } - response = client.get('/api/news/') + response = client.get("/api/news/") assert response.status_code == 400 diff --git a/server/portal/apps/news/apps.py b/server/portal/apps/news/apps.py index dd50331f43..c74df334fe 100644 --- a/server/portal/apps/news/apps.py +++ b/server/portal/apps/news/apps.py @@ -2,4 +2,4 @@ class NewsConfig(AppConfig): - name = 'portal.apps.news' + name = "portal.apps.news" diff --git a/server/portal/apps/news/urls.py b/server/portal/apps/news/urls.py index ac5c5424e6..9e819eb69d 100644 --- a/server/portal/apps/news/urls.py +++ b/server/portal/apps/news/urls.py @@ -2,8 +2,8 @@ from portal.apps.news.views import IndexView -app_name = 'news' +app_name = "news" urlpatterns = [ - path('', IndexView.as_view(), name='index'), - path('/', IndexView.as_view(), name='detail'), + path("", IndexView.as_view(), name="index"), + path("/", IndexView.as_view(), name="detail"), ] diff --git a/server/portal/apps/news/views.py b/server/portal/apps/news/views.py index 08db125c72..c77a78c249 100644 --- a/server/portal/apps/news/views.py +++ b/server/portal/apps/news/views.py @@ -4,15 +4,16 @@ from django.utils.decorators import method_decorator -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class IndexView(TemplateView): """ Render the SPA shell for top-level user updates routes. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - context['setup_complete'] = self.request.user.profile.setup_complete - context['DEBUG'] = settings.DEBUG + context["setup_complete"] = self.request.user.profile.setup_complete + context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/notifications/apps.py b/server/portal/apps/notifications/apps.py index e2df3973ce..c400e4b6ba 100644 --- a/server/portal/apps/notifications/apps.py +++ b/server/portal/apps/notifications/apps.py @@ -2,7 +2,7 @@ class NotificationsConfig(AppConfig): - name = 'portal.apps.notifications' - label = 'notifications' - verbose_name = 'Portal Notifications' - app_label = 'notifications' + name = "portal.apps.notifications" + label = "notifications" + verbose_name = "Portal Notifications" + app_label = "notifications" diff --git a/server/portal/apps/notifications/consumers.py b/server/portal/apps/notifications/consumers.py index b63fa5f627..8f9362202d 100644 --- a/server/portal/apps/notifications/consumers.py +++ b/server/portal/apps/notifications/consumers.py @@ -45,12 +45,8 @@ async def disconnect(self, close_code): if user.is_anonymous: # Connection has no logged in user, nothing to disconnect return - await self.channel_layer.group_discard( - group=str(user.id), channel=self.channel_name - ) - await self.channel_layer.group_discard( - group="portal_events", channel=self.channel_name - ) + await self.channel_layer.group_discard(group=str(user.id), channel=self.channel_name) + await self.channel_layer.group_discard(group="portal_events", channel=self.channel_name) async def portal_notification(self, event): """ diff --git a/server/portal/apps/notifications/migrations/0001_initial.py b/server/portal/apps/notifications/migrations/0001_initial.py index 3aebf4542f..65a02db552 100644 --- a/server/portal/apps/notifications/migrations/0001_initial.py +++ b/server/portal/apps/notifications/migrations/0001_initial.py @@ -5,49 +5,47 @@ class Migration(migrations.Migration): - initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Broadcast', + name="Broadcast", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('event_type', models.CharField(max_length=50)), - ('datetime', models.DateTimeField(blank=True, default=datetime.datetime.now)), - ('status', models.CharField(max_length=255)), - ('jobId', models.CharField(blank=True, max_length=255)), - ('operation', models.CharField(default='', max_length=255)), - ('message', models.TextField(default='')), - ('extra', models.TextField(default='')), - ('action_link', models.TextField(default='')), - ('group', models.CharField(max_length=20)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("event_type", models.CharField(max_length=50)), + ("datetime", models.DateTimeField(blank=True, default=datetime.datetime.now)), + ("status", models.CharField(max_length=255)), + ("jobId", models.CharField(blank=True, max_length=255)), + ("operation", models.CharField(default="", max_length=255)), + ("message", models.TextField(default="")), + ("extra", models.TextField(default="")), + ("action_link", models.TextField(default="")), + ("group", models.CharField(max_length=20)), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='Notification', + name="Notification", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('event_type', models.CharField(max_length=50)), - ('datetime', models.DateTimeField(blank=True, default=datetime.datetime.now)), - ('status', models.CharField(max_length=255)), - ('jobId', models.CharField(blank=True, max_length=255)), - ('operation', models.CharField(default='', max_length=255)), - ('message', models.TextField(default='')), - ('extra', models.TextField(default='')), - ('action_link', models.TextField(default='')), - ('user', models.CharField(db_index=True, max_length=20)), - ('read', models.BooleanField(default=False)), - ('deleted', models.BooleanField(default=False)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("event_type", models.CharField(max_length=50)), + ("datetime", models.DateTimeField(blank=True, default=datetime.datetime.now)), + ("status", models.CharField(max_length=255)), + ("jobId", models.CharField(blank=True, max_length=255)), + ("operation", models.CharField(default="", max_length=255)), + ("message", models.TextField(default="")), + ("extra", models.TextField(default="")), + ("action_link", models.TextField(default="")), + ("user", models.CharField(db_index=True, max_length=20)), + ("read", models.BooleanField(default=False)), + ("deleted", models.BooleanField(default=False)), ], options={ - 'abstract': False, + "abstract": False, }, ), ] diff --git a/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py b/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py index 05dabe73a4..5375fe8003 100644 --- a/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py +++ b/server/portal/apps/notifications/migrations/0002_auto_20200218_2115.py @@ -5,20 +5,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('notifications', '0001_initial'), + ("notifications", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='broadcast', - name='datetime', + model_name="broadcast", + name="datetime", field=models.DateTimeField(blank=True, default=django.utils.timezone.now), ), migrations.AlterField( - model_name='notification', - name='datetime', + model_name="notification", + name="datetime", field=models.DateTimeField(blank=True, default=django.utils.timezone.now), ), ] diff --git a/server/portal/apps/notifications/models.py b/server/portal/apps/notifications/models.py index 46f5e0314e..c324cc3e07 100644 --- a/server/portal/apps/notifications/models.py +++ b/server/portal/apps/notifications/models.py @@ -13,29 +13,30 @@ class BaseNotify(models.Model): These are the base fields that every notification should have. """ + event_type = models.CharField(max_length=50) datetime = models.DateTimeField(default=timezone.now, blank=True) # Status should be SUCCESS, INFO, ERROR, WARNING, status = models.CharField(max_length=255) jobId = models.CharField(max_length=255, blank=True) - operation = models.CharField(max_length=255, default='') - message = models.TextField(default='') - extra = models.TextField(default='') - action_link = models.TextField(default='') - - SUCCESS = GREEN = 'SUCCESS' - INFO = BLUE = 'INFO' - ERROR = RED = 'ERROR' - WARNING = ORANGE = 'WARNING' - EVENT_TYPE = 'event_type' - JOB_ID = 'jobId' - STATUS = 'status' - USER = USERNAME = 'user' - EXTRA = CONTENT = 'extra' - MESSAGE = 'message' - OPERATION = 'operation' - ACTION_LINK = 'action_link' - READ = 'read' + operation = models.CharField(max_length=255, default="") + message = models.TextField(default="") + extra = models.TextField(default="") + action_link = models.TextField(default="") + + SUCCESS = GREEN = "SUCCESS" + INFO = BLUE = "INFO" + ERROR = RED = "ERROR" + WARNING = ORANGE = "WARNING" + EVENT_TYPE = "event_type" + JOB_ID = "jobId" + STATUS = "status" + USER = USERNAME = "user" + EXTRA = CONTENT = "extra" + MESSAGE = "message" + OPERATION = "operation" + ACTION_LINK = "action_link" + READ = "read" def to_dict(self): try: @@ -43,14 +44,14 @@ def to_dict(self): except ValueError: extra = {} d = { - 'event_type': self.event_type, - 'datetime': self.datetime.strftime('%s'), - 'status': self.status, - 'operation': self.operation, - 'message': self.message, - 'extra': extra, - 'pk': self.pk, - 'action_link': self.action_link + "event_type": self.event_type, + "datetime": self.datetime.strftime("%s"), + "status": self.status, + "operation": self.operation, + "message": self.message, + "extra": extra, + "pk": self.pk, + "action_link": self.action_link, } return d @@ -63,7 +64,7 @@ def save(self, *args, **kwargs): try: json.dumps(self.extra[key]) except TypeError: - logger.debug('Keys with error: %s . Value: %s', key, self.extra[key]) + logger.debug("Keys with error: %s . Value: %s", key, self.extra[key]) raise super(BaseNotify, self).save(*args, **kwargs) @@ -92,11 +93,7 @@ def mark_deleted(self): def to_dict(self): event_data = super(Notification, self).to_dict() - event_data.update({ - 'user': self.user, - 'read': self.read, - 'deleted': self.deleted - }) + event_data.update({"user": self.user, "read": self.read, "deleted": self.deleted}) return event_data @@ -105,7 +102,5 @@ class Broadcast(BaseNotify): def to_dict(self): event_data = super(Broadcast, self).to_dict() - event_data.update({ - 'group': self.group - }) + event_data.update({"group": self.group}) return event_data diff --git a/server/portal/apps/notifications/routing.py b/server/portal/apps/notifications/routing.py index e7a1d020af..f3d64bf1d3 100644 --- a/server/portal/apps/notifications/routing.py +++ b/server/portal/apps/notifications/routing.py @@ -7,5 +7,5 @@ from portal.apps.notifications.consumers import NotificationsConsumer websocket_urlpatterns = [ - re_path(r'ws/notifications/$', NotificationsConsumer.as_asgi()), + re_path(r"ws/notifications/$", NotificationsConsumer.as_asgi()), ] diff --git a/server/portal/apps/notifications/unit_test.py b/server/portal/apps/notifications/unit_test.py index 436305d541..e6d249f91c 100644 --- a/server/portal/apps/notifications/unit_test.py +++ b/server/portal/apps/notifications/unit_test.py @@ -12,9 +12,9 @@ logger = logging.getLogger(__name__) -FILEDIR_PENDING = os.path.join(os.path.dirname(__file__), './json/pending.json') -FILEDIR_SUBMITTING = os.path.join(os.path.dirname(__file__), './json/submitting.json') -FILEDIR_PENDING2 = os.path.join(os.path.dirname(__file__), './json/pending2.json') +FILEDIR_PENDING = os.path.join(os.path.dirname(__file__), "./json/pending.json") +FILEDIR_SUBMITTING = os.path.join(os.path.dirname(__file__), "./json/submitting.json") +FILEDIR_PENDING2 = os.path.join(os.path.dirname(__file__), "./json/pending2.json") with open(FILEDIR_PENDING) as f: webhook_body_pending = json.dumps(json.load(f)) @@ -24,33 +24,33 @@ webhook_body_submitting = json.dumps(json.load(f)) -wh_url = reverse('webhooks:jobs_wh_handler') +wh_url = reverse("webhooks:jobs_wh_handler") @skip("Need to mock websocket call to redis") class NotificationsTestCase(TestCase): - fixtures = ['user-data.json', 'agave-oauth-token-data.json'] + fixtures = ["user-data.json", "agave-oauth-token-data.json"] def setUp(self): user = get_user_model().objects.get(pk=2) - user.set_password('password') + user.set_password("password") user.save() self.user = user self.client = Client() - with open('designsafe/apps/api/fixtures/agave-model-config-meta.json') as f: + with open("designsafe/apps/api/fixtures/agave-model-config-meta.json") as f: model_config_meta = json.load(f) self.model_config_meta = model_config_meta - with open('designsafe/apps/api/fixtures/agave-file-meta.json') as f: + with open("designsafe/apps/api/fixtures/agave-file-meta.json") as f: file_meta = json.load(f) self.file_meta = file_meta - with open('designsafe/apps/api/fixtures/agave-experiment-meta.json') as f: + with open("designsafe/apps/api/fixtures/agave-experiment-meta.json") as f: experiment_meta = json.load(f) self.experiment_meta = experiment_meta - with open('designsafe/apps/api/fixtures/agave-project-meta.json') as f: + with open("designsafe/apps/api/fixtures/agave-project-meta.json") as f: project_meta = json.load(f) self.project_meta = project_meta @@ -58,31 +58,31 @@ def test_current_user_is_ds_user(self): """ just making sure the db setup worked. """ - self.assertEqual(self.user.username, 'ds_user') + self.assertEqual(self.user.username, "ds_user") def test_submitting_webhook_returns_200_and_creates_notification(self): - r = self.client.post(wh_url, webhook_body_pending, content_type='application/json') + r = self.client.post(wh_url, webhook_body_pending, content_type="application/json") self.assertEqual(r.status_code, 200) n = Notification.objects.last() - status_from_notification = n.to_dict()['extra']['status'] - self.assertEqual(status_from_notification, 'PENDING') + status_from_notification = n.to_dict()["extra"]["status"] + self.assertEqual(status_from_notification, "PENDING") def test_2_webhooks_same_status_same_jobId_should_give_1_notification(self): - self.client.post(wh_url, webhook_body_pending, content_type='application/json') + self.client.post(wh_url, webhook_body_pending, content_type="application/json") # assert that sending the same status twice doesn't trigger a second notification. - self.client.post(wh_url, webhook_body_pending, content_type='application/json') + self.client.post(wh_url, webhook_body_pending, content_type="application/json") self.assertEqual(Notification.objects.count(), 1) def test_2_webhooks_different_status_same_jobId_should_give_2_notifications(self): - self.client.post(wh_url, webhook_body_pending, content_type='application/json') + self.client.post(wh_url, webhook_body_pending, content_type="application/json") - self.client.post(wh_url, webhook_body_submitting, content_type='application/json') + self.client.post(wh_url, webhook_body_submitting, content_type="application/json") self.assertEqual(Notification.objects.count(), 2) def test_2_webhooks_same_status_different_jobId_should_give_2_notifications(self): - self.client.post(wh_url, webhook_body_pending, content_type='application/json') - self.client.post(wh_url, webhook_body_pending2, content_type='application/json') + self.client.post(wh_url, webhook_body_pending, content_type="application/json") + self.client.post(wh_url, webhook_body_pending2, content_type="application/json") self.assertEqual(Notification.objects.count(), 2) diff --git a/server/portal/apps/notifications/urls.py b/server/portal/apps/notifications/urls.py index 56bf9d8a7a..34d09b56f3 100644 --- a/server/portal/apps/notifications/urls.py +++ b/server/portal/apps/notifications/urls.py @@ -2,7 +2,7 @@ from portal.apps.notifications.views import ManageNotificationsView -app_name = 'notifications' +app_name = "notifications" urlpatterns = [ - re_path(r'^(?P\w+)?$', ManageNotificationsView.as_view(), name='event_type_notifications'), + re_path(r"^(?P\w+)?$", ManageNotificationsView.as_view(), name="event_type_notifications"), ] diff --git a/server/portal/apps/notifications/views.py b/server/portal/apps/notifications/views.py index 4095b04c15..2b77309b62 100644 --- a/server/portal/apps/notifications/views.py +++ b/server/portal/apps/notifications/views.py @@ -10,67 +10,56 @@ class ManageNotificationsView(BaseApiView): - def get(self, request, *args, **kwargs): - """List all notifications of a certain event type. - """ - limit = request.GET.get('limit', 0) - page = request.GET.get('page', 0) - read = request.GET.get('read') - event_types = request.GET.getlist('eventTypes') + """List all notifications of a certain event type.""" + limit = request.GET.get("limit", 0) + page = request.GET.get("page", 0) + read = request.GET.get("read") + event_types = request.GET.getlist("eventTypes") query_params = {} if read is not None: - query_params['read'] = read + query_params["read"] = read if event_types: - notifs = Notification.objects.filter(event_type__in=event_types, - deleted=False, - user=request.user.username, - **query_params).order_by('-datetime') - total = Notification.objects.filter(event_type__in=event_types, - deleted=False, - user=request.user.username).count() - unread = Notification.objects.filter(event_type__in=event_types, - deleted=False, - read=False, - user=request.user.username).count() + notifs = Notification.objects.filter( + event_type__in=event_types, deleted=False, user=request.user.username, **query_params + ).order_by("-datetime") + total = Notification.objects.filter( + event_type__in=event_types, deleted=False, user=request.user.username + ).count() + unread = Notification.objects.filter( + event_type__in=event_types, deleted=False, read=False, user=request.user.username + ).count() else: - notifs = Notification.objects.filter(deleted=False, - user=request.user.username, - **query_params).order_by('-datetime') - total = Notification.objects.filter(deleted=False, - user=request.user.username).count() - unread = Notification.objects.filter(deleted=False, - read=False, - user=request.user.username).count() + notifs = Notification.objects.filter(deleted=False, user=request.user.username, **query_params).order_by( + "-datetime" + ) + total = Notification.objects.filter(deleted=False, user=request.user.username).count() + unread = Notification.objects.filter(deleted=False, read=False, user=request.user.username).count() if limit: limit = int(limit) page = int(page) offset = page * limit - notifs = notifs[offset:offset+limit] + notifs = notifs[offset : offset + limit] notifs = [n.to_dict() for n in notifs] - return JsonResponse({'notifs': notifs, 'page': page, 'total': total, 'unread': unread}) + return JsonResponse({"notifs": notifs, "page": page, "total": total, "unread": unread}) def patch(self, request, *args, **kwargs): - """Mark notifications as read. - """ + """Mark notifications as read.""" body = json.loads(request.body) - nid = body.get('id') - read = body.get('read', True) - event_types = body.get('eventTypes') + nid = body.get("id") + read = body.get("read", True) + event_types = body.get("eventTypes") - if nid == 'all' and read is True: + if nid == "all" and read is True: if event_types is not None: - notifs = Notification.objects.filter(deleted=False, - read=False, - event_type__in=event_types, - user=request.user.username) + notifs = Notification.objects.filter( + deleted=False, read=False, event_type__in=event_types, user=request.user.username + ) else: - notifs = Notification.objects.filter(deleted=False, - read=False, - user=request.user.username) + notifs = Notification.objects.filter(deleted=False, read=False, user=request.user.username) for n in notifs: n.mark_read() else: @@ -78,12 +67,11 @@ def patch(self, request, *args, **kwargs): n.read = read n.save() - return JsonResponse({'message': 'OK'}) + return JsonResponse({"message": "OK"}) def delete(self, request, pk, *args, **kwargs): - """Mark notifications as deleted. - """ - if pk == 'all': + """Mark notifications as deleted.""" + if pk == "all": items = Notification.objects.filter(deleted=False, user=request.user.username) for i in items: i.mark_deleted() @@ -91,4 +79,4 @@ def delete(self, request, pk, *args, **kwargs): x = Notification.objects.get(pk=pk) x.mark_deleted() - return JsonResponse({'message': 'OK'}) + return JsonResponse({"message": "OK"}) diff --git a/server/portal/apps/onboarding/api/urls.py b/server/portal/apps/onboarding/api/urls.py index 75351dacf5..fa7a7940ff 100644 --- a/server/portal/apps/onboarding/api/urls.py +++ b/server/portal/apps/onboarding/api/urls.py @@ -2,13 +2,14 @@ .. :module:: apps.accounts.api.urls :synopsis: Manager handling anything pertaining to accounts """ + from django.urls import path from portal.apps.onboarding.api import views -app_name = 'portal_onboarding_api' +app_name = "portal_onboarding_api" urlpatterns = [ - path('user/', views.SetupStepView.as_view(), name='user_self_view'), - path('user//', views.SetupStepView.as_view(), name='user_view'), - path('admin/', views.SetupAdminView.as_view(), name='user_admin') + path("user/", views.SetupStepView.as_view(), name="user_self_view"), + path("user//", views.SetupStepView.as_view(), name="user_view"), + path("admin/", views.SetupAdminView.as_view(), name="user_admin"), ] diff --git a/server/portal/apps/onboarding/api/views.py b/server/portal/apps/onboarding/api/views.py index 9a948e805c..b00c4638ac 100644 --- a/server/portal/apps/onboarding/api/views.py +++ b/server/portal/apps/onboarding/api/views.py @@ -11,16 +11,8 @@ from django.contrib.admin.views.decorators import staff_member_required from django.utils.decorators import method_decorator from django.conf import settings -from portal.apps.onboarding.models import ( - SetupEvent, - SetupEventEncoder -) -from portal.apps.onboarding.execute import ( - log_setup_state, - load_setup_step, - execute_single_step, - execute_setup_steps -) +from portal.apps.onboarding.models import SetupEvent, SetupEventEncoder +from portal.apps.onboarding.execute import log_setup_state, load_setup_step, execute_single_step, execute_setup_steps from portal.apps.onboarding.state import SetupState from portal.apps.users.utils import q_to_model_queries import json @@ -46,9 +38,7 @@ def get_user_onboarding(user): retried_step = None for step in account_setup_steps: # Get step events in descending order of time - step_events = SetupEvent.objects.filter(user=user, step=step["step"]).order_by( - "-time" - ) + step_events = SetupEvent.objects.filter(user=user, step=step["step"]).order_by("-time") step_instance = load_setup_step(user, step["step"]) @@ -70,10 +60,10 @@ def get_user_onboarding(user): ): retried_step = step_instance step_instance.state = SetupState.PROCESSING - execute_single_step.apply_async(args=[user.username, step["step"]], countdown=2) # slight delay to allow client to render - logger.info( - "Retrying setup step %s for %s", step["step"], user.username - ) + execute_single_step.apply_async( + args=[user.username, step["step"]], countdown=2 + ) # slight delay to allow client to render + logger.info("Retrying setup step %s for %s", step["step"], user.username) step_data = { "step": step["step"], @@ -100,7 +90,7 @@ def get_user_onboarding(user): return result -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class SetupStepView(BaseApiView): def get_user_parameter(self, request, username): """ @@ -165,10 +155,8 @@ def complete(self, request, setup_step): if not request.user.is_staff: raise PermissionDenied setup_step.state = SetupState.COMPLETED - setup_step.log("{step} marked complete by {staff}".format( - step=setup_step.display_name(), - staff=request.user.username - ) + setup_step.log( + "{step} marked complete by {staff}".format(step=setup_step.display_name(), staff=request.user.username) ) def reset(self, request, setup_step): @@ -177,11 +165,7 @@ def reset(self, request, setup_step): """ if not request.user.is_staff: raise PermissionDenied - setup_step.log("{step} reset by {staff}".format( - step=setup_step.display_name(), - staff=request.user.username - ) - ) + setup_step.log("{step} reset by {staff}".format(step=setup_step.display_name(), staff=request.user.username)) # Mark the user's setup_complete as False setup_step.user.profile.setup_complete = False @@ -189,9 +173,8 @@ def reset(self, request, setup_step): log_setup_state( setup_step.user, "{user} setup marked incomplete, due to reset of {step}".format( - user=setup_step.user.username, - step=setup_step.step_name() - ) + user=setup_step.user.username, step=setup_step.step_name() + ), ) setup_step.prepare() @@ -199,11 +182,10 @@ def client_action(self, request, setup_step, action, data): """ Call client_action on a setup step """ - setup_step.log("{action} action on {step} by {username}".format( - action=action, - step=setup_step.step_name(), - username=request.user.username - ) + setup_step.log( + "{action} action on {step} by {username}".format( + action=action, step=setup_step.step_name(), username=request.user.username + ) ) setup_step.client_action(action, data, request) @@ -259,36 +241,32 @@ def post(self, request, username): # Serialize and send back the last event on this step # Requires safe=False since SetupEvent is not a dict - return JsonResponse( - setup_step.last_event, - encoder=SetupEventEncoder, - safe=False - ) + return JsonResponse(setup_step.last_event, encoder=SetupEventEncoder, safe=False) -@method_decorator(login_required, name='dispatch') -@method_decorator(staff_member_required, name='dispatch') +@method_decorator(login_required, name="dispatch") +@method_decorator(staff_member_required, name="dispatch") class SetupAdminView(BaseApiView): def get(self, request): - offset = int(request.GET.get('offset', 0)) - limit = int(request.GET.get('limit', 10)) + offset = int(request.GET.get("offset", 0)) + limit = int(request.GET.get("limit", 10)) users = [] results = get_user_model().objects.all() - q = request.GET.get('q', None) + q = request.GET.get("q", None) if q: query = q_to_model_queries(q) results = results.filter(query) - show_incomplete_only = request.GET.get('showIncompleteOnly', 'False').lower() + show_incomplete_only = request.GET.get("showIncompleteOnly", "False").lower() # Filter users based on the showIncompleteOnly parameter - if show_incomplete_only == 'true': + if show_incomplete_only == "true": results = results.filter(profile__setup_complete=False) # Get users, with most recently joined users that do not have setup_complete, first - results = results.order_by('-date_joined', 'profile__setup_complete', 'last_name', 'first_name') + results = results.order_by("-date_joined", "profile__setup_complete", "last_name", "first_name") # Uncomment this line to simulate many user results # results = list(results) * 105 total = len(results) - page = results[offset:offset + limit] + page = results[offset : offset + limit] # Assemble an array with the User data we care about for user in page: @@ -298,15 +276,6 @@ def get(self, request): # If a user does not have a PortalProfile, skip it logger.info(err) - response = { - "users": users, - "offset": offset, - "limit": limit, - "total": total - } + response = {"users": users, "offset": offset, "limit": limit, "total": total} - return JsonResponse( - response, - encoder=SetupEventEncoder, - safe=False - ) + return JsonResponse(response, encoder=SetupEventEncoder, safe=False) diff --git a/server/portal/apps/onboarding/api/views_unit_test.py b/server/portal/apps/onboarding/api/views_unit_test.py index 85271ae237..b3cb0d381b 100644 --- a/server/portal/apps/onboarding/api/views_unit_test.py +++ b/server/portal/apps/onboarding/api/views_unit_test.py @@ -3,10 +3,7 @@ import json from portal.apps.onboarding.models import SetupEvent from portal.apps.onboarding.state import SetupState -from portal.apps.onboarding.api.views import ( - SetupStepView, - get_user_onboarding -) +from portal.apps.onboarding.api.views import SetupStepView, get_user_onboarding import pytest import logging @@ -17,12 +14,12 @@ @pytest.fixture(autouse=True) def mocked_executor(mocker): - yield mocker.patch('portal.apps.onboarding.api.views.execute_setup_steps') + yield mocker.patch("portal.apps.onboarding.api.views.execute_setup_steps") @pytest.fixture(autouse=True) def mocked_log_setup_state(mocker): - yield mocker.patch('portal.apps.onboarding.api.views.log_setup_state') + yield mocker.patch("portal.apps.onboarding.api.views.log_setup_state") """ @@ -31,19 +28,19 @@ def mocked_log_setup_state(mocker): def test_get_user(client, authenticated_user): - response = client.get('/api/onboarding/user/{}/'.format(authenticated_user.username)) + response = client.get("/api/onboarding/user/{}/".format(authenticated_user.username)) assert response.status_code == 200 result = json.loads(response.content) assert result["username"] == "username" def test_get_user_unauthenticated_forbidden(client, regular_user): - response = client.get('/api/onboarding/user/{}/'.format(regular_user.username)) + response = client.get("/api/onboarding/user/{}/".format(regular_user.username)) assert response.status_code == 302 def test_get_other_user_forbidden(client, authenticated_user, regular_user2): - response = client.get('/api/onboarding/user/{}/'.format(regular_user2.username)) + response = client.get("/api/onboarding/user/{}/".format(regular_user2.username)) assert response.status_code == 403 @@ -77,8 +74,8 @@ def test_get_user_as_user(client, settings, authenticated_user, mock_steps): result = json.loads(response.content) assert result["username"] == authenticated_user.username assert "steps" in result - assert result["steps"][0]["step"] == 'portal.apps.onboarding.steps.test_steps.MockStep' - assert result["steps"][0]["displayName"] == 'Mock Step' + assert result["steps"][0]["step"] == "portal.apps.onboarding.steps.test_steps.MockStep" + assert result["steps"][0]["displayName"] == "Mock Step" assert result["steps"][0]["state"] == SetupState.COMPLETED assert len(result["steps"][0]["events"]) == 2 @@ -86,14 +83,13 @@ def test_get_user_as_user(client, settings, authenticated_user, mock_steps): def test_retry_step(client, settings, authenticated_user, mock_retry_step, mocker): mock_execute_single_step = mocker.patch("portal.apps.onboarding.api.views.execute_single_step") response = client.get("/api/onboarding/user/{}".format(authenticated_user.username), follow=True) - mock_execute_single_step.apply_async.assert_called_with(args=[ - authenticated_user.username, - 'portal.apps.onboarding.steps.test_steps.MockStep' - ], countdown=2) + mock_execute_single_step.apply_async.assert_called_with( + args=[authenticated_user.username, "portal.apps.onboarding.steps.test_steps.MockStep"], countdown=2 + ) result = json.loads(response.content) assert result["username"] == authenticated_user.username assert "steps" in result - assert result["steps"][0]["step"] == 'portal.apps.onboarding.steps.test_steps.MockStep' + assert result["steps"][0]["step"] == "portal.apps.onboarding.steps.test_steps.MockStep" assert result["steps"][0]["state"] == SetupState.PROCESSING @@ -102,14 +98,14 @@ def test_incomplete_post(client, authenticated_user): response = client.post( "/api/onboarding/user/{}/".format(authenticated_user), content_type="application/json", - data=json.dumps({"action": "user_confirm"}) + data=json.dumps({"action": "user_confirm"}), ) assert response.status_code == 400 response = client.post( "/api/onboarding/user/{}/".format(authenticated_user), content_type="application/json", - data=json.dumps({"step": "setupstep"}) + data=json.dumps({"step": "setupstep"}), ) assert response.status_code == 400 @@ -120,28 +116,16 @@ def test_client_action(regular_user, rf): mock_step.step_name.return_value = "Mock Step" request = rf.post("/api/onboarding/user/username") request.user = regular_user - view.client_action( - request, - mock_step, - "user_confirm", - None - ) + view.client_action(request, mock_step, "user_confirm", None) mock_step.log.assert_called() - mock_step.client_action.assert_called_with( - "user_confirm", - None, - request - ) + mock_step.client_action.assert_called_with("user_confirm", None, request) def test_reset_not_staff(client, authenticated_user): response = client.post( "/api/onboarding/user/{}/".format(authenticated_user.username), - content_type='application/json', - data=json.dumps({ - "action": "reset", - "step": "portal.apps.onboarding.steps.test_steps.MockStep" - }) + content_type="application/json", + data=json.dumps({"action": "reset", "step": "portal.apps.onboarding.steps.test_steps.MockStep"}), ) assert response.status_code == 403 @@ -172,11 +156,8 @@ def test_complete_not_staff(client, authenticated_user, regular_user2): def test_complete(client, authenticated_staff, regular_user, mock_steps, mocked_executor): response = client.post( "/api/onboarding/user/{}/".format(regular_user.username), - content_type='application/json', - data=json.dumps({ - "action": "complete", - "step": "portal.apps.onboarding.steps.test_steps.MockStep" - }) + content_type="application/json", + data=json.dumps({"action": "complete", "step": "portal.apps.onboarding.steps.test_steps.MockStep"}), ) # set_state should have put MockStep in COMPLETED, as per request @@ -222,9 +203,7 @@ def test_get_no_profile(client, authenticated_staff, regular_user): response_data = json.loads(response.content) # regular_user should not appear in results - assert not any( - [True for user in response_data['users'] if user['username'] == regular_user.username] - ) + assert not any([True for user in response_data["users"] if user["username"] == regular_user.username]) def test_get(client, authenticated_staff, regular_user, mock_steps): @@ -251,14 +230,14 @@ def test_get(client, authenticated_staff, regular_user, mock_steps): assert users[0]["username"] == regular_user.username # User regular_user's last event should be MockStep - assert users[0]['steps'][0]['step'] == "portal.apps.onboarding.steps.test_steps.MockStep" + assert users[0]["steps"][0]["step"] == "portal.apps.onboarding.steps.test_steps.MockStep" # There should be two users returned assert len(users) == 2 # Assertions with 'showIncompleteOnly=true' assert users_incomplete[0]["username"] == regular_user.username - assert users_incomplete[0]['steps'][0]['step'] == "portal.apps.onboarding.steps.test_steps.MockStep" + assert users_incomplete[0]["steps"][0]["step"] == "portal.apps.onboarding.steps.test_steps.MockStep" # There should be one user since only one user has setup_complete = True assert len(users_incomplete) == 1 @@ -274,7 +253,7 @@ def test_get_search(client, authenticated_staff, regular_user, mock_steps): assert users[0]["username"] == regular_user.username # User regular_user's last event should be MockStep - assert users[0]['steps'][0]['step'] == "portal.apps.onboarding.steps.test_steps.MockStep" + assert users[0]["steps"][0]["step"] == "portal.apps.onboarding.steps.test_steps.MockStep" # There should be two users returned assert len(users) == 1 diff --git a/server/portal/apps/onboarding/apps.py b/server/portal/apps/onboarding/apps.py index 77b22249a4..7e8014f101 100644 --- a/server/portal/apps/onboarding/apps.py +++ b/server/portal/apps/onboarding/apps.py @@ -1,7 +1,5 @@ - - from django.apps import AppConfig class OnboardingConfig(AppConfig): - name = 'portal.apps.onboarding' + name = "portal.apps.onboarding" diff --git a/server/portal/apps/onboarding/conftest.py b/server/portal/apps/onboarding/conftest.py index 8db698c1c9..51f7d1a9bf 100644 --- a/server/portal/apps/onboarding/conftest.py +++ b/server/portal/apps/onboarding/conftest.py @@ -5,16 +5,12 @@ @pytest.fixture def mock_steps(regular_user, settings): - settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockStep' - } - ] + settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [{"step": "portal.apps.onboarding.steps.test_steps.MockStep"}] pending_step = SetupEvent.objects.create( user=regular_user, step="portal.apps.onboarding.steps.test_steps.MockStep", state=SetupState.PENDING, - message="message" + message="message", ) completed_step = SetupEvent.objects.create( @@ -29,16 +25,12 @@ def mock_steps(regular_user, settings): @pytest.fixture def mock_retry_step(regular_user, settings): settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockStep', - 'retry': True, - 'settings': {} - } + {"step": "portal.apps.onboarding.steps.test_steps.MockStep", "retry": True, "settings": {}} ] retry_step = SetupEvent.objects.create( user=regular_user, step="portal.apps.onboarding.steps.test_steps.MockStep", state=SetupState.PENDING, - message="message" + message="message", ) yield retry_step diff --git a/server/portal/apps/onboarding/execute.py b/server/portal/apps/onboarding/execute.py index 200320c804..eb244111dc 100644 --- a/server/portal/apps/onboarding/execute.py +++ b/server/portal/apps/onboarding/execute.py @@ -22,7 +22,7 @@ def __init__(self, message): def new_user_setup_check(user): - extra_steps = getattr(settings, 'PORTAL_USER_ACCOUNT_SETUP_STEPS', []) + extra_steps = getattr(settings, "PORTAL_USER_ACCOUNT_SETUP_STEPS", []) if len(extra_steps) == 0: logger.info("No extra setup steps for user {username}".format(username=user.username)) profile = PortalProfile.objects.get(user=user) @@ -41,27 +41,19 @@ def log_setup_state(user, message): step="portal.apps.onboarding.execute.execute_setup_steps", state=SetupState.COMPLETED if user.profile.setup_complete else SetupState.FAILED, message=message, - data={"setupComplete": user.profile.setup_complete} + data={"setupComplete": user.profile.setup_complete}, ) def load_setup_step(user, step): - module_str, callable_str = step.rsplit('.', 1) + module_str, callable_str = step.rsplit(".", 1) module = import_module(module_str) call = getattr(module, callable_str) if not isclass(call): - raise ValueError( - "Setup step {step} is not a class".format( - step=step - ) - ) + raise ValueError("Setup step {step} is not a class".format(step=step)) setup_step = call(user) if not isinstance(setup_step, AbstractStep): - raise ValueError( - "Setup step {step} is not a subclass of AbstractStep".format( - step=step - ) - ) + raise ValueError("Setup step {step} is not a subclass of AbstractStep".format(step=step)) return setup_step @@ -69,9 +61,9 @@ def prepare_setup_steps(user): """ Set the initial state of all setup steps for a given user """ - extra_steps = getattr(settings, 'PORTAL_USER_ACCOUNT_SETUP_STEPS', []) + extra_steps = getattr(settings, "PORTAL_USER_ACCOUNT_SETUP_STEPS", []) for step in extra_steps: - setup_step = load_setup_step(user, step['step']) + setup_step = load_setup_step(user, step["step"]) if setup_step.last_event is None: setup_step.prepare() @@ -90,12 +82,13 @@ def process_setup_step(setup_step): @shared_task() def execute_setup_steps(username): from django.contrib.auth import get_user_model + user = get_user_model().objects.get(username=username) - extra_steps = getattr(settings, 'PORTAL_USER_ACCOUNT_SETUP_STEPS', []) + extra_steps = getattr(settings, "PORTAL_USER_ACCOUNT_SETUP_STEPS", []) for step in extra_steps: # Restore state of this setup step for this user - setup_step = load_setup_step(user, step['step']) + setup_step = load_setup_step(user, step["step"]) # Run step, if waiting for automatic execution # should have this state if setup_step.state == SetupState.PENDING: @@ -110,17 +103,13 @@ def execute_setup_steps(username): # a step failing to reach the COMPLETED state, mark the user as setup_complete user.profile.setup_complete = True user.profile.save() - log_setup_state( - user, - "{user} setup is now complete".format( - user=user.username - ) - ) + log_setup_state(user, "{user} setup is now complete".format(user=user.username)) @shared_task() def execute_single_step(username, step_name): from django.contrib.auth import get_user_model + user = get_user_model().objects.get(username=username) # Process specified setup step setup_step = load_setup_step(user, step_name) diff --git a/server/portal/apps/onboarding/execute_unit_test.py b/server/portal/apps/onboarding/execute_unit_test.py index 63a8e7c4d2..0bb14428f2 100644 --- a/server/portal/apps/onboarding/execute_unit_test.py +++ b/server/portal/apps/onboarding/execute_unit_test.py @@ -10,7 +10,7 @@ load_setup_step, log_setup_state, new_user_setup_check, - StepExecuteException + StepExecuteException, ) import pytest @@ -20,7 +20,7 @@ @pytest.fixture def mock_event_create(mocker): - yield mocker.patch('portal.apps.onboarding.execute.SetupEvent.objects.create', autospec=True) + yield mocker.patch("portal.apps.onboarding.execute.SetupEvent.objects.create", autospec=True) def test_log_setup_state_complete(authenticated_user, mock_event_create): @@ -34,7 +34,7 @@ def test_log_setup_state_complete(authenticated_user, mock_event_create): step="portal.apps.onboarding.execute.execute_setup_steps", state=SetupState.COMPLETED, message="test message", - data={"setupComplete": True} + data={"setupComplete": True}, ) @@ -49,7 +49,7 @@ def test_log_setup_state_incomplete(authenticated_user, mock_event_create): step="portal.apps.onboarding.execute.execute_setup_steps", state=SetupState.FAILED, message="test message", - data={"setupComplete": False} + data={"setupComplete": False}, ) @@ -57,18 +57,12 @@ def test_prepare_setup_steps(authenticated_user, mocker, settings): """ Test that a step is loaded and prepared for a user that does not have step history """ - settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'TestStep' - } - ] - mock_step = MagicMock( - last_event=None - ) - mock_loader = mocker.patch('portal.apps.onboarding.execute.load_setup_step') + settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [{"step": "TestStep"}] + mock_step = MagicMock(last_event=None) + mock_loader = mocker.patch("portal.apps.onboarding.execute.load_setup_step") mock_loader.return_value = mock_step prepare_setup_steps(authenticated_user) - mock_loader.assert_called_with(authenticated_user, 'TestStep') + mock_loader.assert_called_with(authenticated_user, "TestStep") mock_step.prepare.assert_called() @@ -76,10 +70,7 @@ def test_step_loader(authenticated_user): """ Test the dynamic step loader """ - step = load_setup_step( - authenticated_user, - 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - ) + step = load_setup_step(authenticated_user, "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep") assert step is not None @@ -90,10 +81,7 @@ def test_invalid_step_function(authenticated_user): This may occur due to a legacy setting "portal.apps.accounts.steps.step_one" """ with pytest.raises(ValueError): - load_setup_step( - authenticated_user, - 'portal.apps.onboarding.steps.test_steps.mock_invalid_step_function' - ) + load_setup_step(authenticated_user, "portal.apps.onboarding.steps.test_steps.mock_invalid_step_function") def test_invalid_step_class(authenticated_user): @@ -104,10 +92,7 @@ def test_invalid_step_class(authenticated_user): This may occur due to a legacy setting "portal.apps.accounts.steps.StepThree" """ with pytest.raises(ValueError): - load_setup_step( - authenticated_user, - 'portal.apps.onboarding.steps.test_steps.MockInvalidStepClass' - ) + load_setup_step(authenticated_user, "portal.apps.onboarding.steps.test_steps.MockInvalidStepClass") def test_successful_step(settings, authenticated_user, mocker): @@ -115,20 +100,19 @@ def test_successful_step(settings, authenticated_user, mocker): Test that a step that completes successfully is executed without error """ settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - } + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep"} ] - mock_log_setup_state = mocker.patch('portal.apps.onboarding.execute.log_setup_state') + mock_log_setup_state = mocker.patch("portal.apps.onboarding.execute.log_setup_state") prepare_setup_steps(authenticated_user) execute_setup_steps(authenticated_user.username) # Last event should be COMPLETED for MockPendingCompleteStep - setup_event = SetupEvent.objects.all().filter( - step="portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep", - user=authenticated_user - ).latest("time") + setup_event = ( + SetupEvent.objects.all() + .filter(step="portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep", user=authenticated_user) + .latest("time") + ) assert setup_event.message == "Completed" # After last event has completed, setup_complete should be true for user @@ -147,12 +131,8 @@ def test_fail_step(settings, authenticated_user): should not execute due to the previous step failing. """ settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' - }, - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - } + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep"}, + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep"}, ] with pytest.raises(StepExecuteException): prepare_setup_steps(authenticated_user) @@ -161,8 +141,8 @@ def test_fail_step(settings, authenticated_user): setup_events = SetupEvent.objects.all() assert len(setup_events) == 4 setup_event = SetupEvent.objects.all()[3] - assert setup_event.step == 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' - assert setup_event.message == 'Failure' + assert setup_event.step == "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep" + assert setup_event.message == "Failure" profile = PortalProfile.objects.get(user=authenticated_user) assert not profile.setup_complete @@ -171,19 +151,13 @@ def test_error_step(settings, authenticated_user): """ Assert that when a setup step causes an error that the error is logged """ - settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockErrorStep' - } - ] + settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [{"step": "portal.apps.onboarding.steps.test_steps.MockErrorStep"}] with pytest.raises(StepExecuteException): prepare_setup_steps(authenticated_user) execute_setup_steps(authenticated_user.username) exception_event = SetupEvent.objects.all().filter( - user=authenticated_user, - step='portal.apps.onboarding.steps.test_steps.MockErrorStep', - state=SetupState.ERROR + user=authenticated_user, step="portal.apps.onboarding.steps.test_steps.MockErrorStep", state=SetupState.ERROR )[0] assert exception_event.message == "Exception: MockErrorStep" @@ -198,12 +172,8 @@ def test_userwait_step(settings, authenticated_user): should not execute due to the first one not being "COMPLETE". """ settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockUserStep' - }, - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - } + {"step": "portal.apps.onboarding.steps.test_steps.MockUserStep"}, + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep"}, ] with pytest.raises(StepExecuteException): prepare_setup_steps(authenticated_user) @@ -214,7 +184,7 @@ def test_userwait_step(settings, authenticated_user): setup_events = SetupEvent.objects.all() assert len(setup_events) == 2 setup_event = SetupEvent.objects.all()[1] - assert setup_event.step == 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' + assert setup_event.step == "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep" assert setup_event.state == SetupState.PENDING @@ -226,12 +196,8 @@ def test_sequence(settings, authenticated_user): MockProcessingFailStep should execute and fail, and leave a log event. """ settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - }, - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' - } + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep"}, + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep"}, ] with pytest.raises(StepExecuteException): prepare_setup_steps(authenticated_user) @@ -239,13 +205,13 @@ def test_sequence(settings, authenticated_user): setup_events = SetupEvent.objects.all() assert len(setup_events) == 6 - assert setup_events[2].step == 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' + assert setup_events[2].step == "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep" assert setup_events[2].state == SetupState.PROCESSING - assert setup_events[3].step == 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' + assert setup_events[3].step == "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep" assert setup_events[3].state == SetupState.COMPLETED - assert setup_events[4].step == 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' + assert setup_events[4].step == "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep" assert setup_events[4].state == SetupState.PROCESSING - assert setup_events[5].step == 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' + assert setup_events[5].step == "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep" assert setup_events[5].state == SetupState.FAILED @@ -258,12 +224,8 @@ def test_sequence_with_history(settings, authenticated_user): """ settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' - }, - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' - } + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep"}, + {"step": "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep"}, ] # Artificially fail MockProcessingCompleteStep @@ -294,12 +256,12 @@ def test_sequence_with_history(settings, authenticated_user): # MockPendingCompleteStep should appear in the log exactly twice complete_events = SetupEvent.objects.all().filter( - step='portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' + step="portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep" ) assert len(complete_events) == 2 # Last event should be MockPendingFailStep - assert setup_events[4].step == 'portal.apps.onboarding.steps.test_steps.MockProcessingFailStep' + assert setup_events[4].step == "portal.apps.onboarding.steps.test_steps.MockProcessingFailStep" assert setup_events[4].state == SetupState.FAILED @@ -317,8 +279,8 @@ def test_setup_steps_prepared_from_list(settings, authenticated_user, mocker): """ Assert that when there are setup steps, they are prepared for a user """ - settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = ['onboarding.step'] - mock_prepare = mocker.patch('portal.apps.onboarding.execute.prepare_setup_steps') + settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = ["onboarding.step"] + mock_prepare = mocker.patch("portal.apps.onboarding.execute.prepare_setup_steps") new_user_setup_check(authenticated_user) mock_prepare.assert_called_with(authenticated_user) @@ -328,10 +290,9 @@ def test_execute_single_step(mocker, authenticated_user): Test that the single step executor triggers a follow up execution of the rest of the step queue """ - mock_execute = mocker.patch('portal.apps.onboarding.execute.execute_setup_steps') + mock_execute = mocker.patch("portal.apps.onboarding.execute.execute_setup_steps") execute_single_step( - authenticated_user.username, - 'portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep' + authenticated_user.username, "portal.apps.onboarding.steps.test_steps.MockProcessingCompleteStep" ) mock_execute.assert_called_with(authenticated_user.username) @@ -341,9 +302,6 @@ def test_execute_single_step_does_not_complete(mocker, authenticated_user): Test that the single step executor does not trigger a follow up execution of the rest of the step queue if the step does not complete """ - mock_execute = mocker.patch('portal.apps.onboarding.execute.execute_setup_steps') - execute_single_step( - authenticated_user.username, - 'portal.apps.onboarding.steps.test_steps.MockUserStep' - ) + mock_execute = mocker.patch("portal.apps.onboarding.execute.execute_setup_steps") + execute_single_step(authenticated_user.username, "portal.apps.onboarding.steps.test_steps.MockUserStep") mock_execute.assert_not_called() diff --git a/server/portal/apps/onboarding/migrations/0001_initial.py b/server/portal/apps/onboarding/migrations/0001_initial.py index 5c0772d070..b53eb797cc 100644 --- a/server/portal/apps/onboarding/migrations/0001_initial.py +++ b/server/portal/apps/onboarding/migrations/0001_initial.py @@ -7,7 +7,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -16,15 +15,20 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='SetupEvent', + name="SetupEvent", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('time', models.DateTimeField(auto_now_add=True)), - ('step', models.CharField(max_length=300)), - ('state', models.CharField(max_length=16)), - ('message', models.CharField(max_length=300)), - ('data', django.contrib.postgres.fields.jsonb.JSONField(null=True)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("time", models.DateTimeField(auto_now_add=True)), + ("step", models.CharField(max_length=300)), + ("state", models.CharField(max_length=16)), + ("message", models.CharField(max_length=300)), + ("data", django.contrib.postgres.fields.jsonb.JSONField(null=True)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), ], ), ] diff --git a/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py b/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py index 3c6a143bf9..c84ed2ae04 100644 --- a/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py +++ b/server/portal/apps/onboarding/migrations/0001_squashed_0002_alter_setupevent_data.py @@ -6,8 +6,7 @@ class Migration(migrations.Migration): - - replaces = [('onboarding', '0001_initial'), ('onboarding', '0002_alter_setupevent_data')] + replaces = [("onboarding", "0001_initial"), ("onboarding", "0002_alter_setupevent_data")] initial = True @@ -17,15 +16,20 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='SetupEvent', + name="SetupEvent", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('time', models.DateTimeField(auto_now_add=True)), - ('step', models.CharField(max_length=300)), - ('state', models.CharField(max_length=16)), - ('message', models.CharField(max_length=300)), - ('data', models.JSONField(null=True)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("time", models.DateTimeField(auto_now_add=True)), + ("step", models.CharField(max_length=300)), + ("state", models.CharField(max_length=16)), + ("message", models.CharField(max_length=300)), + ("data", models.JSONField(null=True)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), ], ), ] diff --git a/server/portal/apps/onboarding/migrations/0002_alter_setupevent_data.py b/server/portal/apps/onboarding/migrations/0002_alter_setupevent_data.py index 3e56f025b8..e3e767ffb2 100644 --- a/server/portal/apps/onboarding/migrations/0002_alter_setupevent_data.py +++ b/server/portal/apps/onboarding/migrations/0002_alter_setupevent_data.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('onboarding', '0001_initial'), + ("onboarding", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='setupevent', - name='data', + model_name="setupevent", + name="data", field=models.JSONField(null=True), ), ] diff --git a/server/portal/apps/onboarding/models.py b/server/portal/apps/onboarding/models.py index e953828944..b0a295594f 100644 --- a/server/portal/apps/onboarding/models.py +++ b/server/portal/apps/onboarding/models.py @@ -13,11 +13,8 @@ class SetupEvent(models.Model): A log of events for setup steps """ - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name="+", - on_delete=models.CASCADE - ) + + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="+", on_delete=models.CASCADE) # Auto increment auto add timestamp for event time = models.DateTimeField(auto_now_add=True) @@ -36,13 +33,13 @@ class SetupEvent(models.Model): data = models.JSONField(null=True) def __str__(self): - return '{username} {time} {step} ({state}) - {message} ({data})'.format( + return "{username} {time} {step} ({state}) - {message} ({data})".format( username=self.user.username, time=self.time, step=self.step, state=self.state, message=self.message, - data=self.data + data=self.data, ) def to_dict(self): @@ -58,7 +55,7 @@ def to_dict(self): class SetupEventEncoder(DjangoJSONEncoder): def default(self, obj): # pylint: disable=method-hidden, arguments-differ - if (isinstance(obj, SetupEvent)): + if isinstance(obj, SetupEvent): event = obj return event.to_dict() else: diff --git a/server/portal/apps/onboarding/models_unit_test.py b/server/portal/apps/onboarding/models_unit_test.py index 39a60cb817..f36849a674 100644 --- a/server/portal/apps/onboarding/models_unit_test.py +++ b/server/portal/apps/onboarding/models_unit_test.py @@ -1,4 +1,3 @@ - from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.models import SetupEvent import pytest @@ -10,10 +9,7 @@ @pytest.fixture def onboarding_event(authenticated_user): event = SetupEvent.objects.create( - user=authenticated_user, - state=SetupState.PENDING, - step="TestStep", - message="test message" + user=authenticated_user, state=SetupState.PENDING, step="TestStep", message="test message" ) yield event diff --git a/server/portal/apps/onboarding/steps/abstract.py b/server/portal/apps/onboarding/steps/abstract.py index f96f998fe4..26dbc03ec8 100644 --- a/server/portal/apps/onboarding/steps/abstract.py +++ b/server/portal/apps/onboarding/steps/abstract.py @@ -23,20 +23,15 @@ def __init__(self, user): try: steps = settings.PORTAL_USER_ACCOUNT_SETUP_STEPS - step_dict = next( - step for step in steps if step['step'] == self.step_name() - ) - self.settings = step_dict['settings'] + step_dict = next(step for step in steps if step["step"] == self.step_name()) + self.settings = step_dict["settings"] except Exception: self.settings = None try: # Restore event history self.events = [ - event for event in SetupEvent.objects.filter( - user=user, - step=self.step_name() - ).order_by('time') + event for event in SetupEvent.objects.filter(user=user, step=self.step_name()).order_by("time") ] self.last_event = self.events[-1] if len(self.events) > 0 else None self.state = self.last_event.state @@ -49,11 +44,7 @@ def log(self, message, data=None): needs to set the state of the setup step for this user. """ self.last_event = SetupEvent.objects.create( - user=self.user, - step=self.step_name(), - state=self.state, - message=message, - data=data + user=self.user, step=self.step_name(), state=self.state, message=message, data=data ) self.events.append(self.last_event) @@ -80,16 +71,11 @@ def complete(self, message, data=None): def __str__(self): return "<{step} for {username} is {state}>".format( - step=self.step_name(), - state=self.state, - username=self.user.username + step=self.step_name(), state=self.state, username=self.user.username ) def step_name(self): - return "{module}.{classname}".format( - module=self.__module__, - classname=self.__class__.__name__ - ) + return "{module}.{classname}".format(module=self.__module__, classname=self.__class__.__name__) @abstractmethod def display_name(self): diff --git a/server/portal/apps/onboarding/steps/abstract_unit_test.py b/server/portal/apps/onboarding/steps/abstract_unit_test.py index 108b3f2cad..31887afe08 100644 --- a/server/portal/apps/onboarding/steps/abstract_unit_test.py +++ b/server/portal/apps/onboarding/steps/abstract_unit_test.py @@ -64,7 +64,7 @@ def test_str(mock_step): def test_settings(mock_step): - assert mock_step.settings == {'key': 'value'} + assert mock_step.settings == {"key": "value"} def test_step_missing(regular_user, settings): @@ -76,7 +76,7 @@ def test_step_missing(regular_user, settings): def test_step_setting_missing(regular_user, settings): settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { - 'step': 'portal.apps.onboarding.steps.test_steps.MockStep', + "step": "portal.apps.onboarding.steps.test_steps.MockStep", } ] mock_step = MockStep(regular_user) diff --git a/server/portal/apps/onboarding/steps/access.py b/server/portal/apps/onboarding/steps/access.py index eaa000ed5a..23e6577fef 100644 --- a/server/portal/apps/onboarding/steps/access.py +++ b/server/portal/apps/onboarding/steps/access.py @@ -43,18 +43,8 @@ def client_action(self, action, data, request): return if action == "staff_approve": - self.complete( - "Portal access request approved by {user}".format( - user=request.user.username - ) - ) + self.complete("Portal access request approved by {user}".format(user=request.user.username)) elif action == "staff_deny": - self.deny( - "Portal access request has not been approved." - ) + self.deny("Portal access request has not been approved.") else: - self.fail( - "Invalid client action {action}".format( - action=action - ) - ) + self.fail("Invalid client action {action}".format(action=action)) diff --git a/server/portal/apps/onboarding/steps/access_unit_test.py b/server/portal/apps/onboarding/steps/access_unit_test.py index 10d8eab8b0..ee9c10a3c2 100644 --- a/server/portal/apps/onboarding/steps/access_unit_test.py +++ b/server/portal/apps/onboarding/steps/access_unit_test.py @@ -13,15 +13,15 @@ def setUp(self): # Create a test user User = get_user_model() - self.user = User.objects.create_user('test', 'test@test.com', 'test') + self.user = User.objects.create_user("test", "test@test.com", "test") - self.staff = User.objects.create_user('staff', 'staff@staff.com', 'staff') + self.staff = User.objects.create_user("staff", "staff@staff.com", "staff") self.staff.is_staff = True def tearDown(self): super(TestRequestAccessStep, self).tearDown() - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.log') + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.log") def test_prepare(self, mock_log): # prepare should log a STAFFWAIT state step = RequestAccessStep(self.user) @@ -29,31 +29,31 @@ def test_prepare(self, mock_log): self.assertEqual(step.state, SetupState.PENDING) mock_log.assert_called_with(ANY) - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.fail') - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.complete') + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.fail") + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.complete") def test_not_staff(self, mock_complete, mock_fail): step = RequestAccessStep(self.user) - request = RequestFactory().post('/api/setup/test') + request = RequestFactory().post("/api/setup/test") request.user = self.user step.client_action("staff_approve", None, request) mock_complete.assert_not_called() mock_fail.assert_not_called() - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.complete') + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.complete") def test_staff_approve(self, mock_complete): # staff_approve should log a COMPLETED state step = RequestAccessStep(self.user) - request = RequestFactory().post('/api/setup/test') + request = RequestFactory().post("/api/setup/test") request.user = self.staff step.client_action("staff_approve", None, request) mock_complete.assert_called_with(ANY) - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.fail') - @patch('portal.apps.onboarding.steps.access.RequestAccessStep.deny') + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.fail") + @patch("portal.apps.onboarding.steps.access.RequestAccessStep.deny") def test_fail_actions(self, mock_deny, mock_fail): # staff_approve should log a FAILED state step = RequestAccessStep(self.user) - request = RequestFactory().post('/api/setup/test') + request = RequestFactory().post("/api/setup/test") request.user = self.staff step.client_action("staff_deny", None, request) mock_deny.assert_called_with(ANY) diff --git a/server/portal/apps/onboarding/steps/allocation.py b/server/portal/apps/onboarding/steps/allocation.py index 99d73feb83..f3f5d4b379 100644 --- a/server/portal/apps/onboarding/steps/allocation.py +++ b/server/portal/apps/onboarding/steps/allocation.py @@ -44,11 +44,7 @@ def process(self): if missing_hosts: self.state = SetupState.FAILED - self.log( - "User {0} is missing allocations on: {1}".format( - self.user.username, missing_hosts - ) - ) + self.log("User {0} is missing allocations on: {1}".format(self.user.username, missing_hosts)) return self.log("Expected host allocations found: {0}".format(matched_hosts)) diff --git a/server/portal/apps/onboarding/steps/allocation_unit_test.py b/server/portal/apps/onboarding/steps/allocation_unit_test.py index bfa926b654..80d024764c 100644 --- a/server/portal/apps/onboarding/steps/allocation_unit_test.py +++ b/server/portal/apps/onboarding/steps/allocation_unit_test.py @@ -6,7 +6,7 @@ @pytest.fixture def get_allocations_mock(mocker): - get_allocations = mocker.patch('portal.apps.onboarding.steps.allocation.get_allocations') + get_allocations = mocker.patch("portal.apps.onboarding.steps.allocation.get_allocations") get_allocations.return_value = { "hosts": { "vista.tacc.utexas.edu": {}, @@ -21,19 +21,14 @@ def get_allocations_mock(mocker): @pytest.fixture def get_allocations_failure_mock(mocker): - get_allocations = mocker.patch('portal.apps.onboarding.steps.allocation.get_allocations') - get_allocations.return_value = {'hosts': {}, - 'portal_alloc': None, - 'active': [], - 'inactive': []} + get_allocations = mocker.patch("portal.apps.onboarding.steps.allocation.get_allocations") + get_allocations.return_value = {"hosts": {}, "portal_alloc": None, "active": [], "inactive": []} yield get_allocations @pytest.fixture def get_allocations_with_expected_systems_check_failure_mock(mocker): - get_allocations = mocker.patch( - "portal.apps.onboarding.steps.allocation.get_allocations" - ) + get_allocations = mocker.patch("portal.apps.onboarding.steps.allocation.get_allocations") get_allocations.return_value = { "hosts": { "vista.tacc.utexas.edu": {}, @@ -48,12 +43,12 @@ def get_allocations_with_expected_systems_check_failure_mock(mocker): @pytest.fixture def allocation_step_complete_mock(mocker): - yield mocker.patch.object(AllocationStep, 'complete') + yield mocker.patch.object(AllocationStep, "complete") @pytest.fixture def allocation_step_log_mock(mocker): - yield mocker.patch.object(AllocationStep, 'log') + yield mocker.patch.object(AllocationStep, "log") def test_get_allocations_with_expected_systems_check_success( @@ -62,9 +57,7 @@ def test_get_allocations_with_expected_systems_check_success( settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { "step": "portal.apps.onboarding.steps.allocation.AllocationStep", - "settings": { - "expected_hosts": ["vista.tacc.utexas.edu", "frontera.tacc.utexas.edu"] - }, + "settings": {"expected_hosts": ["vista.tacc.utexas.edu", "frontera.tacc.utexas.edu"]}, } ] step = AllocationStep(regular_user) @@ -92,19 +85,13 @@ def test_get_allocations_with_expected_systems_check_failure( settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { "step": "portal.apps.onboarding.steps.allocation.AllocationStep", - "settings": { - "expected_hosts": ["blah.tacc.utexas.edu", "frontera.tacc.utexas.edu"] - }, + "settings": {"expected_hosts": ["blah.tacc.utexas.edu", "frontera.tacc.utexas.edu"]}, } ] step = AllocationStep(regular_user) step.process() - get_allocations_with_expected_systems_check_failure_mock.assert_called_with( - "username", force=True - ) - allocation_step_log_mock.assert_called_with( - "User username is missing allocations on: ['blah.tacc.utexas.edu']" - ) + get_allocations_with_expected_systems_check_failure_mock.assert_called_with("username", force=True) + allocation_step_log_mock.assert_called_with("User username is missing allocations on: ['blah.tacc.utexas.edu']") def test_get_allocations_without_expected_systems_check_success( diff --git a/server/portal/apps/onboarding/steps/mfa.py b/server/portal/apps/onboarding/steps/mfa.py index 9629cbad12..0800fe89eb 100644 --- a/server/portal/apps/onboarding/steps/mfa.py +++ b/server/portal/apps/onboarding/steps/mfa.py @@ -34,8 +34,8 @@ def prepare(self): def mfa_check(self): auth = HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) response = requests.get(self.tas_pairings_url(), auth=auth) - pairings = response.json()['result'] - return any(pairing['type'] == 'tacc-soft-token' for pairing in pairings) + pairings = response.json()["result"] + return any(pairing["type"] == "tacc-soft-token" for pairing in pairings) def process(self): if self.mfa_check(): diff --git a/server/portal/apps/onboarding/steps/mfa_unit_test.py b/server/portal/apps/onboarding/steps/mfa_unit_test.py index f2b6254bcd..89f13de521 100644 --- a/server/portal/apps/onboarding/steps/mfa_unit_test.py +++ b/server/portal/apps/onboarding/steps/mfa_unit_test.py @@ -4,31 +4,29 @@ @pytest.fixture def mock_mfa_check(mocker): - yield mocker.patch('portal.apps.onboarding.steps.mfa_unit_test.MFAStep.mfa_check', autospec=True) + yield mocker.patch("portal.apps.onboarding.steps.mfa_unit_test.MFAStep.mfa_check", autospec=True) @pytest.fixture def mock_mfa_log(mocker): - yield mocker.patch.object(MFAStep, 'log') + yield mocker.patch.object(MFAStep, "log") @pytest.fixture def mock_mfa_complete(mocker): - yield mocker.patch.object(MFAStep, 'complete') + yield mocker.patch.object(MFAStep, "complete") @pytest.fixture def mock_mfa_prepare(mocker): - yield mocker.patch.object(MFAStep, 'prepare') + yield mocker.patch.object(MFAStep, "prepare") def test_mfa_found(authenticated_user, mock_mfa_check, mock_mfa_complete): mock_mfa_check.return_value = True step = MFAStep(authenticated_user) step.process() - mock_mfa_complete.assert_called_with( - "Multi-factor authentication pairing verified" - ) + mock_mfa_complete.assert_called_with("Multi-factor authentication pairing verified") def test_mfa_not_found(mocker, authenticated_user, mock_mfa_check, mock_mfa_log): diff --git a/server/portal/apps/onboarding/steps/project_membership.py b/server/portal/apps/onboarding/steps/project_membership.py index 70e9091497..8aa509476f 100644 --- a/server/portal/apps/onboarding/steps/project_membership.py +++ b/server/portal/apps/onboarding/steps/project_membership.py @@ -22,23 +22,16 @@ def __init__(self, user): if isinstance(self.settings["project_sql_id"], list) else [self.settings["project_sql_id"]] ) - self.default_project_sql_id = self.settings.get( - "default_project_sql_id", self.project_sql_ids[0] - ) + self.default_project_sql_id = self.settings.get("default_project_sql_id", self.project_sql_ids[0]) self.default_project = self.get_tas_project(self.default_project_sql_id) self.user_confirm = "Request Project Access" - self.staff_approve = "Add to {project}".format( - project=self.default_project["title"] - ) + self.staff_approve = "Add to {project}".format(project=self.default_project["title"]) self.staff_deny = "Deny Project Access Request" def get_tas_client(self): tas_client = TASClient( baseURL=settings.TAS_URL, - credentials={ - 'username': settings.TAS_CLIENT_KEY, - 'password': settings.TAS_CLIENT_SECRET - } + credentials={"username": settings.TAS_CLIENT_KEY, "password": settings.TAS_CLIENT_SECRET}, ) return tas_client @@ -46,8 +39,8 @@ def get_tas_project(self, project_sql_id): return self.get_tas_client().project(project_sql_id) def description(self): - if self.settings is not None and 'description' in self.settings: - return self.settings['description'] + if self.settings is not None and "description" in self.settings: + return self.settings["description"] return """This confirms if you have access to the project. If not, request access and wait for the system administrator’s approval.""" @@ -60,13 +53,7 @@ def prepare(self): def get_tracker(self): return Rt( - settings.RT_HOST, - settings.RT_UN, - settings.RT_PW, - http_auth=HTTPBasicAuth( - settings.RT_UN, - settings.RT_PW - ) + settings.RT_HOST, settings.RT_UN, settings.RT_PW, http_auth=HTTPBasicAuth(settings.RT_UN, settings.RT_PW) ) def is_project_member(self): @@ -74,53 +61,43 @@ def is_project_member(self): tas_client = self.get_tas_client() for project_id in self.project_sql_ids: project_users = tas_client.get_project_users(project_id) - if any([u['username'] == username for u in project_users]): + if any([u["username"] == username for u in project_users]): return True return False def send_project_request(self, request): tracker = self.get_tracker() - ticket_text = 'User {username} is requesting membership on the {project} project. Please visit ' - ticket_text += '{onboarding_url} to complete this request.' + ticket_text = "User {username} is requesting membership on the {project} project. Please visit " + ticket_text += "{onboarding_url} to complete this request." ticket_text = ticket_text.format( username=self.user.username, - project=self.default_project['title'], + project=self.default_project["title"], onboarding_url=request.build_absolute_uri( - '/workbench/onboarding/setup/{username}'.format( - username=self.user.username - ) + "/workbench/onboarding/setup/{username}".format(username=self.user.username) ), ) try: if tracker.login(): result = tracker.create_ticket( - Queue=self.settings.get('rt_queue') or 'Accounting', - Subject='{project} Project Membership Request for {username}'.format( - project=self.default_project['title'], - username=self.user.username + Queue=self.settings.get("rt_queue") or "Accounting", + Subject="{project} Project Membership Request for {username}".format( + project=self.default_project["title"], username=self.user.username ), Text=ticket_text, Requestor=self.user.email, - CF_resource=settings.RT_TAG + CF_resource=settings.RT_TAG, ) tracker.logout() self.state = SetupState.STAFFWAIT - self.log( - "Thank you for your request. It will be reviewed by TACC staff.", - data={ - "ticket": result - } - ) + self.log("Thank you for your request. It will be reviewed by TACC staff.", data={"ticket": result}) else: raise Exception("Could not log in to RT") except Exception as err: logger.exception(msg="Could not create ticket on behalf of user during ProjectMembershipStep") logger.error(err.args) - self.fail( - "We were unable to submit a portal access request ticket on your behalf." - ) + self.fail("We were unable to submit a portal access request ticket on your behalf.") def add_to_project(self): tas_client = self.get_tas_client() @@ -136,16 +113,13 @@ def add_to_project(self): if "is already a member" in reason: self.complete( "{username} is already a member of the {project}".format( - username=self.user.username, - project=self.default_project['title'] + username=self.user.username, project=self.default_project["title"] ) ) else: self.fail( "{username} could not be added to {project} due to error {reason}".format( - project=self.default_project['title'], - username=self.user.username, - reason=reason + project=self.default_project["title"], username=self.user.username, reason=reason ) ) raise e @@ -158,26 +132,20 @@ def deny_project_request(self): tracker = self.get_tracker() request_text = """Your request for membership on the {project} project has been denied. If you believe this is an error, please submit a help ticket. - """.format( - project=self.default_project['title'] - ) + """.format(project=self.default_project["title"]) if tracker.login(): tracker.reply(ticket_id, text=request_text) tracker.comment( ticket_id, text="User was not added to the {project} TAS Project (GID {gid}) at {base_url}".format( - project=self.default_project['title'], - gid=self.default_project['gid'], - base_url=settings.VANITY_BASE_URL - ) + project=self.default_project["title"], + gid=self.default_project["gid"], + base_url=settings.VANITY_BASE_URL, + ), ) - tracker.edit_ticket(ticket_id, Status='resolved') + tracker.edit_ticket(ticket_id, Status="resolved") else: - self.fail( - "The portal was unable to close RT Ticket {ticket_id}".format( - ticket_id=ticket_id - ) - ) + self.fail("The portal was unable to close RT Ticket {ticket_id}".format(ticket_id=ticket_id)) def close_project_request(self, deny=False): ticket_id = None @@ -187,27 +155,20 @@ def close_project_request(self, deny=False): tracker = self.get_tracker() request_text = """Your request for membership on the {project} project has been granted. Please login at {base_url}/workbench/onboarding/setup to continue setting up your account. - """.format( - project=self.default_project['title'], - base_url=settings.VANITY_BASE_URL - ) + """.format(project=self.default_project["title"], base_url=settings.VANITY_BASE_URL) if tracker.login(): tracker.reply(ticket_id, text=request_text) tracker.comment( ticket_id, text="User has been added to the {project} TAS Project (GID {gid}) via {base_url}".format( - project=self.default_project['title'], - gid=self.default_project['gid'], - base_url=settings.VANITY_BASE_URL - ) + project=self.default_project["title"], + gid=self.default_project["gid"], + base_url=settings.VANITY_BASE_URL, + ), ) - tracker.edit_ticket(ticket_id, Status='resolved') + tracker.edit_ticket(ticket_id, Status="resolved") else: - self.fail( - "The portal was unable to close RT Ticket {ticket_id}".format( - ticket_id=ticket_id - ) - ) + self.fail("The portal was unable to close RT Ticket {ticket_id}".format(ticket_id=ticket_id)) def process(self): if self.is_project_member(): @@ -215,14 +176,9 @@ def process(self): else: self.state = SetupState.USERWAIT data = None - if self.settings is not None and 'userlink' in self.settings: - data = { - 'userlink': self.settings['userlink'] - } - self.log( - "Please confirm your request to use this portal.", - data=data - ) + if self.settings is not None and "userlink" in self.settings: + data = {"userlink": self.settings["userlink"]} + self.log("Please confirm your request to use this portal.", data=data) def client_action(self, action, data, request): if action == "user_confirm": @@ -233,25 +189,13 @@ def client_action(self, action, data, request): try: self.add_to_project() self.close_project_request() - self.complete( - "Portal access request approved by {user}".format( - user=request.user.username - ) - ) + self.complete("Portal access request approved by {user}".format(user=request.user.username)) except Exception as err: logger.exception(msg="Error during staff_approve on {}".format(self.step_name())) logger.error(err.args) - self.fail( - "An error occurred while trying to add this user to the project" - ) + self.fail("An error occurred while trying to add this user to the project") elif action == "staff_deny": self.deny_project_request() - self.deny( - "Portal access request has not been approved." - ) + self.deny("Portal access request has not been approved.") else: - self.fail( - "Invalid client action {action}".format( - action=action - ) - ) + self.fail("Invalid client action {action}".format(action=action)) diff --git a/server/portal/apps/onboarding/steps/project_membership_unit_test.py b/server/portal/apps/onboarding/steps/project_membership_unit_test.py index 971695dd72..708ecf7b6a 100644 --- a/server/portal/apps/onboarding/steps/project_membership_unit_test.py +++ b/server/portal/apps/onboarding/steps/project_membership_unit_test.py @@ -9,11 +9,11 @@ @pytest.fixture def tas_client(mocker): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_project.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_project.json")) as f: tas_project = json.load(f) - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_project_users.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_project_users.json")) as f: tas_project_users = json.load(f) - tas_client_mock = mocker.patch('portal.apps.onboarding.steps.project_membership.TASClient', autospec=True) + tas_client_mock = mocker.patch("portal.apps.onboarding.steps.project_membership.TASClient", autospec=True) tas_client_mock.return_value.project.return_value = tas_project tas_client_mock.return_value.get_project_users.return_value = tas_project_users yield tas_client_mock @@ -21,9 +21,7 @@ def tas_client(mocker): @pytest.fixture def mock_rt(mocker): - mock_tracker = mocker.patch( - 'portal.apps.onboarding.steps.project_membership.ProjectMembershipStep.get_tracker' - ) + mock_tracker = mocker.patch("portal.apps.onboarding.steps.project_membership.ProjectMembershipStep.get_tracker") mock_tracker.return_value.login.return_value = True yield mock_tracker @@ -32,10 +30,8 @@ def mock_rt(mocker): def project_membership_step(settings, regular_user, tas_client, mock_rt): settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { - 'step': 'portal.apps.onboarding.steps.project_membership.ProjectMembershipStep', - 'settings': { - 'project_sql_id': 12345 - } + "step": "portal.apps.onboarding.steps.project_membership.ProjectMembershipStep", + "settings": {"project_sql_id": 12345}, } ] step = ProjectMembershipStep(regular_user) @@ -46,15 +42,12 @@ def project_membership_step(settings, regular_user, tas_client, mock_rt): def project_membership_step_with_userlink(settings, regular_user, tas_client): settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { - 'step': 'portal.apps.onboarding.steps.project_membership.ProjectMembershipStep', - 'settings': { - 'project_sql_id': 12345, - 'userlink': { - 'url': '/', - 'text': 'Request Access' - }, + "step": "portal.apps.onboarding.steps.project_membership.ProjectMembershipStep", + "settings": { + "project_sql_id": 12345, + "userlink": {"url": "/", "text": "Request Access"}, }, - 'retry': True + "retry": True, } ] step = ProjectMembershipStep(regular_user) @@ -63,17 +56,17 @@ def project_membership_step_with_userlink(settings, regular_user, tas_client): @pytest.fixture def project_membership_log(mocker): - yield mocker.patch.object(ProjectMembershipStep, 'log') + yield mocker.patch.object(ProjectMembershipStep, "log") @pytest.fixture def project_membership_fail(mocker): - yield mocker.patch.object(ProjectMembershipStep, 'fail') + yield mocker.patch.object(ProjectMembershipStep, "fail") @pytest.fixture def project_membership_complete(mocker): - yield mocker.patch.object(ProjectMembershipStep, 'complete') + yield mocker.patch.object(ProjectMembershipStep, "complete") def test_is_project_member(tas_client, project_membership_step): @@ -85,37 +78,29 @@ def test_is_project_member(tas_client, project_membership_step): def test_process_user_is_member(monkeypatch, project_membership_step, project_membership_complete): def mock_is_project_member(): return True - monkeypatch.setattr(project_membership_step, 'is_project_member', mock_is_project_member) + + monkeypatch.setattr(project_membership_step, "is_project_member", mock_is_project_member) project_membership_step.process() - project_membership_complete.assert_called_with( - "You have the required project membership to access this portal." - ) + project_membership_complete.assert_called_with("You have the required project membership to access this portal.") def test_process_user_is_not_member(monkeypatch, project_membership_step, project_membership_log): def mock_is_project_member(): return False - monkeypatch.setattr(project_membership_step, 'is_project_member', mock_is_project_member) + + monkeypatch.setattr(project_membership_step, "is_project_member", mock_is_project_member) project_membership_step.process() - project_membership_log.assert_called_with( - "Please confirm your request to use this portal.", - data=None - ) + project_membership_log.assert_called_with("Please confirm your request to use this portal.", data=None) def test_process_userlink(monkeypatch, project_membership_step_with_userlink, project_membership_log): def mock_is_project_member(): return False - monkeypatch.setattr(project_membership_step_with_userlink, 'is_project_member', mock_is_project_member) + + monkeypatch.setattr(project_membership_step_with_userlink, "is_project_member", mock_is_project_member) project_membership_step_with_userlink.process() project_membership_log.assert_called_with( - "Please confirm your request to use this portal.", - data={ - 'userlink': { - 'url': '/', - 'text': 'Request Access' - } - } + "Please confirm your request to use this portal.", data={"userlink": {"url": "/", "text": "Request Access"}} ) @@ -127,12 +112,9 @@ def test_send_project_request(rf, project_membership_step, project_membership_lo def test_add_to_project(regular_user, project_membership_step, tas_client, mocker): - mocker.patch('portal.apps.onboarding.steps.project_membership.index_allocations') + mocker.patch("portal.apps.onboarding.steps.project_membership.index_allocations") project_membership_step.add_to_project() - tas_client.return_value.add_project_user.assert_called_with( - 12345, - regular_user.username - ) + tas_client.return_value.add_project_user.assert_called_with(12345, regular_user.username) def test_close_project_request(regular_user, project_membership_step, mock_rt): @@ -140,12 +122,12 @@ def test_close_project_request(regular_user, project_membership_step, mock_rt): SetupEvent(user=regular_user), SetupEvent(user=regular_user, data={}), SetupEvent(user=regular_user, data={"ticket": "1234"}), - SetupEvent(user=regular_user, data={"ticket": "12345"}) + SetupEvent(user=regular_user, data={"ticket": "12345"}), ] project_membership_step.close_project_request() mock_rt.return_value.reply.assert_called_with("12345", text=ANY) mock_rt.return_value.comment.assert_called_with("12345", text=ANY) - mock_rt.return_value.edit_ticket.assert_called_with("12345", Status='resolved') + mock_rt.return_value.edit_ticket.assert_called_with("12345", Status="resolved") def test_client_action(regular_user, rf, monkeypatch, project_membership_step, project_membership_complete): @@ -154,9 +136,9 @@ def test_client_action(regular_user, rf, monkeypatch, project_membership_step, p mock_send = MagicMock() mock_add = MagicMock() mock_close = MagicMock() - monkeypatch.setattr(project_membership_step, 'send_project_request', mock_send) - monkeypatch.setattr(project_membership_step, 'add_to_project', mock_add) - monkeypatch.setattr(project_membership_step, 'close_project_request', mock_close) + monkeypatch.setattr(project_membership_step, "send_project_request", mock_send) + monkeypatch.setattr(project_membership_step, "add_to_project", mock_add) + monkeypatch.setattr(project_membership_step, "close_project_request", mock_close) project_membership_step.client_action("user_confirm", {}, request) mock_send.assert_called_with(request) request.user.is_staff = True @@ -168,11 +150,9 @@ def test_client_action(regular_user, rf, monkeypatch, project_membership_step, p def test_client_action_fail(rf, regular_user, monkeypatch, project_membership_step, project_membership_fail): mock_add = MagicMock(side_effect=Exception("Mock exception", "Mock reason")) - monkeypatch.setattr(project_membership_step, 'add_to_project', mock_add) + monkeypatch.setattr(project_membership_step, "add_to_project", mock_add) request = rf.get("/api/onboarding") request.user = regular_user request.user.is_staff = True project_membership_step.client_action("staff_approve", {}, request) - project_membership_fail.assert_called_with( - "An error occurred while trying to add this user to the project" - ) + project_membership_fail.assert_called_with("An error occurred while trying to add this user to the project") diff --git a/server/portal/apps/onboarding/steps/system_access.py b/server/portal/apps/onboarding/steps/system_access.py index a65b889340..32ef470d51 100644 --- a/server/portal/apps/onboarding/steps/system_access.py +++ b/server/portal/apps/onboarding/steps/system_access.py @@ -28,13 +28,13 @@ def prepare(self): self.log("Awaiting system access check") def has_required_systems(self): - systems = self.settings['required_systems'] + systems = self.settings["required_systems"] if len(systems) == 0: return True resources = [] try: - resources = get_allocations(self.user.username)['hosts'].keys() + resources = get_allocations(self.user.username)["hosts"].keys() # If the intersection of the set of systems and resources has # items, the user has the necessary allocation return len(set(systems).intersection(resources)) > 0 @@ -48,6 +48,4 @@ def process(self): self.complete("You have the required systems for accessing this portal") else: self.state = SetupState.USERWAIT - self.log( - "Please confirm your request to use this portal." - ) + self.log("Please confirm your request to use this portal.") diff --git a/server/portal/apps/onboarding/steps/system_access_unit_test.py b/server/portal/apps/onboarding/steps/system_access_unit_test.py index 3a2610c522..a629236b65 100644 --- a/server/portal/apps/onboarding/steps/system_access_unit_test.py +++ b/server/portal/apps/onboarding/steps/system_access_unit_test.py @@ -7,9 +7,9 @@ @pytest.fixture def tas_client(mocker): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_project.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_project.json")) as f: tas_project = json.load(f) - tas_client_mock = mocker.patch('portal.apps.onboarding.steps.project_membership.TASClient', autospec=True) + tas_client_mock = mocker.patch("portal.apps.onboarding.steps.project_membership.TASClient", autospec=True) tas_client_mock.return_value.project.return_value = tas_project tas_client_mock.return_value.projects_for_user.return_value = [tas_project] yield tas_client_mock @@ -17,18 +17,18 @@ def tas_client(mocker): @pytest.fixture def mock_user_allocations(mocker): - yield mocker.patch('portal.apps.onboarding.steps.system_access.get_allocations', autospec=True) + yield mocker.patch("portal.apps.onboarding.steps.system_access.get_allocations", autospec=True) @pytest.fixture def system_access_step(settings, regular_user, tas_client, mock_user_allocations): settings.PORTAL_USER_ACCOUNT_SETUP_STEPS = [ { - 'step': 'portal.apps.onboarding.steps.system_access.SystemAccessStep', - 'settings': { - 'required_systems': ['stampede2.tacc.utexas.edu', 'ls5.tacc.utexas.edu'], - 'project_sql_id': 12345 - } + "step": "portal.apps.onboarding.steps.system_access.SystemAccessStep", + "settings": { + "required_systems": ["stampede2.tacc.utexas.edu", "ls5.tacc.utexas.edu"], + "project_sql_id": 12345, + }, } ] step = SystemAccessStep(regular_user) @@ -36,9 +36,9 @@ def system_access_step(settings, regular_user, tas_client, mock_user_allocations def test_has_required_systems(system_access_step, mock_user_allocations): - with open(os.path.join(settings.BASE_DIR, 'apps/users/fixtures/user_allocations.json')) as f: + with open(os.path.join(settings.BASE_DIR, "apps/users/fixtures/user_allocations.json")) as f: user_allocations = json.load(f) mock_user_allocations.return_value = user_allocations assert system_access_step.has_required_systems() - mock_user_allocations.return_value = {'hosts': {}} + mock_user_allocations.return_value = {"hosts": {}} assert not system_access_step.has_required_systems() diff --git a/server/portal/apps/onboarding/steps/system_access_v3.py b/server/portal/apps/onboarding/steps/system_access_v3.py index 79a9f3fd60..6b00fd0097 100644 --- a/server/portal/apps/onboarding/steps/system_access_v3.py +++ b/server/portal/apps/onboarding/steps/system_access_v3.py @@ -19,9 +19,7 @@ def create_system_credentials_with_password( """ Set a username/password as the user's auth credential on a Tapis system. """ - logger.info( - f"Creating user credential for {username} on Tapis system {system_id} using password" - ) + logger.info(f"Creating user credential for {username} on Tapis system {system_id} using password") data = { "password": password, "loginUser": loginUser or username, @@ -46,9 +44,7 @@ def create_system_credentials_with_keys( """ Set an RSA key pair as the user's auth credential on a Tapis system. """ - logger.info( - f"Creating user credential for {username} on Tapis system {system_id} using keys" - ) + logger.info(f"Creating user credential for {username} on Tapis system {system_id} using keys") data = { "privateKey": private_key, "publicKey": public_key, @@ -71,14 +67,9 @@ def create_system_credentials_with_tms( """ Create user's auth credential on a Tapis system. This Tapis API uses TMS. """ - logger.info( - f"Creating user credential for {username} on Tapis system {system_id} using TMS" - ) + logger.info(f"Creating user credential for {username} on Tapis system {system_id} using TMS") client.systems.createUserCredential( - systemId=system_id, - userName=username, - createTmsKeys=True, - skipCredentialCheck=skipCredentialCheck + systemId=system_id, userName=username, createTmsKeys=True, skipCredentialCheck=skipCredentialCheck ) @@ -86,16 +77,11 @@ def set_user_permissions(user, system_id): """Apply read/write/execute permissions to files and read permissions on the system.""" logger.info(f"Adding {user.username} permissions to Tapis system {system_id}") client = service_account() - client.systems.grantUserPerms( - systemId=system_id, userName=user.username, permissions=["READ"] - ) - client.files.grantPermissions( - systemId=system_id, path="/", username=user.username, permission="MODIFY" - ) + client.systems.grantUserPerms(systemId=system_id, userName=user.username, permissions=["READ"]) + client.files.grantPermissions(systemId=system_id, path="/", username=user.username, permission="MODIFY") class SystemAccessStepV3(AbstractStep): - def __init__(self, user): """ Call super class constructor diff --git a/server/portal/apps/onboarding/steps/test_steps.py b/server/portal/apps/onboarding/steps/test_steps.py index 4edfcb0383..ad46c32289 100644 --- a/server/portal/apps/onboarding/steps/test_steps.py +++ b/server/portal/apps/onboarding/steps/test_steps.py @@ -1,4 +1,3 @@ - from mock import MagicMock from portal.apps.onboarding.state import SetupState from portal.apps.onboarding.steps.abstract import AbstractStep @@ -123,18 +122,10 @@ def client_action(self, action, data, request): return if action == "staff_approve": - self.complete( - "Approved by {user}".format( - user=request.user.username - ) - ) + self.complete("Approved by {user}".format(user=request.user.username)) self.staff_approve_spy(action, data, request) elif action == "staff_deny": - self.fail( - "Denied by {user}".format( - user=request.user.username - ) - ) + self.fail("Denied by {user}".format(user=request.user.username)) self.staff_deny_spy(action, data, request) diff --git a/server/portal/apps/portal_messages/admin.py b/server/portal/apps/portal_messages/admin.py index e1c1bba919..945b443dc9 100644 --- a/server/portal/apps/portal_messages/admin.py +++ b/server/portal/apps/portal_messages/admin.py @@ -4,4 +4,4 @@ @admin.register(CustomMessageTemplate) class CustomMessageTemplateAdmin(admin.ModelAdmin): - fields = ('message_type', 'component', 'message', 'dismissible') + fields = ("message_type", "component", "message", "dismissible") diff --git a/server/portal/apps/portal_messages/apps.py b/server/portal/apps/portal_messages/apps.py index 4b25cbddd9..8f9b68a0d8 100644 --- a/server/portal/apps/portal_messages/apps.py +++ b/server/portal/apps/portal_messages/apps.py @@ -2,5 +2,5 @@ class PortalMessagesConfig(AppConfig): - name = 'portal.apps.portal_messages' - app_label = 'portal_messages' + name = "portal.apps.portal_messages" + app_label = "portal_messages" diff --git a/server/portal/apps/portal_messages/intro_unit_test.py b/server/portal/apps/portal_messages/intro_unit_test.py index 7ee90993b2..d58104aaa4 100644 --- a/server/portal/apps/portal_messages/intro_unit_test.py +++ b/server/portal/apps/portal_messages/intro_unit_test.py @@ -15,7 +15,7 @@ def intromessage_mock(authenticated_user): @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_intromessages_get(client, authenticated_user, intromessage_mock): - response = client.get('/api/portal_messages/intro/') + response = client.get("/api/portal_messages/intro/") data = response.json() assert response.status_code == 200 assert data["response"] == [{"component": "HISTORY", "unread": False}] @@ -29,7 +29,7 @@ def test_intromessages_get(client, authenticated_user, intromessage_mock): @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_intromessages_get_unauthenticated_user(client, regular_user): - response = client.get('/api/portal_messages/intro/') + response = client.get("/api/portal_messages/intro/") assert response.status_code == 302 @@ -39,19 +39,17 @@ def test_intromessages_get_unauthenticated_user(client, regular_user): @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_intromessages_put(client, authenticated_user): body = { - 'ACCOUNT': 'True', - 'ALLOCATIONS': 'True', - 'APPLICATIONS': 'True', - 'DASHBOARD': 'True', - 'DATA': 'True', - 'HISTORY': 'False', - 'TICKETS': 'True', - 'UI': 'True' + "ACCOUNT": "True", + "ALLOCATIONS": "True", + "APPLICATIONS": "True", + "DASHBOARD": "True", + "DATA": "True", + "HISTORY": "False", + "TICKETS": "True", + "UI": "True", } - response = client.put('/api/portal_messages/intro/', - content_type="application/json", - data=body) + response = client.put("/api/portal_messages/intro/", content_type="application/json", data=body) assert response.status_code == 200 # should be eight rows in the database for the user assert len(IntroMessages.objects.all()) == 8 @@ -67,7 +65,9 @@ def test_intromessages_put(client, authenticated_user): @pytest.fixture def custommessagetemplate_mock(): - template = CustomMessageTemplate.objects.create(component='HISTORY', message_type='warning', message='test message', dismissible=True) + template = CustomMessageTemplate.objects.create( + component="HISTORY", message_type="warning", message="test message", dismissible=True + ) yield template @@ -85,20 +85,22 @@ def custommessage_mock(authenticated_user, custommessagetemplate_mock): @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_custommessages_get(client, authenticated_user, custommessage_mock, custommessagetemplate_mock): - response = client.get('/api/portal_messages/custom/') + response = client.get("/api/portal_messages/custom/") data = response.json() assert response.status_code == 200 assert data["response"] == { - 'messages': [{ - "template": { - 'id': custommessagetemplate_mock.id, - 'component': 'HISTORY', - 'message_type': 'warning', - 'dismissible': True, - 'message': 'test message' - }, - "unread": True - }] + "messages": [ + { + "template": { + "id": custommessagetemplate_mock.id, + "component": "HISTORY", + "message_type": "warning", + "dismissible": True, + "message": "test message", + }, + "unread": True, + } + ] } @@ -110,7 +112,7 @@ def test_custommessages_get(client, authenticated_user, custommessage_mock, cust @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_custommessages_get_unauthenticated_user(client, regular_user): - response = client.get('/api/portal_messages/custom/') + response = client.get("/api/portal_messages/custom/") assert response.status_code == 302 @@ -122,18 +124,13 @@ def test_custommessages_put(client, authenticated_user, custommessage_mock, cust original_message = CustomMessages.objects.get(template__id=custommessagetemplate_mock.id) assert original_message.unread is True - body = { - 'templateId': custommessagetemplate_mock.id, - 'unread': False - } + body = {"templateId": custommessagetemplate_mock.id, "unread": False} - response = client.put('/api/portal_messages/custom/', - content_type="application/json", - data=body) + response = client.put("/api/portal_messages/custom/", content_type="application/json", data=body) assert response.status_code == 200 assert len(CustomMessages.objects.all()) == 1 - db_message = CustomMessages.objects.get(template__id=body['templateId']) + db_message = CustomMessages.objects.get(template__id=body["templateId"]) # Ensure that it updated the value correctly - assert db_message.unread == body['unread'] + assert db_message.unread == body["unread"] diff --git a/server/portal/apps/portal_messages/migrations/0001_initial.py b/server/portal/apps/portal_messages/migrations/0001_initial.py index 7a7854b038..0124af887f 100644 --- a/server/portal/apps/portal_messages/migrations/0001_initial.py +++ b/server/portal/apps/portal_messages/migrations/0001_initial.py @@ -7,7 +7,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -16,16 +15,21 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='IntroMessages', + name="IntroMessages", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('datetime', models.DateTimeField(blank=True, default=django.utils.timezone.now)), - ('component', models.CharField(default='', max_length=300)), - ('unread', models.BooleanField(default=True)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("datetime", models.DateTimeField(blank=True, default=django.utils.timezone.now)), + ("component", models.CharField(default="", max_length=300)), + ("unread", models.BooleanField(default=True)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), ], options={ - 'unique_together': {('user', 'component')}, + "unique_together": {("user", "component")}, }, ), ] diff --git a/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py b/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py index 4367f3f0ab..1d390170c7 100644 --- a/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py +++ b/server/portal/apps/portal_messages/migrations/0002_custommessages_custommessagetemplate.py @@ -6,33 +6,71 @@ class Migration(migrations.Migration): - dependencies = [ - ('portal_messages', '0001_initial'), + ("portal_messages", "0001_initial"), ] operations = [ migrations.CreateModel( - name='CustomMessageTemplate', + name="CustomMessageTemplate", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('component', models.CharField(choices=[('Dashboard', 'DASHBOARD'), ('Data Files', 'DATA'), ('Applications', 'APPLICATIONS'), ('Allocations', 'ALLOCATIONS'), ('History', 'HISTORY'), ('Account', 'ACCOUNT')], default='Dashboard', help_text='Component type', max_length=20)), - ('message_type', models.CharField(choices=[('info', 'Info'), ('success', 'Success'), ('warning', 'Warn'), ('error', 'Error')], default='info', help_text='Message type', max_length=20)), - ('dismissible', models.BooleanField(default=False)), - ('message', models.CharField(blank=True, max_length=200, default='', help_text='Message content (max 200 characters)')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "component", + models.CharField( + choices=[ + ("Dashboard", "DASHBOARD"), + ("Data Files", "DATA"), + ("Applications", "APPLICATIONS"), + ("Allocations", "ALLOCATIONS"), + ("History", "HISTORY"), + ("Account", "ACCOUNT"), + ], + default="Dashboard", + help_text="Component type", + max_length=20, + ), + ), + ( + "message_type", + models.CharField( + choices=[("info", "Info"), ("success", "Success"), ("warning", "Warn"), ("error", "Error")], + default="info", + help_text="Message type", + max_length=20, + ), + ), + ("dismissible", models.BooleanField(default=False)), + ( + "message", + models.CharField( + blank=True, max_length=200, default="", help_text="Message content (max 200 characters)" + ), + ), ], ), - migrations.CreateModel( - name='CustomMessages', + name="CustomMessages", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), - ('template', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='portal_messages.CustomMessageTemplate')), - ('unread', models.BooleanField(default=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), + ( + "template", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="portal_messages.CustomMessageTemplate", + ), + ), + ("unread", models.BooleanField(default=True)), ], options={ - 'unique_together': {('user', 'template')}, + "unique_together": {("user", "template")}, }, ), ] diff --git a/server/portal/apps/portal_messages/migrations/0003_auto_20220819_2213.py b/server/portal/apps/portal_messages/migrations/0003_auto_20220819_2213.py index 5ae1a64855..13173be3b5 100644 --- a/server/portal/apps/portal_messages/migrations/0003_auto_20220819_2213.py +++ b/server/portal/apps/portal_messages/migrations/0003_auto_20220819_2213.py @@ -4,21 +4,39 @@ class Migration(migrations.Migration): - dependencies = [ - ('portal_messages', '0002_custommessages_custommessagetemplate'), + ("portal_messages", "0002_custommessages_custommessagetemplate"), ] operations = [ migrations.AlterField( - model_name='custommessagetemplate', - name='component', - field=models.CharField(choices=[('DASHBOARD', 'Dashboard'), ('DATA', 'Data Files'), ('APPLICATIONS', 'Applications'), ('ALLOCATIONS', 'Allocations'), ('HISTORY', 'History'), ('UI', 'UI'), ('ACCOUNT', 'Account'), ( - 'UNPROTECTED', 'Unprotected'), ('ONBOARDING', 'Onboarding'), ('SUBMISSIONS', 'Submissions'), ('ONBOARDINGADMIN', 'Onboarding Admin'), ('SEARCH', 'Search')], default='Dashboard', help_text='Component type', max_length=20), + model_name="custommessagetemplate", + name="component", + field=models.CharField( + choices=[ + ("DASHBOARD", "Dashboard"), + ("DATA", "Data Files"), + ("APPLICATIONS", "Applications"), + ("ALLOCATIONS", "Allocations"), + ("HISTORY", "History"), + ("UI", "UI"), + ("ACCOUNT", "Account"), + ("UNPROTECTED", "Unprotected"), + ("ONBOARDING", "Onboarding"), + ("SUBMISSIONS", "Submissions"), + ("ONBOARDINGADMIN", "Onboarding Admin"), + ("SEARCH", "Search"), + ], + default="Dashboard", + help_text="Component type", + max_length=20, + ), ), migrations.AlterField( - model_name='custommessagetemplate', - name='message', - field=models.TextField(blank=True, default='', help_text='Message content (max 200 characters)', max_length=200), + model_name="custommessagetemplate", + name="message", + field=models.TextField( + blank=True, default="", help_text="Message content (max 200 characters)", max_length=200 + ), ), ] diff --git a/server/portal/apps/portal_messages/migrations/0004_migrate_intro_messages.py b/server/portal/apps/portal_messages/migrations/0004_migrate_intro_messages.py index e3bd259a8e..f9cdefa729 100644 --- a/server/portal/apps/portal_messages/migrations/0004_migrate_intro_messages.py +++ b/server/portal/apps/portal_messages/migrations/0004_migrate_intro_messages.py @@ -9,34 +9,26 @@ "ACCOUNT": "This page allows you to manage your account profile, change your password and view software licenses.", "TICKETS": "This page allows you to submit a help request via an RT Ticket.", "UI": "This hidden page allows developers to review UI components in isolation.", - "UNPROTECTED": "Note: this area is not authorized for protected data (i.e. PHI files). Please do not place any confidential/protected data in this space." + "UNPROTECTED": "Note: this area is not authorized for protected data (i.e. PHI files). Please do not place any confidential/protected data in this space.", } def migrate_intro_messages(apps, schema_editor): - IntroMessages = apps.get_model('portal_messages', 'IntroMessages') - CustomMessages = apps.get_model('portal_messages', 'CustomMessages') - CustomMessageTemplate = apps.get_model('portal_messages', 'CustomMessageTemplate') + IntroMessages = apps.get_model("portal_messages", "IntroMessages") + CustomMessages = apps.get_model("portal_messages", "CustomMessages") + CustomMessageTemplate = apps.get_model("portal_messages", "CustomMessageTemplate") for component, message in intro_messages.items(): template = CustomMessageTemplate.objects.create( - component=component, - message_type='info', - dismissible=True, - message=message + component=component, message_type="info", dismissible=True, message=message ) for intro_message in IntroMessages.objects.filter(component=component): - CustomMessages.objects.create( - user=intro_message.user, - template=template, - unread=intro_message.unread - ) + CustomMessages.objects.create(user=intro_message.user, template=template, unread=intro_message.unread) class Migration(migrations.Migration): - dependencies = [ - ('portal_messages', '0003_auto_20220819_2213'), + ("portal_messages", "0003_auto_20220819_2213"), ] operations = [ diff --git a/server/portal/apps/portal_messages/migrations/0005_migrate_longer_messages.py b/server/portal/apps/portal_messages/migrations/0005_migrate_longer_messages.py index 4a83c829cb..f3e35966ea 100644 --- a/server/portal/apps/portal_messages/migrations/0005_migrate_longer_messages.py +++ b/server/portal/apps/portal_messages/migrations/0005_migrate_longer_messages.py @@ -4,15 +4,16 @@ class Migration(migrations.Migration): - dependencies = [ - ('portal_messages', '0004_migrate_intro_messages'), + ("portal_messages", "0004_migrate_intro_messages"), ] operations = [ migrations.AlterField( - model_name='custommessagetemplate', - name='message', - field=models.TextField(blank=True, default='', help_text='Message content (max 1000 characters)', max_length=1000), + model_name="custommessagetemplate", + name="message", + field=models.TextField( + blank=True, default="", help_text="Message content (max 1000 characters)", max_length=1000 + ), ), ] diff --git a/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py b/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py index f19be3ee3e..2e4e038e98 100644 --- a/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py +++ b/server/portal/apps/portal_messages/migrations/0006_migrate_intro_messages_cpu.py @@ -2,9 +2,7 @@ from django.db import migrations -CPU_DASHBOARD_MESSAGE = ( - "This page allows you to monitor your job status and get help with tickets. " -) +CPU_DASHBOARD_MESSAGE = "This page allows you to monitor your job status and get help with tickets. " CPU_DATAFILES_MESSAGE = ( "This page allows you to upload and manage your files. Management and actions " @@ -28,7 +26,6 @@ def migrate_intro_messages(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("portal_messages", "0005_migrate_longer_messages"), ] diff --git a/server/portal/apps/portal_messages/models.py b/server/portal/apps/portal_messages/models.py index bc5ae7bb39..b44496a904 100644 --- a/server/portal/apps/portal_messages/models.py +++ b/server/portal/apps/portal_messages/models.py @@ -12,22 +12,21 @@ class IntroMessages(models.Model): Used for storing the visited status of each of the Intro (formerly Welcome) messages. """ - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name="+", - on_delete=models.CASCADE - ) + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="+", on_delete=models.CASCADE) datetime = models.DateTimeField(default=timezone.now, blank=True) # Each variable represents that intro message status # True means message has not been dismissed by user - component = models.CharField(max_length=300, default='') + component = models.CharField(max_length=300, default="") unread = models.BooleanField(default=True) # Make each type of IntroMessage unique class Meta: - unique_together = ('user', 'component',) + unique_together = ( + "user", + "component", + ) class CustomMessageTemplate(models.Model): @@ -36,33 +35,46 @@ class CustomMessageTemplate(models.Model): Used for storing admin-controlled messages for specific components that utilize CustomMessages. """ - MESSAGE_TYPES = [('info', 'Info'), ('success', 'Success'), - ('warning', 'Warn'), ('error', 'Error')] - - COMPONENTS = [('DASHBOARD', 'Dashboard'), ('DATA', 'Data Files'), - ('APPLICATIONS', 'Applications'), ('ALLOCATIONS', 'Allocations'), - ('HISTORY', 'History'), ('UI', 'UI'), ('ACCOUNT', 'Account'), - ('UNPROTECTED', 'Unprotected'), ('ONBOARDING', 'Onboarding'), - ('SUBMISSIONS', 'Submissions'), ('ONBOARDINGADMIN', 'Onboarding Admin'), - ('SEARCH', 'Search')] - - component = models.CharField(help_text='Component type', max_length=20, choices=COMPONENTS, default='Dashboard') - message_type = models.CharField(help_text='Message type', max_length=20, choices=MESSAGE_TYPES, default='info') + MESSAGE_TYPES = [("info", "Info"), ("success", "Success"), ("warning", "Warn"), ("error", "Error")] + + COMPONENTS = [ + ("DASHBOARD", "Dashboard"), + ("DATA", "Data Files"), + ("APPLICATIONS", "Applications"), + ("ALLOCATIONS", "Allocations"), + ("HISTORY", "History"), + ("UI", "UI"), + ("ACCOUNT", "Account"), + ("UNPROTECTED", "Unprotected"), + ("ONBOARDING", "Onboarding"), + ("SUBMISSIONS", "Submissions"), + ("ONBOARDINGADMIN", "Onboarding Admin"), + ("SEARCH", "Search"), + ] + + component = models.CharField(help_text="Component type", max_length=20, choices=COMPONENTS, default="Dashboard") + message_type = models.CharField(help_text="Message type", max_length=20, choices=MESSAGE_TYPES, default="info") dismissible = models.BooleanField(default=False) - message = models.TextField(help_text='Message content (max 1000 characters)', max_length=1000, default='', blank=True) + message = models.TextField( + help_text="Message content (max 1000 characters)", max_length=1000, default="", blank=True + ) def to_dict(self): return { - 'id': self.id, - 'component': self.component, - 'message_type': self.message_type, - 'dismissible': self.dismissible, - 'message': self.message + "id": self.id, + "component": self.component, + "message_type": self.message_type, + "dismissible": self.dismissible, + "message": self.message, } def __str__(self): - return "%s | %s | %s | %s" % (self.message_type, self.component, - ('dismissible' if self.dismissible else 'not dismissible'), self.message[0:20]) + return "%s | %s | %s | %s" % ( + self.message_type, + self.component, + ("dismissible" if self.dismissible else "not dismissible"), + self.message[0:20], + ) class CustomMessages(models.Model): @@ -70,11 +82,8 @@ class CustomMessages(models.Model): Used for storing messages instances that were created by admin and handles status of each message. """ - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name="+", - on_delete=models.CASCADE - ) + + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="+", on_delete=models.CASCADE) template = models.ForeignKey( CustomMessageTemplate, @@ -85,4 +94,7 @@ class CustomMessages(models.Model): unread = models.BooleanField(default=True) class Meta: - unique_together = ('user', 'template',) + unique_together = ( + "user", + "template", + ) diff --git a/server/portal/apps/portal_messages/urls.py b/server/portal/apps/portal_messages/urls.py index f0f446a344..a9b21bc9e7 100644 --- a/server/portal/apps/portal_messages/urls.py +++ b/server/portal/apps/portal_messages/urls.py @@ -1,11 +1,8 @@ -"""Message URLs -""" +"""Message URLs""" + from django.urls import path from portal.apps.portal_messages import views -app_name = 'message' -urlpatterns = [ - path('intro/', views.IntroMessagesView.as_view()), - path('custom/', views.CustomMessagesView.as_view()) -] +app_name = "message" +urlpatterns = [path("intro/", views.IntroMessagesView.as_view()), path("custom/", views.CustomMessagesView.as_view())] diff --git a/server/portal/apps/portal_messages/views.py b/server/portal/apps/portal_messages/views.py index 7d8642c0cb..16f18ad7b2 100644 --- a/server/portal/apps/portal_messages/views.py +++ b/server/portal/apps/portal_messages/views.py @@ -20,41 +20,45 @@ def get_or_create_custom_messages(user, template): message, _ = CustomMessages.objects.get_or_create(user=user, template=template) return { - 'template': message.template.to_dict(), - 'unread': message.unread, + "template": message.template.to_dict(), + "unread": message.unread, } -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class IntroMessagesView(BaseApiView): def get(self, request, *args, **kwargs): - messages_array = IntroMessages.objects.filter(user=request.user).values('component', 'unread') - return JsonResponse({'response': list(messages_array)}) + messages_array = IntroMessages.objects.filter(user=request.user).values("component", "unread") + return JsonResponse({"response": list(messages_array)}) def put(self, request, *args): body = json.loads(request.body) for component_name, component_value in body.items(): - IntroMessages.objects.update_or_create(user=request.user, component=component_name, defaults={'unread': component_value}) - return JsonResponse({'status': 'OK'}) + IntroMessages.objects.update_or_create( + user=request.user, component=component_name, defaults={"unread": component_value} + ) + return JsonResponse({"status": "OK"}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class CustomMessagesView(BaseApiView): def get(self, request, *args, **kwargs): templates = CustomMessageTemplate.objects.all() messages = [get_or_create_custom_messages(request.user, template) for template in templates] - return JsonResponse({ - 'response': { - 'messages': list(messages), + return JsonResponse( + { + "response": { + "messages": list(messages), + } } - }) + ) def put(self, request, *args): body = json.loads(request.body) - template_id = body['templateId'] - unread = body['unread'] + template_id = body["templateId"] + unread = body["unread"] message = CustomMessages.objects.get(user=request.user, template__id=template_id) message.unread = unread message.save() - return JsonResponse({'status': 'OK'}) + return JsonResponse({"status": "OK"}) diff --git a/server/portal/apps/projects/apps.py b/server/portal/apps/projects/apps.py index e6c4f42b90..7da37bf480 100644 --- a/server/portal/apps/projects/apps.py +++ b/server/portal/apps/projects/apps.py @@ -2,4 +2,4 @@ class ProjectsConfig(AppConfig): - name = 'portal.apps.projects' + name = "portal.apps.projects" diff --git a/server/portal/apps/projects/conftest.py b/server/portal/apps/projects/conftest.py index b1b243258f..0f12423148 100644 --- a/server/portal/apps/projects/conftest.py +++ b/server/portal/apps/projects/conftest.py @@ -5,25 +5,25 @@ @pytest.mark.django_db @pytest.fixture() def service_account(mocker): - yield mocker.patch('portal.apps.projects.models.base.service_account') + yield mocker.patch("portal.apps.projects.models.base.service_account") @pytest.fixture() @pytest.mark.django_db def project_model(mocker): - mocker.patch('portal.apps.projects.models.base.Project._create_dir') - mocker.patch('portal.apps.projects.models.base.Project._delete_dir') + mocker.patch("portal.apps.projects.models.base.Project._create_dir") + mocker.patch("portal.apps.projects.models.base.Project._delete_dir") yield Project @pytest.fixture() def mock_projects(project_model, service_account, mock_storage_system, mock_tapis_client, regular_user): - prj1 = Project.create(mock_tapis_client, 'First Project', 'test.project-123', regular_user) - prj1.storage.name = 'test.project-123' - prj1.storage.id = 'test.project-123' - prj2 = Project.create(mock_tapis_client, 'Second Project', 'test.project-124', regular_user) - prj2.storage.name = 'test.project-124' - prj2.storage.id = 'test.project-124' + prj1 = Project.create(mock_tapis_client, "First Project", "test.project-123", regular_user) + prj1.storage.name = "test.project-123" + prj1.storage.id = "test.project-123" + prj2 = Project.create(mock_tapis_client, "Second Project", "test.project-124", regular_user) + prj2.storage.name = "test.project-124" + prj2.storage.id = "test.project-124" yield [prj1, prj2] @@ -34,12 +34,12 @@ def mock_projects_storage_systems(mock_projects): @pytest.fixture() def portal_project(mocker): - mocker.patch('portal.apps.projects.models.base.Project._create_dir') - mocker.patch('portal.apps.projects.models.base.Project._delete_dir') - mocker.patch('portal.apps.projects.models.base.Project._create_storage') + mocker.patch("portal.apps.projects.models.base.Project._create_dir") + mocker.patch("portal.apps.projects.models.base.Project._delete_dir") + mocker.patch("portal.apps.projects.models.base.Project._create_storage") yield Project @pytest.fixture() def mock_project_save_signal(mocker): - yield mocker.patch('portal.apps.signals.receivers.index_project') + yield mocker.patch("portal.apps.signals.receivers.index_project") diff --git a/server/portal/apps/projects/exceptions.py b/server/portal/apps/projects/exceptions.py index 1f229bfc7a..c8f9349d92 100644 --- a/server/portal/apps/projects/exceptions.py +++ b/server/portal/apps/projects/exceptions.py @@ -3,6 +3,7 @@ .. module:: portal.apps.projects.execeptions :synopsis: Exception classes for projects. """ + import logging from portal.exceptions.api import ApiException @@ -17,13 +18,7 @@ class NotAuthorizedError(ApiException): # pylint:disable=too-many-ancestors something it's not supposed to. """ - def __init__( - self, - message=None, - status=None, - extra=None, - **kwargs - ): + def __init__(self, message=None, status=None, extra=None, **kwargs): """Exception based on :class:`~requests.exceptions.RequestException`. :param str message: Exception message. @@ -32,9 +27,4 @@ def __init__( """ msg = "User is not Authorized." sts = 403 - super(NotAuthorizedError, self).__init__( - message=message or msg, - status=status or sts, - extra=extra, - **kwargs - ) + super(NotAuthorizedError, self).__init__(message=message or msg, status=status or sts, extra=extra, **kwargs) diff --git a/server/portal/apps/projects/management/commands/migrate-projects.py b/server/portal/apps/projects/management/commands/migrate-projects.py index 43af459c93..54135d01f2 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects.py +++ b/server/portal/apps/projects/management/commands/migrate-projects.py @@ -16,9 +16,9 @@ class Command(BaseCommand): """Command class.""" help = ( - 'Reconcile projects from CEPv1 migration that do not have ' - 'a PI assigned. This will attempt to assign the creator of ' - 'these projects as PI, as well as index all projects.' + "Reconcile projects from CEPv1 migration that do not have " + "a PI assigned. This will attempt to assign the creator of " + "these projects as PI, as well as index all projects." ) def handle(self, *args, **options): @@ -33,8 +33,10 @@ def handle(self, *args, **options): roles = storage.roles.to_dict().items() admins = list( filter( - lambda role_tuple: role_tuple[0] != 'wma_prtl' and (role_tuple[1] == 'ADMIN' or role_tuple[1] == 'OWNER'), - roles + lambda role_tuple: ( + role_tuple[0] != "wma_prtl" and (role_tuple[1] == "ADMIN" or role_tuple[1] == "OWNER") + ), + roles, ) ) if len(admins) != 1: @@ -42,10 +44,9 @@ def handle(self, *args, **options): # Get first role tuple, first item in tuple which is username admin = get_user_model().objects.get(username=admins[0][0]) project.add_pi(admin) - logger.info("Set {admin} as PI on {project_id}".format( - admin=admin.username, - project_id=meta.project_id - )) + logger.info( + "Set {admin} as PI on {project_id}".format(admin=admin.username, project_id=meta.project_id) + ) except Exception as e: logger.error("Could not migrate {project_id}".format(project_id=meta.project_id)) diff --git a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py index ce991d8ff6..097dd8bfd8 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py +++ b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py @@ -29,32 +29,31 @@ def ownerless_project(django_db_reset_sequences): @pytest.fixture def mock_project_metadata(mocker, ownerless_project): - yield mocker.patch.object(Project, '_get_metadata', return_value=ownerless_project) + yield mocker.patch.object(Project, "_get_metadata", return_value=ownerless_project) @pytest.fixture def mock_project_storage(mocker): - yield mocker.patch.object(Project, '_get_storage') + yield mocker.patch.object(Project, "_get_storage") @pytest.fixture def mock_service_account(mocker): - mock = mocker.patch('portal.apps.projects.management.commands.migrate-projects.service_account') + mock = mocker.patch("portal.apps.projects.management.commands.migrate-projects.service_account") yield mock.return_value @pytest.fixture def mock_index_project(mocker): - mock = mocker.patch('portal.apps.search.tasks.index_project') + mock = mocker.patch("portal.apps.search.tasks.index_project") yield mock @pytest.mark.skip(reason="role management different in v3") -def test_migrate_projects(regular_user, mock_project_metadata, mock_project_storage, mock_index_project, service_account, mock_service_account): - mock_project_storage.return_value.roles.to_dict.return_value = { - 'wma_prtl': 'OWNER', - 'username': 'ADMIN' - } +def test_migrate_projects( + regular_user, mock_project_metadata, mock_project_storage, mock_index_project, service_account, mock_service_account +): + mock_project_storage.return_value.roles.to_dict.return_value = {"wma_prtl": "OWNER", "username": "ADMIN"} management.call_command("migrate-projects") assert LegacyProjectMetadata.objects.all()[0].pi.username == regular_user.username assert mock_index_project.apply_async.called @@ -63,9 +62,9 @@ def test_migrate_projects(regular_user, mock_project_metadata, mock_project_stor @pytest.mark.skip(reason="role management different in v3") def test_migrate_projects_wrong_admins(regular_user, mock_project_metadata, mock_project_storage, mock_service_account): mock_project_storage.return_value.roles.to_dict.return_value = { - 'wma_prtl': 'OWNER', - 'username': 'ADMIN', - 'username2': 'ADMIN' + "wma_prtl": "OWNER", + "username": "ADMIN", + "username2": "ADMIN", } management.call_command("migrate-projects") assert LegacyProjectMetadata.objects.all()[0].pi is None diff --git a/server/portal/apps/projects/management/commands/projects_id.py b/server/portal/apps/projects/management/commands/projects_id.py index 9cab3f5fdf..ae42a8f55f 100644 --- a/server/portal/apps/projects/management/commands/projects_id.py +++ b/server/portal/apps/projects/management/commands/projects_id.py @@ -31,63 +31,58 @@ class Command(BaseCommand): >>> ./manage.py projects_id --update-using-max-value-found --max-project-id 1000000 """ + help = ( - 'Manage projects latest project id. By default this command will print ' - 'the current latest project id, the last project id used in ' - 'storage systems, and the last project id used in folders created.' + "Manage projects latest project id. By default this command will print " + "the current latest project id, the last project id used in " + "storage systems, and the last project id used in folders created." ) def add_arguments(self, parser): """Add arguments.""" update_group = parser.add_mutually_exclusive_group() + update_group.add_argument("--update", action="store", type=int, help="Update project id DB value.") update_group.add_argument( - '--update', - action='store', - type=int, - help='Update project id DB value.' - ) - update_group.add_argument( - '--update-using-max-value-found', - action='store_true', - help='Update project id DB value using value derived from latest storage system project id or latest ' - 'directory project id (whichever is higher).' + "--update-using-max-value-found", + action="store_true", + help="Update project id DB value using value derived from latest storage system project id or latest " + "directory project id (whichever is higher).", ) parser.add_argument( - '--max-project-id', - action='store', - type=int, - help='Ignore project ids larger than a certain value' + "--max-project-id", action="store", type=int, help="Ignore project ids larger than a certain value" ) def handle(self, *args, **options): """Handle command.""" max_project_id = options["max_project_id"] if max_project_id: - self.stdout.write('NOTE(!!!!): Ignoring project ids >= {} when ' - 'processing/updating the storage systems and directories'.format(max_project_id)) + self.stdout.write( + "NOTE(!!!!): Ignoring project ids >= {} when " + "processing/updating the storage systems and directories".format(max_project_id) + ) latest_storage_system_id = get_latest_project_storage(max_project_id=max_project_id) latest_project_id = get_latest_project_directory(max_project_id=max_project_id) if latest_storage_system_id == -1: - self.stdout.write('There are no project storage systems.') + self.stdout.write("There are no project storage systems.") if latest_project_id == -1: - self.stdout.write('There are no project directories.') + self.stdout.write("There are no project directories.") - self.stdout.write('Latest storage system project id: {}'.format(latest_storage_system_id)) - self.stdout.write('Latest directory project id: {}'.format(latest_project_id)) + self.stdout.write("Latest storage system project id: {}".format(latest_storage_system_id)) + self.stdout.write("Latest directory project id: {}".format(latest_project_id)) try: with transaction.atomic(): - model_project_id = ProjectId.objects.select_for_update().latest('last_updated').value - self.stdout.write('Latest project id in ProjectId model: {}'.format(model_project_id)) + model_project_id = ProjectId.objects.select_for_update().latest("last_updated").value + self.stdout.write("Latest project id in ProjectId model: {}".format(model_project_id)) except ObjectDoesNotExist: - self.stdout.write('Latest project id in ProjectId model: None') + self.stdout.write("Latest project id in ProjectId model: None") - if options.get('update'): - self.stdout.write('Updating to user provided value of: {}'.format(options.get('update'))) - ProjectId.update(options.get('update')) + if options.get("update"): + self.stdout.write("Updating to user provided value of: {}".format(options.get("update"))) + ProjectId.update(options.get("update")) elif options["update_using_max_value_found"]: max_value_found = max(latest_storage_system_id, latest_project_id, 0) - self.stdout.write('Updating to value latest storage system id: {}'.format(max_value_found)) + self.stdout.write("Updating to value latest storage system id: {}".format(max_value_found)) ProjectId.update(max_value_found) diff --git a/server/portal/apps/projects/management/commands/projects_id_unit_test.py b/server/portal/apps/projects/management/commands/projects_id_unit_test.py index 96d5a3b75a..44e9d0da7b 100644 --- a/server/portal/apps/projects/management/commands/projects_id_unit_test.py +++ b/server/portal/apps/projects/management/commands/projects_id_unit_test.py @@ -9,27 +9,27 @@ @pytest.fixture def mock_project_listing(mocker): - project_mock = mocker.patch('portal.apps.projects.models.utils.Project') + project_mock = mocker.patch("portal.apps.projects.models.utils.Project") project_mock.listing.return_value = [] yield project_mock @pytest.fixture def mock_project_listing_with_projects(mocker, mock_projects): - project_mock = mocker.patch('portal.apps.projects.models.utils.Project') + project_mock = mocker.patch("portal.apps.projects.models.utils.Project") project_mock.listing.return_value = mock_projects @pytest.fixture def mock_iterate_listings(mocker): - iterate_listing_mock = mocker.patch('portal.apps.projects.models.utils.iterate_listing') + iterate_listing_mock = mocker.patch("portal.apps.projects.models.utils.iterate_listing") iterate_listing_mock.return_value = [] yield iterate_listing_mock @pytest.fixture() def mock_service_account(mocker): - yield mocker.patch('portal.apps.projects.models.utils.service_account', autospec=True) + yield mocker.patch("portal.apps.projects.models.utils.service_account", autospec=True) @pytest.mark.skip(reason="TODOv3: update test after projects implemented") @@ -55,7 +55,9 @@ def test_default_command_with_no_projects(mock_iterate_listings, mock_project_li @pytest.mark.skip(reason="TODOv3: update test after projects implemented") -def test_default_command_with_two_projects(mock_iterate_listings, mock_project_listing_with_projects, mock_service_account): +def test_default_command_with_two_projects( + mock_iterate_listings, mock_project_listing_with_projects, mock_service_account +): out = StringIO() call_command("projects_id", stdout=out) output = out.getvalue() @@ -81,7 +83,9 @@ def test_update_using_storage_system_id(mock_iterate_listings, mock_project_list @pytest.mark.skip(reason="TODOv3: update test after projects implemented") -def test_update_using_storage_system_id_with_two_projects(mock_iterate_listings, mock_project_listing_with_projects, mock_service_account): +def test_update_using_storage_system_id_with_two_projects( + mock_iterate_listings, mock_project_listing_with_projects, mock_service_account +): out = StringIO() call_command("projects_id", "--update-using-max-value-found", stdout=out) output = out.getvalue() diff --git a/server/portal/apps/projects/managers/base.py b/server/portal/apps/projects/managers/base.py index 6a497a3c30..092737959c 100644 --- a/server/portal/apps/projects/managers/base.py +++ b/server/portal/apps/projects/managers/base.py @@ -3,10 +3,12 @@ .. :module:: portal.apps.projects.managers.base :synopsis: Manager for projects """ + import logging from django.conf import settings from django.contrib.auth import get_user_model from portal.libs.agave.utils import service_account + # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem from portal.libs.elasticsearch.docs.base import IndexedProject @@ -18,7 +20,7 @@ # pylint: disable=invalid-name logger = logging.getLogger(__name__) -METRICS = logging.getLogger('{}.{}'.format('metrics', __name__)) +METRICS = logging.getLogger("{}.{}".format("metrics", __name__)) # pylint: enable=invalid-name @@ -27,12 +29,7 @@ class ProjectsManager(object): meta_serializer_cls = MetadataJSONSerializer - def __init__( - self, - user, - *args, - **kwagrs - ): # pylint: disable=unused-argument + def __init__(self, user, *args, **kwagrs): # pylint: disable=unused-argument """Projects Manager init. :param user: Django user instance. @@ -45,23 +42,22 @@ def _add_acls(self, username, project_id, project_root): :param str username: Username. :param str project_id: Project Id. """ - logger.info('Adding ACLs for %s in project %s', username, project_id) + logger.info("Adding ACLs for %s in project %s", username, project_id) client = service_account() - job = client.jobs.submit(body={ - "name": "{username}-{project_id}-acls".format( - username=username, - project_id=project_id - ), - "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, - "archive": False, - "parameters": { - "projectId": project_id, - "username": username, - "action": "add", - "root_dir": project_root, + job = client.jobs.submit( + body={ + "name": "{username}-{project_id}-acls".format(username=username, project_id=project_id), + "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, + "archive": False, + "parameters": { + "projectId": project_id, + "username": username, + "action": "add", + "root_dir": project_root, + }, } - }) - logger.info('Add ACLs job id: %s', job.id) + ) + logger.info("Add ACLs job id: %s", job.id) def _remove_acls(self, username, project_id, project_root): """Run an agave job to set ACLs. @@ -69,23 +65,22 @@ def _remove_acls(self, username, project_id, project_root): :param str username: Username. :param str project_id: Project Id. """ - logger.info('Removing ACLs for %s in project %s', username, project_id) + logger.info("Removing ACLs for %s in project %s", username, project_id) client = service_account() - job = client.jobs.submit(body={ - "name": "{username}-{project_id}-acls".format( - username=username, - project_id=project_id - ), - "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, - "archive": False, - "parameters": { - "projectId": project_id, - "username": username, - "action": "remove", - "root_dir": project_root, + job = client.jobs.submit( + body={ + "name": "{username}-{project_id}-acls".format(username=username, project_id=project_id), + "appId": settings.PORTAL_PROJECTS_PEMS_APP_ID, + "archive": False, + "parameters": { + "projectId": project_id, + "username": username, + "action": "remove", + "root_dir": project_root, + }, } - }) - logger.info('Remove ACLs job id: %s', job.id) + ) + logger.info("Remove ACLs job id: %s", job.id) # TOODv3: deprecate with projects # def get_by_system_id(self, system_id): @@ -109,10 +104,7 @@ def get_by_project_id(self, project_id): :param str project_id: Project Id. """ - prj = Project( - self.user.tapis_oauth.client, - project_id - ) + prj = Project(self.user.tapis_oauth.client, project_id) if not prj.storage.uuid: raise Exception("No project.") return prj @@ -159,81 +151,56 @@ def create(self, title): ProjectId.objects.create(value=max_value_found) prjId = ProjectId.next_id() - project_id = '{prefix}-{prjId}'.format( - prefix=settings.PORTAL_PROJECTS_ID_PREFIX, - prjId=prjId - ) + project_id = "{prefix}-{prjId}".format(prefix=settings.PORTAL_PROJECTS_ID_PREFIX, prjId=prjId) try: - prj = Project.create( - self.user.tapis_oauth.client, - title, - project_id, - self.user - ) + prj = Project.create(self.user.tapis_oauth.client, title, project_id, self.user) except ValueError: # Tapis StorageSystem or ProjectMetadata with this ProjectID already exists, # try to update to latest project value and recreate - logger.info('Project with id: {} already exists'.format(project_id)) + logger.info("Project with id: {} already exists".format(project_id)) latest_storage_system_id = get_latest_project_storage() latest_project_id = get_latest_project_directory() max_value_found = max(latest_storage_system_id, latest_project_id, 0) - logger.info('Updating ProjectId to latest project dir or storage system id: {}'.format(max_value_found)) + logger.info("Updating ProjectId to latest project dir or storage system id: {}".format(max_value_found)) ProjectId.update(max_value_found) - project_id = '{prefix}-{prj_id}'.format( - prefix=settings.PORTAL_PROJECTS_ID_PREFIX, - prj_id=ProjectId.next_id() - ) - prj = Project.create( - self.user.tapis_oauth.client, - title, - project_id, - self.user + project_id = "{prefix}-{prj_id}".format( + prefix=settings.PORTAL_PROJECTS_ID_PREFIX, prj_id=ProjectId.next_id() ) + prj = Project.create(self.user.tapis_oauth.client, title, project_id, self.user) - prj.storage.update_role( - self.user.username, - 'ADMIN' - ) - METRICS.info('user:{} created project: id={}, title:{}'.format(self.user.username, project_id, title)) + prj.storage.update_role(self.user.username, "ADMIN") + METRICS.info("user:{} created project: id={}, title:{}".format(self.user.username, project_id, title)) return prj def list(self, offset=0, limit=100): """List projects.""" - return [prj.storage for prj in Project.listing( - self.user.tapis_oauth.client, - offset=offset, - limit=limit - )] + return [prj.storage for prj in Project.listing(self.user.tapis_oauth.client, offset=offset, limit=limit)] def search(self, query_string, offset=0, limit=100): """Search projects by query string""" search_result = IndexedProject.search() - search_result = search_result.query("query_string", - query=query_string, - minimum_should_match="80%") + search_result = search_result.query("query_string", query=query_string, minimum_should_match="80%") search_result = search_result.execute() result_ids = list(map(lambda hit: hit.projectId, search_result)) project_list = self.list() - filtered_list = filter(lambda prj: prj.name in result_ids, - project_list) + filtered_list = filter(lambda prj: prj.name in result_ids, project_list) return list(filtered_list) def apply_permissions(self, project, username, acl): - """Index project and update acls - """ + """Index project and update acls""" project_id = project.project_id project_root = project.storage.storage.root_dir - if acl == 'add': + if acl == "add": self._add_acls(username, project_id, project_root) - elif acl == 'remove': + elif acl == "remove": self._remove_acls(username, project_id, project_root) def transfer_ownership(self, project_id, old_owner, new_owner): @@ -242,10 +209,7 @@ def transfer_ownership(self, project_id, old_owner, new_owner): """ old_pi = get_user_model().objects.get(username=old_owner) new_pi = get_user_model().objects.get(username=new_owner) - prj = Project( - service_account(), - project_id - ) + prj = Project(service_account(), project_id) prj.transfer_pi(old_pi, new_pi) return prj @@ -261,15 +225,15 @@ def add_member(self, project_id, member_type, username): """ user = get_user_model().objects.get(username=username) prj = self.get_project(project_id) - if member_type == 'team_member': + if member_type == "team_member": prj.add_member(user) - elif member_type == 'co_pi': + elif member_type == "co_pi": prj.add_co_pi(user) - elif member_type == 'pi': + elif member_type == "pi": prj.add_pi(user) else: - raise Exception('Invalid member type.') - self.apply_permissions(prj, username, 'add') + raise Exception("Invalid member type.") + self.apply_permissions(prj, username, "add") return prj def remove_member(self, project_id, member_type, username): @@ -281,15 +245,15 @@ def remove_member(self, project_id, member_type, username): """ user = get_user_model().objects.get(username=username) prj = self.get_project(project_id) - if member_type == 'team_member': + if member_type == "team_member": prj.remove_member(user) - elif member_type == 'co_pi': + elif member_type == "co_pi": prj.remove_co_pi(user) - elif member_type == 'pi': + elif member_type == "pi": prj.remove_pi(user) else: - raise Exception('Invalid member type.') - self.apply_permissions(prj, username, 'remove') + raise Exception("Invalid member type.") + self.apply_permissions(prj, username, "remove") return prj def change_system_role(self, project_id, username, new_role): @@ -311,7 +275,7 @@ def _update_meta(self, project, **data): # pylint: disable=no-self-use :param dict data: Data to update. """ meta = project.metadata - logger.debug('data: %s', data) + logger.debug("data: %s", data) for field in data: try: # We have to check if the attribute is in the class @@ -336,7 +300,7 @@ def _update_storage(self, project, **data): # pylint: disable=no-self-use :param project: project instance. :param dict data: Data to update. """ - title = data.get('title') + title = data.get("title") if title is not None: project.storage.description = title @@ -353,8 +317,8 @@ def update_prj(self, project_id=None, system_id=None, **data): :param str project_id: Project Id. :param dict data: Dictionary where keys are project's field names. """ - data.pop('id', None) - data.pop('project_id', None) + data.pop("id", None) + data.pop("project_id", None) # When accessing a project's metadata it's a good practice to # instantiate a Project class instead of using ProjectMetadata # directly. This way we use Agave's permission model indirectly. diff --git a/server/portal/apps/projects/managers/unit_test.py b/server/portal/apps/projects/managers/unit_test.py index 22ca3e32e3..f4abeb3e8c 100644 --- a/server/portal/apps/projects/managers/unit_test.py +++ b/server/portal/apps/projects/managers/unit_test.py @@ -3,6 +3,7 @@ .. :module:: portal.apps.projects.unit_test :synopsis: Projects app unit tests. """ + import logging import os from django.conf import settings @@ -15,22 +16,22 @@ @pytest.fixture() def agave_client(mocker): - yield mocker.patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) + yield mocker.patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) @pytest.fixture() def mock_index(mocker): - yield mocker.patch('portal.apps.projects.managers.base.IndexedProject') + yield mocker.patch("portal.apps.projects.managers.base.IndexedProject") @pytest.fixture() def service_account(mocker): - yield mocker.patch('portal.apps.projects.managers.base.service_account') + yield mocker.patch("portal.apps.projects.managers.base.service_account") @pytest.fixture() def project_manager(mocker, authenticated_user): - mocker.patch('portal.apps.projects.managers.base.ProjectsManager.get_project') + mocker.patch("portal.apps.projects.managers.base.ProjectsManager.get_project") project = ProjectsManager(authenticated_user) project.get_project().project_id = "PRJ-123" project.get_project().storage.storage.root_dir = os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123") @@ -39,20 +40,18 @@ def project_manager(mocker, authenticated_user): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_search(mocker, authenticated_user, project_manager, mock_index): - mock_listing = mocker.patch('portal.apps.projects.managers.base.ProjectsManager.list') + mock_listing = mocker.patch("portal.apps.projects.managers.base.ProjectsManager.list") mock_listing.return_value = [] mock_index.search().query().execute().return_value = [] - project_manager.search('testquery') + project_manager.search("testquery") assert mock_listing.call_count == 1 - mock_index.search().query.assert_called_with('query_string', - query='testquery', - minimum_should_match="80%") + mock_index.search().query.assert_called_with("query_string", query="testquery", minimum_should_match="80%") @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_add_member_pi(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.add_member('PRJ-123', 'pi', 'username') + project_manager.add_member("PRJ-123", "pi", "username") project_manager.get_project().add_member.assert_not_called() project_manager.get_project().add_co_pi.assert_not_called() project_manager.get_project().add_pi.assert_called_with(authenticated_user) @@ -67,7 +66,7 @@ def test_add_member_pi(authenticated_user, project_manager, service_account): "username": "username", "action": "add", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @@ -75,7 +74,7 @@ def test_add_member_pi(authenticated_user, project_manager, service_account): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_add_member_co_pi(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.add_member('PRJ-123', 'co_pi', 'username') + project_manager.add_member("PRJ-123", "co_pi", "username") project_manager.get_project().add_member.assert_not_called() project_manager.get_project().add_pi.assert_not_called() project_manager.get_project().add_co_pi.assert_called_with(authenticated_user) @@ -90,7 +89,7 @@ def test_add_member_co_pi(authenticated_user, project_manager, service_account): "username": "username", "action": "add", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @@ -98,7 +97,7 @@ def test_add_member_co_pi(authenticated_user, project_manager, service_account): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_add_member(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.add_member('PRJ-123', 'team_member', 'username') + project_manager.add_member("PRJ-123", "team_member", "username") project_manager.get_project().add_co_pi.assert_not_called() project_manager.get_project().add_pi.assert_not_called() @@ -114,7 +113,7 @@ def test_add_member(authenticated_user, project_manager, service_account): "username": "username", "action": "add", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @@ -122,7 +121,7 @@ def test_add_member(authenticated_user, project_manager, service_account): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_remove_member_pi(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.remove_member('PRJ-123', 'pi', 'username') + project_manager.remove_member("PRJ-123", "pi", "username") project_manager.get_project().remove_member.assert_not_called() project_manager.get_project().remove_co_pi.assert_not_called() project_manager.get_project().remove_pi.assert_called_with(authenticated_user) @@ -137,7 +136,7 @@ def test_remove_member_pi(authenticated_user, project_manager, service_account): "username": "username", "action": "remove", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @@ -145,7 +144,7 @@ def test_remove_member_pi(authenticated_user, project_manager, service_account): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_remove_member_co_pi(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.remove_member('PRJ-123', 'co_pi', 'username') + project_manager.remove_member("PRJ-123", "co_pi", "username") project_manager.get_project().remove_member.assert_not_called() project_manager.get_project().remove_pi.assert_not_called() project_manager.get_project().remove_co_pi.assert_called_with(authenticated_user) @@ -160,7 +159,7 @@ def test_remove_member_co_pi(authenticated_user, project_manager, service_accoun "username": "username", "action": "remove", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @@ -168,7 +167,7 @@ def test_remove_member_co_pi(authenticated_user, project_manager, service_accoun @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_remove_member(authenticated_user, project_manager, service_account): """Test add a PI to a project.""" - project_manager.remove_member('PRJ-123', 'team_member', 'username') + project_manager.remove_member("PRJ-123", "team_member", "username") project_manager.get_project().remove_co_pi.assert_not_called() project_manager.get_project().remove_pi.assert_not_called() @@ -184,27 +183,27 @@ def test_remove_member(authenticated_user, project_manager, service_account): "username": "username", "action": "remove", "root_dir": os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, "PRJ-123"), - } + }, } ) @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_change_project_role(authenticated_user, project_manager, service_account): - project_manager.change_project_role('PRJ-123', 'username', 'co_pi', 'member') - project_manager.get_project().change_project_role.assert_called_with(authenticated_user, 'co_pi', 'member') + project_manager.change_project_role("PRJ-123", "username", "co_pi", "member") + project_manager.get_project().change_project_role.assert_called_with(authenticated_user, "co_pi", "member") @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_change_system_role(authenticated_user, project_manager, service_account): - project_manager.change_system_role('PRJ-123', 'username', 'USER') - project_manager.get_project().change_storage_system_role.assert_called_with(authenticated_user, 'USER') + project_manager.change_system_role("PRJ-123", "username", "USER") + project_manager.get_project().change_storage_system_role.assert_called_with(authenticated_user, "USER") @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_project_manager_create(mocker, authenticated_user, project_manager, portal_project, mock_project_save_signal): - mock_get_latest_project_directory = mocker.patch('portal.apps.projects.managers.base.get_latest_project_directory') - mock_get_latest_project_storage = mocker.patch('portal.apps.projects.managers.base.get_latest_project_storage') + mock_get_latest_project_directory = mocker.patch("portal.apps.projects.managers.base.get_latest_project_directory") + mock_get_latest_project_storage = mocker.patch("portal.apps.projects.managers.base.get_latest_project_storage") mock_get_latest_project_directory.return_value = 11 mock_get_latest_project_storage.return_value = 12 @@ -212,7 +211,7 @@ def test_project_manager_create(mocker, authenticated_user, project_manager, por assert len(ProjectId.objects.all()) == 0 # Project creation should initialize ProjectId - project_manager.create('PRJ-1') + project_manager.create("PRJ-1") assert len(ProjectId.objects.all()) == 1 assert ProjectId.objects.all()[0].value == 13 # max of prj dir and storage values @@ -222,5 +221,5 @@ def test_project_manager_create(mocker, authenticated_user, project_manager, por ProjectId.update(12) portal_project._create_storage.side_effect = ValueError() with pytest.raises(ValueError): - project_manager.create('PRJ-13') + project_manager.create("PRJ-13") assert ProjectId.objects.all()[0].value == 22 # max of prj dir and storage values diff --git a/server/portal/apps/projects/migrations/0001_initial.py b/server/portal/apps/projects/migrations/0001_initial.py index b18e62b1d8..54172f8d31 100644 --- a/server/portal/apps/projects/migrations/0001_initial.py +++ b/server/portal/apps/projects/migrations/0001_initial.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -15,33 +14,81 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='AbstractProjectMetadata', + name="AbstractProjectMetadata", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('title', models.TextField()), - ('project_id', models.CharField(db_index=True, max_length=255)), - ('description', models.TextField(blank=True, null=True)), - ('created', models.DateTimeField(auto_now_add=True)), - ('last_modified', models.DateTimeField(auto_now=True)), - ('co_pis', models.ManyToManyField(blank=True, null=True, related_name='rel_co_pi_abstractprojectmetadata', related_query_name='co_pi_abstractprojectmetadata', to=settings.AUTH_USER_MODEL)), - ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_owner_abstractprojectmetadata', related_query_name='owner_abstractprojectmetadata', to=settings.AUTH_USER_MODEL)), - ('pi', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_pi_abstractprojectmetadata', related_query_name='pi_abstractprojectmetadata', to=settings.AUTH_USER_MODEL)), - ('team_members', models.ManyToManyField(blank=True, null=True, related_name='rel_member_abstractprojectmetadata', related_query_name='member_abstractprojectmetadata', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("title", models.TextField()), + ("project_id", models.CharField(db_index=True, max_length=255)), + ("description", models.TextField(blank=True, null=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ( + "co_pis", + models.ManyToManyField( + blank=True, + null=True, + related_name="rel_co_pi_abstractprojectmetadata", + related_query_name="co_pi_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="rel_owner_abstractprojectmetadata", + related_query_name="owner_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "pi", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="rel_pi_abstractprojectmetadata", + related_query_name="pi_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "team_members", + models.ManyToManyField( + blank=True, + null=True, + related_name="rel_member_abstractprojectmetadata", + related_query_name="member_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.CreateModel( - name='ProjectId', + name="ProjectId", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('value', models.IntegerField()), - ('last_updated', models.DateTimeField(auto_now=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("value", models.IntegerField()), + ("last_updated", models.DateTimeField(auto_now=True)), ], ), migrations.CreateModel( - name='ProjectMetadata', + name="ProjectMetadata", fields=[ - ('abstractprojectmetadata_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='projects.AbstractProjectMetadata')), + ( + "abstractprojectmetadata_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="projects.AbstractProjectMetadata", + ), + ), ], - bases=('projects.abstractprojectmetadata',), + bases=("projects.abstractprojectmetadata",), ), ] diff --git a/server/portal/apps/projects/migrations/0002_auto_20210312_1743.py b/server/portal/apps/projects/migrations/0002_auto_20210312_1743.py index aad2775772..169fecf1e9 100644 --- a/server/portal/apps/projects/migrations/0002_auto_20210312_1743.py +++ b/server/portal/apps/projects/migrations/0002_auto_20210312_1743.py @@ -5,20 +5,29 @@ class Migration(migrations.Migration): - dependencies = [ - ('projects', '0001_initial'), + ("projects", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='abstractprojectmetadata', - name='co_pis', - field=models.ManyToManyField(blank=True, related_name='rel_co_pi_abstractprojectmetadata', related_query_name='co_pi_abstractprojectmetadata', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="co_pis", + field=models.ManyToManyField( + blank=True, + related_name="rel_co_pi_abstractprojectmetadata", + related_query_name="co_pi_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='abstractprojectmetadata', - name='team_members', - field=models.ManyToManyField(blank=True, related_name='rel_member_abstractprojectmetadata', related_query_name='member_abstractprojectmetadata', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="team_members", + field=models.ManyToManyField( + blank=True, + related_name="rel_member_abstractprojectmetadata", + related_query_name="member_abstractprojectmetadata", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py b/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py index af9d42c602..6d7c9e212e 100644 --- a/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py +++ b/server/portal/apps/projects/migrations/0003_alter_abstractprojectmetadata_co_pis_and_more.py @@ -6,31 +6,54 @@ class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('projects', '0002_auto_20210312_1743'), + ("projects", "0002_auto_20210312_1743"), ] operations = [ migrations.AlterField( - model_name='abstractprojectmetadata', - name='co_pis', - field=models.ManyToManyField(blank=True, related_name='rel_co_pi_%(class)s', related_query_name='co_pi_%(class)s', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="co_pis", + field=models.ManyToManyField( + blank=True, + related_name="rel_co_pi_%(class)s", + related_query_name="co_pi_%(class)s", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='abstractprojectmetadata', - name='owner', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_owner_%(class)s', related_query_name='owner_%(class)s', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="owner", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="rel_owner_%(class)s", + related_query_name="owner_%(class)s", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='abstractprojectmetadata', - name='pi', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_pi_%(class)s', related_query_name='pi_%(class)s', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="pi", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="rel_pi_%(class)s", + related_query_name="pi_%(class)s", + to=settings.AUTH_USER_MODEL, + ), ), migrations.AlterField( - model_name='abstractprojectmetadata', - name='team_members', - field=models.ManyToManyField(blank=True, related_name='rel_member_%(class)s', related_query_name='member_%(class)s', to=settings.AUTH_USER_MODEL), + model_name="abstractprojectmetadata", + name="team_members", + field=models.ManyToManyField( + blank=True, + related_name="rel_member_%(class)s", + related_query_name="member_%(class)s", + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py b/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py index a3c15aec2a..8006a92e2d 100644 --- a/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py +++ b/server/portal/apps/projects/migrations/0004_projectsmetadata_squashed_0008_delete_projectsmetadata.py @@ -11,37 +11,94 @@ class Migration(migrations.Migration): - - replaces = [('projects', '0004_projectsmetadata'), ('projects', '0005_projectsmetadata_created_at_and_more'), ('projects', - '0006_rename_projectmetadata_legacyprojectmetadata'), ('projects', '0007_projectmetadata_and_more'), ('projects', '0008_delete_projectsmetadata')] + replaces = [ + ("projects", "0004_projectsmetadata"), + ("projects", "0005_projectsmetadata_created_at_and_more"), + ("projects", "0006_rename_projectmetadata_legacyprojectmetadata"), + ("projects", "0007_projectmetadata_and_more"), + ("projects", "0008_delete_projectsmetadata"), + ] dependencies = [ - ('projects', '0003_alter_abstractprojectmetadata_co_pis_and_more'), + ("projects", "0003_alter_abstractprojectmetadata_co_pis_and_more"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.RenameModel( - old_name='ProjectMetadata', - new_name='LegacyProjectMetadata', + old_name="ProjectMetadata", + new_name="LegacyProjectMetadata", ), migrations.CreateModel( - name='ProjectMetadata', + name="ProjectMetadata", fields=[ - ('uuid', models.CharField(default=portal.apps.projects.models.project_metadata.uuid_pk, editable=False, max_length=100, primary_key=True, serialize=False)), - ('name', models.CharField(help_text="Metadata namespace, e.g. 'designsafe.project'", - max_length=100, validators=[django.core.validators.MinLengthValidator(1)])), - ('value', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, - help_text='JSON document containing file metadata, including title/description')), - ('created', models.DateTimeField(default=django.utils.timezone.now)), - ('last_updated', models.DateTimeField(auto_now=True)), - ('base_project', models.ForeignKey(help_text='Base project containing this entity.For top-level project metadata, this is `self`.', - on_delete=django.db.models.deletion.CASCADE, to='projects.projectmetadata')), - ('users', models.ManyToManyField(help_text='Users who have access to a project.', related_name='projects', to=settings.AUTH_USER_MODEL)), + ( + "uuid", + models.CharField( + default=portal.apps.projects.models.project_metadata.uuid_pk, + editable=False, + max_length=100, + primary_key=True, + serialize=False, + ), + ), + ( + "name", + models.CharField( + help_text="Metadata namespace, e.g. 'designsafe.project'", + max_length=100, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "value", + models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, + help_text="JSON document containing file metadata, including title/description", + ), + ), + ("created", models.DateTimeField(default=django.utils.timezone.now)), + ("last_updated", models.DateTimeField(auto_now=True)), + ( + "base_project", + models.ForeignKey( + help_text="Base project containing this entity.For top-level project metadata, this is `self`.", + on_delete=django.db.models.deletion.CASCADE, + to="projects.projectmetadata", + ), + ), + ( + "users", + models.ManyToManyField( + help_text="Users who have access to a project.", + related_name="projects", + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'indexes': [models.Index(models.F('value__projectId'), name='value_project_id'), models.Index(fields=['name'], name='projects_pr_name_5382e9_idx')], - 'constraints': [models.UniqueConstraint(models.F('value__projectId'), condition=models.Q(('name', constants.PROJECT)), name='unique_id_per_project'), models.UniqueConstraint(condition=models.Q(('name', constants.PROJECT_GRAPH)), fields=('base_project_id',), name='unique_graph_per_project'), models.CheckConstraint(condition=models.Q(('name', constants.PROJECT), ('value__projectId__isnull', True), _negated=True), name='base_projectId_not_null')], + "indexes": [ + models.Index(models.F("value__projectId"), name="value_project_id"), + models.Index(fields=["name"], name="projects_pr_name_5382e9_idx"), + ], + "constraints": [ + models.UniqueConstraint( + models.F("value__projectId"), + condition=models.Q(("name", constants.PROJECT)), + name="unique_id_per_project", + ), + models.UniqueConstraint( + condition=models.Q(("name", constants.PROJECT_GRAPH)), + fields=("base_project_id",), + name="unique_graph_per_project", + ), + models.CheckConstraint( + condition=models.Q( + ("name", constants.PROJECT), ("value__projectId__isnull", True), _negated=True + ), + name="base_projectId_not_null", + ), + ], }, ), ] diff --git a/server/portal/apps/projects/models/__init__.py b/server/portal/apps/projects/models/__init__.py index 0b11b024bf..b19913ac1e 100644 --- a/server/portal/apps/projects/models/__init__.py +++ b/server/portal/apps/projects/models/__init__.py @@ -3,4 +3,4 @@ from portal.apps.projects.models.base import Project, ProjectId from portal.apps.projects.models.metadata import LegacyProjectMetadata -__all__ = ['Project', 'ProjectId', 'LegacyProjectMetadata'] +__all__ = ["Project", "ProjectId", "LegacyProjectMetadata"] diff --git a/server/portal/apps/projects/models/base.py b/server/portal/apps/projects/models/base.py index 2c70b74ab8..754ba957ef 100644 --- a/server/portal/apps/projects/models/base.py +++ b/server/portal/apps/projects/models/base.py @@ -11,6 +11,7 @@ context, if these classes were a direct representation of Agave resources then they should live in `portal.libs.agave.models` """ + import logging import os from django.db import models, transaction @@ -19,6 +20,7 @@ from django.contrib.auth import get_user_model from portal.utils import encryption as EncryptionUtil from portal.libs.agave.utils import service_account + # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem # from portal.libs.agave.serializers import BaseAgaveSystemSerializer @@ -37,39 +39,23 @@ def set_storage_auth(storage): """Set up storage auth details.""" if not settings.PORTAL_PROJECTS_PRIVATE_KEY: key = EncryptionUtil.create_private_key() - priv_key = EncryptionUtil.export_key( - key, - 'PEM' - ) - pub_key = EncryptionUtil.export_key( - EncryptionUtil.create_public_key(key), - 'OpenSSH' - ) + priv_key = EncryptionUtil.export_key(key, "PEM") + pub_key = EncryptionUtil.export_key(EncryptionUtil.create_public_key(key), "OpenSSH") storage.storage.auth.public_key = pub_key storage.storage.auth.private_key = priv_key try: SSHKeys.objects.save_keys( - get_user_model().objects.get( - username=settings.PORTAL_ADMIN_USERNAME - ), + get_user_model().objects.get(username=settings.PORTAL_ADMIN_USERNAME), system_id=storage.id, priv_key=priv_key, - pub_key=pub_key + pub_key=pub_key, ) except Exception as exc: # pylint:disable=broad-except - logger.error( - 'There was an error saving the ssh keys locally: %s', - exc, - exc_info=True - ) + logger.error("There was an error saving the ssh keys locally: %s", exc, exc_info=True) else: - storage.storage.auth.private_key = ( - settings.PORTAL_PROJECTS_PRIVATE_KEY - ) - storage.storage.auth.public_key = ( - settings.PORTAL_PROJECTS_PUBLIC_KEY - ) + storage.storage.auth.private_key = settings.PORTAL_PROJECTS_PRIVATE_KEY + storage.storage.auth.public_key = settings.PORTAL_PROJECTS_PUBLIC_KEY storage.storage.auth.username = settings.PORTAL_ADMIN_USERNAME storage.storage.auth.type = storage.AUTH_TYPES.SSHKEYS @@ -81,13 +67,7 @@ class Project(object): metadata_name = settings.PORTAL_PROJECTS_SYSTEM_PREFIX - def __init__( - self, - client, - project_id, - metadata=None, - storage=None - ): + def __init__(self, client, project_id, metadata=None, storage=None): """Project Init. .. note:: When initializing a project we first retrieve the @@ -138,10 +118,7 @@ def absolute_path(self): if self.storage.storage.root_dir: return self.storage.storage.root_dir - return os.path.join( - settings.PORTAL_PROJECTS_ROOT_DIR, - self.storage.name - ) + return os.path.join(settings.PORTAL_PROJECTS_ROOT_DIR, self.storage.name) def _get_metadata(self): """Get metadata record for project. @@ -164,10 +141,12 @@ def _get_metadata(self): roles = self.storage.roles.to_dict().items() admins = list( filter( - lambda role_tuple: role_tuple[0] != 'wma_prtl' and (role_tuple[1] == 'ADMIN' or role_tuple[1] == 'OWNER'), - roles - ) + lambda role_tuple: ( + role_tuple[0] != "wma_prtl" and (role_tuple[1] == "ADMIN" or role_tuple[1] == "OWNER") + ), + roles, ) + ) if len(admins) == 1: # Exactly one admin found, assign as PI try: @@ -239,19 +218,14 @@ def _create_metadata(title, project_id, owner=None): """ # Create a default metadata object - defaults = { - 'title': title - } + defaults = {"title": title} # If owner is specified for metadata, insert it # into the parameters for the model if owner: - defaults['owner'] = owner + defaults["owner"] = owner - (meta, created) = LegacyProjectMetadata.objects.get_or_create( - project_id=project_id, - defaults=defaults - ) + (meta, created) = LegacyProjectMetadata.objects.get_or_create(project_id=project_id, defaults=defaults) return meta @staticmethod @@ -265,13 +239,7 @@ def _delete_dir(project_id): ProjectsUtils.delete_project_dir(project_id) @classmethod - def create( - cls, - client, - title, - project_id, - owner - ): + def create(cls, client, title, project_id, owner): """Create a project. :param client: Agave client @@ -281,17 +249,9 @@ def create( cls._create_dir(project_id) try: - storage = cls._create_storage( - title, - ProjectsUtils.project_id_to_system_id(project_id), - project_id - ) + storage = cls._create_storage(title, ProjectsUtils.project_id_to_system_id(project_id), project_id) - meta = cls._create_metadata( - title, - project_id, - owner - ) + meta = cls._create_metadata(title, project_id, owner) except Exception as e: cls._delete_dir(project_id) @@ -346,18 +306,16 @@ def _can_edit_member(self, username): :param str username: Username to check for pems. """ role = self.storage.roles.for_user(username) - if (role is not None and - (role.role == role.ADMIN or - role.role == role.OWNER)): + if role is not None and (role.role == role.ADMIN or role.role == role.OWNER): return True return False def transfer_pi(self, old_pi, new_pi): new_pi_role = self.get_project_role(new_pi) - if new_pi_role == 'team_member': + if new_pi_role == "team_member": self.remove_member(new_pi) - elif new_pi_role == 'co_pi': + elif new_pi_role == "co_pi": self.remove_co_pi(new_pi) self.add_pi(new_pi) self.add_co_pi(old_pi) @@ -369,7 +327,7 @@ def _auth_check(self, user): """ if not self._ac._token == service_account()._token: if not self._can_edit_member(self._ac.token.token_username): - raise NotAuthorizedError(extra={'user': user}) + raise NotAuthorizedError(extra={"user": user}) def add_pi(self, user): """Add PI to project. @@ -379,10 +337,7 @@ def add_pi(self, user): """ self._auth_check(user) - self.storage.roles.add( - user.username, - 'OWNER' - ) + self.storage.roles.add(user.username, "OWNER") self.storage.roles.save() self.metadata.pi = user self.save_metadata() @@ -409,10 +364,7 @@ def add_co_pi(self, user): """ self._auth_check(user) - self.storage.roles.add( - user.username, - 'ADMIN' - ) + self.storage.roles.add(user.username, "ADMIN") self.storage.roles.save() self.metadata.co_pis.add(user) self.save_metadata() @@ -442,10 +394,7 @@ def add_member(self, user): """ self._auth_check(user) - self.storage.roles.add( - user.username, - 'USER' - ) + self.storage.roles.add(user.username, "USER") self.storage.roles.save() self.metadata.team_members.add(user) self.save_metadata() @@ -467,10 +416,10 @@ def remove_member(self, user): def change_project_role(self, user, old_role, new_role): # account for difference between role name (team_member) and method # names (add_member, remove_member) - if old_role == 'team_member': - old_role = 'member' - if new_role == 'team_member': - new_role = 'member' + if old_role == "team_member": + old_role = "member" + if new_role == "team_member": + new_role = "member" add_new_role = getattr(self, "add_{}".format(new_role)) remove_old_role = getattr(self, "remove_{}".format(old_role)) remove_old_role(user) @@ -484,17 +433,17 @@ def get_project_role(self, username): role = None if self.metadata.pi.username == username: - role = 'pi' + role = "pi" try: self.metadata.co_pis.get(username=username) - role = 'co_pi' + role = "co_pi" except get_user_model().DoesNotExist: pass try: self.metadata.team_members.get(username=username) - role = 'team_member' + role = "team_member" except get_user_model().DoesNotExist: pass @@ -524,18 +473,13 @@ def save_storage(self): self.storage.update() def change_storage_system_role(self, user, new_role): - self.storage.roles.add( - user.username, - new_role - ) + self.storage.roles.add(user.username, new_role) self.storage.roles.save() def __repr__(self): """Repr.""" - return 'Project({project_id}, {metadata}, {storage})'.format( - project_id=self.project_id, - metadata=self.metadata, - storage=self.storage + return "Project({project_id}, {metadata}, {storage})".format( + project_id=self.project_id, metadata=self.metadata, storage=self.storage ) def __str__(self): @@ -561,7 +505,7 @@ def update(cls, value): If there is no project id row to update, one is created. """ try: - row = cls.objects.select_for_update().latest('last_updated') + row = cls.objects.select_for_update().latest("last_updated") row.value = value row.save() except ObjectDoesNotExist: @@ -572,7 +516,7 @@ def update(cls, value): @transaction.atomic def next_id(cls): """Return next id.""" - row = cls.objects.select_for_update().latest('last_updated') + row = cls.objects.select_for_update().latest("last_updated") row.value += 1 row.save() return row.value diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index 9ce2a14f93..a7ec0b2ede 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -3,6 +3,7 @@ .. :module:: portal.apps.projects.models.metadata :synopsis: Metadata model for projects. """ + import logging from django.conf import settings from django.db import models @@ -33,6 +34,7 @@ class AbstractProjectMetadata(models.Model): :param co_pis: Django user Many-to-Many relation. :param team_members: Django user Many-to-Many relation. """ + title = models.TextField() project_id = models.CharField(max_length=255, db_index=True) description = models.TextField(blank=True, null=True) @@ -44,35 +46,32 @@ class AbstractProjectMetadata(models.Model): related_query_name="owner_%(class)s", blank=True, null=True, - on_delete=models.CASCADE + on_delete=models.CASCADE, ) pi = models.ForeignKey( settings.AUTH_USER_MODEL, - related_name='rel_pi_%(class)s', - related_query_name='pi_%(class)s', + related_name="rel_pi_%(class)s", + related_query_name="pi_%(class)s", blank=True, null=True, - on_delete=models.CASCADE + on_delete=models.CASCADE, ) co_pis = models.ManyToManyField( settings.AUTH_USER_MODEL, - related_name='rel_co_pi_%(class)s', - related_query_name='co_pi_%(class)s', + related_name="rel_co_pi_%(class)s", + related_query_name="co_pi_%(class)s", blank=True, ) team_members = models.ManyToManyField( settings.AUTH_USER_MODEL, - related_name='rel_member_%(class)s', - related_query_name='member_%(class)s', + related_name="rel_member_%(class)s", + related_query_name="member_%(class)s", blank=True, ) def __str__(self): """Str -> self.prj_id - self.title.""" - return '{prj_id} - {title}'.format( - prj_id=self.project_id, - title=self.title - ) + return "{prj_id} - {title}".format(prj_id=self.project_id, title=self.title) class LegacyProjectMetadata(AbstractProjectMetadata): diff --git a/server/portal/apps/projects/models/project_metadata.py b/server/portal/apps/projects/models/project_metadata.py index 1055066079..c3b24851bb 100644 --- a/server/portal/apps/projects/models/project_metadata.py +++ b/server/portal/apps/projects/models/project_metadata.py @@ -1,4 +1,5 @@ """Models for representing project metadata""" + import uuid from django.utils import timezone from django.db import models @@ -11,8 +12,8 @@ def snake_to_camel(snake_str): - components = snake_str.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) + components = snake_str.split("_") + return components[0] + "".join(x.title() for x in components[1:]) def uuid_pk(): @@ -32,9 +33,7 @@ class ProjectMetadata(models.Model): """ - uuid = models.CharField( - max_length=100, primary_key=True, default=uuid_pk, editable=False - ) + uuid = models.CharField(max_length=100, primary_key=True, default=uuid_pk, editable=False) name = models.CharField( max_length=100, validators=[MinLengthValidator(1)], @@ -42,23 +41,16 @@ class ProjectMetadata(models.Model): ) value = models.JSONField( encoder=DjangoJSONEncoder, - help_text=( - "JSON document containing file metadata, including title/description" - ), + help_text=("JSON document containing file metadata, including title/description"), ) users = models.ManyToManyField( - to=user_model, - related_name="projects", - help_text="Users who have access to a project." + to=user_model, related_name="projects", help_text="Users who have access to a project." ) base_project = models.ForeignKey( "self", on_delete=models.CASCADE, - help_text=( - "Base project containing this entity." - "For top-level project metadata, this is `self`." - ), + help_text=("Base project containing this entity.For top-level project metadata, this is `self`."), ) created = models.DateTimeField(default=timezone.now) last_updated = models.DateTimeField(auto_now=True) @@ -71,9 +63,7 @@ def project_id(self) -> str: @property def project_graph(self): """Convenience method for returning the project graph metadata""" - return self.__class__.objects.get( - name=constants.PROJECT_GRAPH, base_project=self.base_project - ) + return self.__class__.objects.get(name=constants.PROJECT_GRAPH, base_project=self.base_project) @classmethod def get_project_by_id(cls, project_id: str): @@ -100,7 +90,7 @@ def to_dict(self): "name": self.name, "value": self.value, "created": self.created, - "lastUpdated": self.last_updated + "lastUpdated": self.last_updated, } def sync_users(self): diff --git a/server/portal/apps/projects/models/unit_test.py b/server/portal/apps/projects/models/unit_test.py index 58d9f801d3..8555e157ff 100644 --- a/server/portal/apps/projects/models/unit_test.py +++ b/server/portal/apps/projects/models/unit_test.py @@ -7,6 +7,7 @@ from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.projects.models.base import Project from portal.apps.projects.models.utils import get_latest_project_storage + # TODOv3: deprecate with projects # from portal.libs.agave.models.systems.storage import StorageSystem import pytest @@ -14,50 +15,40 @@ @pytest.fixture() def agave_client(mocker): - yield mocker.patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) + yield mocker.patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) @pytest.fixture() def mock_owner(django_user_model): - return django_user_model.objects.create_user(username='username', - password='password') + return django_user_model.objects.create_user(username="username", password="password") @pytest.fixture() def mock_service_account(mocker): - yield mocker.patch('portal.apps.projects.models.utils.service_account', autospec=True) + yield mocker.patch("portal.apps.projects.models.utils.service_account", autospec=True) def test_create_metadata(mock_owner, mock_project_save_signal): - project_id = 'PRJ-123' - defaults = { - 'title': 'Project Title', - 'owner': mock_owner - } - (meta, result) = LegacyProjectMetadata.objects.get_or_create( - project_id=project_id, - defaults=defaults - ) + project_id = "PRJ-123" + defaults = {"title": "Project Title", "owner": mock_owner} + (meta, result) = LegacyProjectMetadata.objects.get_or_create(project_id=project_id, defaults=defaults) assert meta is not None - assert meta.project_id == 'PRJ-123' - assert meta.title == 'Project Title' - assert meta.owner.username == 'username' + assert meta.project_id == "PRJ-123" + assert meta.title == "Project Title" + assert meta.owner.username == "username" assert meta.co_pis.count() == 0 assert meta.team_members.count() == 0 def test_metadata_str(mock_owner, mock_project_save_signal): - project_id = 'PRJ-123' + project_id = "PRJ-123" defaults = { - 'title': 'Project Title', + "title": "Project Title", } - meta = LegacyProjectMetadata.objects.get_or_create( - project_id=project_id, - defaults=defaults - ) + meta = LegacyProjectMetadata.objects.get_or_create(project_id=project_id, defaults=defaults) meta_str = str(meta) - assert meta_str == '(, True)' + assert meta_str == "(, True)" @pytest.mark.skip(reason="TODOv3: deprecate with projects") @@ -84,7 +75,7 @@ def test_project_create_storage_failure(mock_owner, portal_project, agave_client @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_metadata_create_on_project_load(agave_client, mock_owner, mock_project_save_signal): - agave_client.systems.listRoles.return_value = [{'username': 'username', 'role': 'ADMIN'}] + agave_client.systems.listRoles.return_value = [{"username": "username", "role": "ADMIN"}] # TODOv3: deprecate with projects # sys = StorageSystem(agave_client, 'cep.test.PRJ-123') # sys.last_modified = '1234' @@ -101,7 +92,7 @@ def test_metadata_create_on_project_load(agave_client, mock_owner, mock_project_ @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_project_change_system_role(agave_client, mock_owner, mock_project_save_signal): - agave_client.systems.listRoles.return_value = [{'username': 'username', 'role': 'ADMIN'}] + agave_client.systems.listRoles.return_value = [{"username": "username", "role": "ADMIN"}] # TODOv3: deprecate with projects # sys = StorageSystem(agave_client, 'cep.test.PRJ-123') # sys.last_modified = '1234' @@ -113,14 +104,14 @@ def test_project_change_system_role(agave_client, mock_owner, mock_project_save_ # ) # prj.change_storage_system_role(mock_owner, 'USER') agave_client.systems.updateRole.assert_called_with( - body={'role': 'USER', 'username': 'username'}, - systemId='cep.test.PRJ-123') + body={"role": "USER", "username": "username"}, systemId="cep.test.PRJ-123" + ) @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_project_change_project_role(agave_client, mock_owner, mock_project_save_signal, mocker): - mock_remove = mocker.patch('portal.apps.projects.models.base.Project.remove_co_pi') - mock_add = mocker.patch('portal.apps.projects.models.base.Project.add_member') + mock_remove = mocker.patch("portal.apps.projects.models.base.Project.remove_co_pi") + mock_add = mocker.patch("portal.apps.projects.models.base.Project.add_member") # TODOv3: deprecate with projects # sys = StorageSystem(agave_client, 'cep.test.PRJ-123') @@ -139,7 +130,9 @@ def test_project_change_project_role(agave_client, mock_owner, mock_project_save @pytest.mark.skip(reason="TODOv3: deprecate with projects") -def test_get_latest_project_storage(mock_owner, portal_project, agave_client, mock_project_save_signal, service_account, mocker, mock_service_account): +def test_get_latest_project_storage( + mock_owner, portal_project, agave_client, mock_project_save_signal, service_account, mocker, mock_service_account +): # TODOv3: deprecate with projects # sys = StorageSystem(agave_client, 'cep.test.SOME-PRJ-5678') # sys.last_modified = '1234' diff --git a/server/portal/apps/projects/models/utils.py b/server/portal/apps/projects/models/utils.py index 79f35749e9..73866e8c77 100644 --- a/server/portal/apps/projects/models/utils.py +++ b/server/portal/apps/projects/models/utils.py @@ -1,4 +1,3 @@ - from django.conf import settings from portal.libs.agave.utils import service_account from portal.libs.agave.operations import iterate_listing @@ -15,24 +14,17 @@ def get_latest_project_storage(max_project_id=None): latest = -1 all_projects = [] while True: - prjs = [p for p in Project.listing( - service_account(), - offset=offset, - limit=limit - )] + prjs = [p for p in Project.listing(service_account(), offset=offset, limit=limit)] all_projects += prjs offset += limit if len(prjs) < limit: break for prj in all_projects: - prj_id = prj.storage.id.replace( - settings.PORTAL_PROJECTS_SYSTEM_PREFIX, - '' - ) - if '-' not in prj_id: + prj_id = prj.storage.id.replace(settings.PORTAL_PROJECTS_SYSTEM_PREFIX, "") + if "-" not in prj_id: continue - prj_id = prj_id.rsplit('-')[-1] + prj_id = prj_id.rsplit("-")[-1] prj_id = int(prj_id) if prj_id > latest and (max_project_id is None or prj_id < max_project_id): @@ -47,13 +39,11 @@ def get_latest_project_directory(max_project_id=None): :param max_project_id: If provided, then ignore projects ids that are greater than or equal to this value. """ latest = -1 - for f in iterate_listing(service_account(), - system=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, - path='/'): + for f in iterate_listing(service_account(), system=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, path="/"): name = f["name"] - if '-' not in name or not name.startswith(settings.PORTAL_PROJECTS_ID_PREFIX): + if "-" not in name or not name.startswith(settings.PORTAL_PROJECTS_ID_PREFIX): continue - _, dir_id = name.rsplit('-', 1) + _, dir_id = name.rsplit("-", 1) dir_id = int(dir_id) if dir_id > latest and (max_project_id is None or dir_id < max_project_id): latest = dir_id diff --git a/server/portal/apps/projects/schema_models/base_metadata.py b/server/portal/apps/projects/schema_models/base_metadata.py index 5a01b692b7..92a0a68a09 100644 --- a/server/portal/apps/projects/schema_models/base_metadata.py +++ b/server/portal/apps/projects/schema_models/base_metadata.py @@ -20,9 +20,7 @@ class BaseMetadataModel(BaseModel): def model_dump(self, *args, **kwargs): # default by_alias to true for camelCase serialization - return partial(super().model_dump, by_alias=True, exclude_none=True)( - *args, **kwargs - ) + return partial(super().model_dump, by_alias=True, exclude_none=True)(*args, **kwargs) class BaseFileMetadata(BaseMetadataModel): diff --git a/server/portal/apps/projects/schema_models/constants.py b/server/portal/apps/projects/schema_models/constants.py index 7a6f243b7e..90974bd5bf 100644 --- a/server/portal/apps/projects/schema_models/constants.py +++ b/server/portal/apps/projects/schema_models/constants.py @@ -15,9 +15,7 @@ # Override with portal-specific constants (domain entity types, or explicit # overrides of the names above) try: - _portal_constants = importlib.import_module( - f"portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.constants" - ) + _portal_constants = importlib.import_module(f"portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.constants") for _name in dir(_portal_constants): if _name.isupper() and not _name.startswith("_"): globals()[_name] = getattr(_portal_constants, _name) diff --git a/server/portal/apps/projects/schema_models/schema.py b/server/portal/apps/projects/schema_models/schema.py index 2120ac4db9..ae93db258b 100644 --- a/server/portal/apps/projects/schema_models/schema.py +++ b/server/portal/apps/projects/schema_models/schema.py @@ -20,9 +20,7 @@ # Merge the active portal's schema extension (domain entity types + overrides). try: - _portal_schema = importlib.import_module( - f"portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.schema" - ) + _portal_schema = importlib.import_module(f"portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.schema") SCHEMA_MAPPING.update(_portal_schema.SCHEMA_MAPPING) except ModuleNotFoundError: pass diff --git a/server/portal/apps/projects/serializers.py b/server/portal/apps/projects/serializers.py index 8559efc089..c2fcb8b532 100644 --- a/server/portal/apps/projects/serializers.py +++ b/server/portal/apps/projects/serializers.py @@ -3,6 +3,7 @@ .. module:: portal.apps.projects.serializers :synopsis: Serializer classes for project objects. """ + import logging import datetime import json @@ -13,7 +14,7 @@ LOGGER = logging.getLogger(__name__) # pylint: disable=redefined-builtin, invalid-name -all = ['MetadataJSONSerializer'] +all = ["MetadataJSONSerializer"] def _seralize_user(user): @@ -21,12 +22,7 @@ def _seralize_user(user): :param user: User model instance. """ - return { - 'last_name': user.last_name, - 'first_name': user.first_name, - 'email': user.email, - 'username': user.username - } + return {"last_name": user.last_name, "first_name": user.first_name, "email": user.email, "username": user.username} class MetadataJSONSerializer(json.JSONEncoder): @@ -43,10 +39,7 @@ def default(self, obj): # pylint: disable=method-hidden, arguments-differ val = field.value_from_object(obj) if isinstance(val, datetime.datetime): ret[attname] = val.isoformat() - elif ( - field.remote_field and - field.remote_field.model is get_user_model() - ): + elif field.remote_field and field.remote_field.model is get_user_model(): # is a foreignkey field to UserModel. attname = to_camel_case(field.name) related = getattr(obj, field.name) @@ -64,10 +57,7 @@ def default(self, obj): # pylint: disable=method-hidden, arguments-differ attname = to_camel_case(field.name) related = getattr(obj, field.name) if field.remote_field.model is get_user_model(): - ret[attname] = [ - _seralize_user(user) for user in - related.iterator() - ] + ret[attname] = [_seralize_user(user) for user in related.iterator()] else: ret[attname] = field.value_to_string(obj) return ret diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 5506d981ec..a0e433914a 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -11,7 +11,7 @@ add_file_associations, create_file_obj, get_file_obj, - get_ordered_value + get_ordered_value, ) from portal.apps.projects.workspace_operations.graph_operations import get_path_uuid_mapping from portal.apps.projects.schema_models.base_metadata import FileObj @@ -21,27 +21,25 @@ conf_tiff, create_animation, create_histogram, - create_thumbnail + create_thumbnail, ) from portal.apps.notifications.models import Notification logger = logging.getLogger(__name__) -@shared_task(bind=True, max_retries=3, queue='default') +@shared_task(bind=True, max_retries=3, queue="default") def sync_files_without_metadata(self, user_access_token, project_id: str): client = user_account(user_access_token) path_uuid_map = get_path_uuid_mapping(project_id) - tapis_files_listing = client.files.listFiles(systemId=project_id, path='/', recurse=True) - files = [file for file in tapis_files_listing if file.type != 'dir'] + tapis_files_listing = client.files.listFiles(systemId=project_id, path="/", recurse=True) + files = [file for file in tapis_files_listing if file.type != "dir"] required_uuids = set(path_uuid_map.values()) # Cache to avoid repeated database queries for the same parent path - entity_cache = { - entity.uuid: entity for entity in ProjectMetadata.objects.filter(uuid__in=required_uuids) - } + entity_cache = {entity.uuid: entity for entity in ProjectMetadata.objects.filter(uuid__in=required_uuids)} files_to_add_dict = {} @@ -49,8 +47,8 @@ def sync_files_without_metadata(self, user_access_token, project_id: str): file_path = file.path parent_path = str(Path(file_path).parent) - if parent_path == '.': - parent_path = '' + if parent_path == ".": + parent_path = "" # Check if the parent path exists in path_uuid_map if parent_path not in path_uuid_map: @@ -65,113 +63,119 @@ def sync_files_without_metadata(self, user_access_token, project_id: str): continue entity_value = get_ordered_value(entity.name, entity.value) - file_objs = entity_value.get('file_objs', []) - file_paths_set = {file_obj.get('path') for file_obj in file_objs} + file_objs = entity_value.get("file_objs", []) + file_paths_set = {file_obj.get("path") for file_obj in file_objs} if file_path not in file_paths_set: - new_file_obj = create_file_obj(project_id, file.name, file.size, file_path, {'data_type': 'file'}) + new_file_obj = create_file_obj(project_id, file.name, file.size, file_path, {"data_type": "file"}) files_to_add_dict[entity.uuid] = files_to_add_dict.get(entity.uuid, []) + [new_file_obj] for entity_uuid, file_objs in files_to_add_dict.items(): - logger.info(f'Adding {len(file_objs)} files to entity {entity_uuid} in project {project_id}') + logger.info(f"Adding {len(file_objs)} files to entity {entity_uuid} in project {project_id}") add_file_associations(entity_uuid, file_objs) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def process_file(self, project_id: str, path: str, user_access_token: str, username: str, encoded_file=None): client = user_account(user_access_token) - logger.info(f'Processing file {path} in project {project_id}') + logger.info(f"Processing file {path} in project {project_id}") if encoded_file: - logger.info('Decoding file') + logger.info("Decoding file") file = base64.b64decode(encoded_file) else: - logger.info('Retrieving file using Tapis') + logger.info("Retrieving file using Tapis") file = client.files.getContents(systemId=project_id, path=path) - logger.info('File retrieved') + logger.info("File retrieved") parent_path = str(Path(path).parent) file_obj: FileObj = get_file_obj(project_id, path) if file and file_obj: - value = get_ordered_value(constants.FILE, file_obj.get('value')) + value = get_ordered_value(constants.FILE, file_obj.get("value")) - file_name = file_obj.get('name') + file_name = file_obj.get("name") - _, file_ext = os.path.splitext(file_obj.get('name')) + _, file_ext = os.path.splitext(file_obj.get("name")) try: - if file_ext in ['.tif', '.tiff']: + if file_ext in [".tif", ".tiff"]: adv_image = conf_tiff(file) else: adv_image = conf_raw(value, file) except Exception as e: - logger.error(f'Could not generate advanced image for {file_name} due to error: {e}') + logger.error(f"Could not generate advanced image for {file_name} due to error: {e}") - Notification.objects.create(**{ - Notification.EVENT_TYPE: 'projects', - Notification.STATUS: Notification.INFO, - Notification.USER: username, - Notification.MESSAGE: f'Failed to Generate Images for {Path(path).name}', - }) + Notification.objects.create( + **{ + Notification.EVENT_TYPE: "projects", + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: f"Failed to Generate Images for {Path(path).name}", + } + ) return - Notification.objects.create(**{ - Notification.EVENT_TYPE: 'projects', - Notification.STATUS: Notification.INFO, - Notification.USER: username, - Notification.MESSAGE: f'Generating Images for {Path(path).name}', - }) + Notification.objects.create( + **{ + Notification.EVENT_TYPE: "projects", + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: f"Generating Images for {Path(path).name}", + } + ) try: - if value.get('use_binary_correction'): + if value.get("use_binary_correction"): adv_image = binary_correction(adv_image) except Exception as e: - logger.error(f'Error applying binary correction: {e}') + logger.error(f"Error applying binary correction: {e}") try: thumbnail = create_thumbnail(adv_image) - thumbnail_path = f'{parent_path}/{file_name}.thumb.jpg' + thumbnail_path = f"{parent_path}/{file_name}.thumb.jpg" - logger.info('Uploading generated thumbnail') + logger.info("Uploading generated thumbnail") client.files.insert(systemId=project_id, path=thumbnail_path, file=thumbnail) except Exception as e: - logger.error(f'Error generating thumbnail: {e}') + logger.error(f"Error generating thumbnail: {e}") try: histogram_img, histogram_csv = create_histogram(adv_image) - histogram_img_path = f'{parent_path}/{file_name}.histogram.jpg' - histogram_csv_path = f'{parent_path}/{file_name}.histogram.csv' + histogram_img_path = f"{parent_path}/{file_name}.histogram.jpg" + histogram_csv_path = f"{parent_path}/{file_name}.histogram.csv" - logger.info('Uploading generated histogram') + logger.info("Uploading generated histogram") client.files.insert(systemId=project_id, path=histogram_img_path, file=histogram_img) client.files.insert(systemId=project_id, path=histogram_csv_path, file=histogram_csv) except Exception as e: - logger.error(f'Error generating histogram: {e}') + logger.error(f"Error generating histogram: {e}") try: animation = create_animation(adv_image) - animation_path = f'{parent_path}/{file_name}.gif' + animation_path = f"{parent_path}/{file_name}.gif" - logger.info('Uploading generated animation') + logger.info("Uploading generated animation") client.files.insert(systemId=project_id, path=animation_path, file=animation) except Exception as e: - logger.error(f'Error generating animation: {e}') + logger.error(f"Error generating animation: {e}") with transaction.atomic(): - Notification.objects.create(**{ - Notification.EVENT_TYPE: 'projects', - Notification.STATUS: Notification.INFO, - Notification.USER: username, - Notification.MESSAGE: 'Image generation complete. Please refresh the page.', - }) + Notification.objects.create( + **{ + Notification.EVENT_TYPE: "projects", + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: "Image generation complete. Please refresh the page.", + } + ) else: print(f"File {path} does not exist in project {project_id}") diff --git a/server/portal/apps/projects/unit_test.py b/server/portal/apps/projects/unit_test.py index 610016ef4a..f9af3e377a 100644 --- a/server/portal/apps/projects/unit_test.py +++ b/server/portal/apps/projects/unit_test.py @@ -25,9 +25,7 @@ # Fixtures @pytest.fixture def mock_service_account(mocker): - yield mocker.patch( - "portal.apps.projects.models.base.service_account", autospec=True - ) + yield mocker.patch("portal.apps.projects.models.base.service_account", autospec=True) @pytest.fixture() @@ -130,9 +128,7 @@ def create_shared_workspace( owner=mock_owner, rootDir=f"/corral/tacc/aci/CEP/projects/test.project-{workspace_num}", ) - client.systems.getSystem.assert_called_with( - systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME - ) + client.systems.getSystem.assert_called_with(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME) client.systems.patchSystem.assert_called_with( systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, notes={"count": workspace_num}, @@ -334,7 +330,9 @@ def test_project_init(mock_tapis_client, mock_owner): "owner": mock_owner, } client.systems.createSystem.assert_called_with(**system_args) - client.systems.getSystems.return_value = system_args # If there are multiple projects, this is the one "mock returned" from "storage" + client.systems.getSystems.return_value = ( + system_args # If there are multiple projects, this is the one "mock returned" from "storage" + ) # Assertion tests # Grab the current project that is in the storage that we want to assert @@ -384,15 +382,10 @@ def test_project_create(mock_tapis_client, mock_owner, authenticated_user): created_project = client.systems.createSystem.call_args[1] assert created_project["id"] == expected_result.id assert created_project["notes"]["title"] == expected_result.notes.title - assert ( - created_project["notes"]["description"] == expected_result.notes.description - ) + assert created_project["notes"]["description"] == expected_result.notes.description assert created_project["effectiveUserId"] == expected_result.effectiveUserId assert created_project["port"] == expected_result.port - assert ( - created_project["authnCredential"]["privateKey"] - == expected_result.authnCredential.privateKey - ) + assert created_project["authnCredential"]["privateKey"] == expected_result.authnCredential.privateKey # Testing if there are two projects Tapis @@ -841,9 +834,7 @@ def test_get_workspace_role(mock_tapis_client, mock_owner, authenticated_user): {"user": ws_o.get_project_user("username"), "access": "owner"}, {"user": ws_o.get_project_user(new_username), "access": "writer"}, ] - client.files.getPermissions.return_value = TapisResult( - id=new_username, permission="MODIFY" - ) + client.files.getPermissions.return_value = TapisResult(id=new_username, permission="MODIFY") role = mock_get_workspace_role(mock_tapis_client, workspace_id, new_username) assert role == "USER" @@ -877,9 +868,7 @@ def test_get_workspace_role(mock_tapis_client, mock_owner, authenticated_user): {"user": ws_o.get_project_user(new_username), "access": "writer"}, {"user": ws_o.get_project_user("GuestAccount"), "access": "reader"}, ] - client.files.getPermissions.return_value = TapisResult( - id="GuestAccount", permission="READ" - ) + client.files.getPermissions.return_value = TapisResult(id="GuestAccount", permission="READ") role = mock_get_workspace_role(mock_tapis_client, workspace_id, "GuestAccount") assert role == "GUEST" @@ -1123,16 +1112,11 @@ def test_update_project(mock_tapis_client, mock_owner, authenticated_user): created_project = client.systems.createSystem.call_args[1] assert created_project["id"] == expected_result.id assert created_project["notes"]["title"] == expected_result.notes.title - assert ( - created_project["notes"]["description"] == expected_result.notes.description - ) + assert created_project["notes"]["description"] == expected_result.notes.description assert created_project["notes"]["keywords"] == expected_result.notes.keywords assert created_project["effectiveUserId"] == expected_result.effectiveUserId assert created_project["port"] == expected_result.port - assert ( - created_project["authnCredential"]["privateKey"] - == expected_result.authnCredential.privateKey - ) + assert created_project["authnCredential"]["privateKey"] == expected_result.authnCredential.privateKey # Change the title and description # Change the title and description diff --git a/server/portal/apps/projects/urls.py b/server/portal/apps/projects/urls.py index a0933e80b4..6d1c6b6a14 100644 --- a/server/portal/apps/projects/urls.py +++ b/server/portal/apps/projects/urls.py @@ -1,17 +1,17 @@ -"""Data Depot API Urls -""" +"""Data Depot API Urls""" + from portal.apps.projects import views from django.urls import path -app_name = 'projects' +app_name = "projects" urlpatterns = [ - path('system//', views.ProjectInstanceApiView.as_view(), name='project_sys'), - path('/members/', views.ProjectMembersApiView.as_view()), - path('/project-role//', views.get_project_role), - path('/system-role//', views.get_system_role), - path('/', views.ProjectInstanceApiView.as_view(), name='project'), - path('/entities/create', views.ProjectEntityView.as_view()), - path('/tree/', views.ProjectTreeView.as_view(), name='project_tree'), - path('', views.ProjectsApiView.as_view()), - path('', views.ProjectsApiView.as_view(), name='projects_api') + path("system//", views.ProjectInstanceApiView.as_view(), name="project_sys"), + path("/members/", views.ProjectMembersApiView.as_view()), + path("/project-role//", views.get_project_role), + path("/system-role//", views.get_system_role), + path("/", views.ProjectInstanceApiView.as_view(), name="project"), + path("/entities/create", views.ProjectEntityView.as_view()), + path("/tree/", views.ProjectTreeView.as_view(), name="project_tree"), + path("", views.ProjectsApiView.as_view()), + path("", views.ProjectsApiView.as_view(), name="projects_api"), ] diff --git a/server/portal/apps/projects/utils.py b/server/portal/apps/projects/utils.py index 98f7ab8340..2d5132cd77 100644 --- a/server/portal/apps/projects/utils.py +++ b/server/portal/apps/projects/utils.py @@ -2,10 +2,12 @@ .. :module:: portal.apps.projects.models.utils :synopsis: Utils for projects """ + import logging from django.conf import settings from portal.libs.agave.utils import service_account from portal.libs.agave.operations import mkdir, delete + # pylint: disable=invalid-name logger = logging.getLogger(__name__) # pylint: enable=invalid-name @@ -13,7 +15,7 @@ def create_project_dir(project_id): client = service_account() - return mkdir(client, settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, '', project_id) + return mkdir(client, settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, "", project_id) def delete_project_dir(project_id): @@ -36,7 +38,4 @@ def project_id_to_system_id(project_id): :param str project_id: Project Id. """ - return '{prefix}.{prj_id}'.format( - prefix=settings.PORTAL_PROJECTS_SYSTEM_PREFIX, - prj_id=project_id - ) + return "{prefix}.{prj_id}".format(prefix=settings.PORTAL_PROJECTS_SYSTEM_PREFIX, prj_id=project_id) diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index b2c27b79b7..f91fa0080d 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -3,6 +3,7 @@ .. :module:: apps.projects.views :synopsis: Views to handle Projects """ + import json import logging from django.http import HttpRequest, JsonResponse @@ -16,10 +17,18 @@ from portal.exceptions.api import ApiException from portal.views.base import BaseApiView from portal.apps.projects.managers.base import ProjectsManager -from portal.apps.projects.workspace_operations.shared_workspace_operations import \ - list_projects, get_project, create_shared_workspace, \ - update_project, get_workspace_role, change_user_role, add_user_to_workspace, \ - remove_user, transfer_ownership, increment_workspace_count +from portal.apps.projects.workspace_operations.shared_workspace_operations import ( + list_projects, + get_project, + create_shared_workspace, + update_project, + get_workspace_role, + change_user_role, + add_user_to_workspace, + remove_user, + transfer_ownership, + increment_workspace_count, +) from portal.apps.search.tasks import tapis_project_listing_indexer from portal.libs.elasticsearch.indexes import IndexedProject from elasticsearch_dsl import Q @@ -27,13 +36,24 @@ from django.db import transaction from portal.apps.projects.schema_models.schema import SCHEMA_MAPPING from django.db import models -from portal.apps.projects.workspace_operations.project_meta_operations import create_entity_metadata, \ - create_project_metadata, get_ordered_value, move_entity, patch_entity_and_node, \ - patch_file_obj_entity, patch_project_entity +from portal.apps.projects.workspace_operations.project_meta_operations import ( + create_entity_metadata, + create_project_metadata, + get_ordered_value, + move_entity, + patch_entity_and_node, + patch_file_obj_entity, + patch_project_entity, +) from portal.libs.agave.operations import mkdir from pathlib import Path from portal.apps.projects.schema_models import constants -from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path, build_project_tree +from portal.apps.projects.workspace_operations.graph_operations import ( + add_node_to_project, + initialize_project_graph, + get_node_from_path, + build_project_tree, +) from portal.apps.projects.tasks import sync_files_without_metadata from portal.libs.files.file_processing import resize_cover_image from django.http.multipartparser import MultiPartParser @@ -64,7 +84,7 @@ def get_workspace_id(project_id): """Return a workspace id from a system-style project id.""" prefix = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}." if project_id.startswith(prefix): - return project_id[len(prefix):] + return project_id[len(prefix) :] return project_id @@ -84,8 +104,8 @@ def get_project_for_user(project_id, user): return project -@method_decorator(agave_jwt_login, name='dispatch') -@method_decorator(login_required, name='dispatch') +@method_decorator(agave_jwt_login, name="dispatch") +@method_decorator(login_required, name="dispatch") class ProjectsApiView(BaseApiView): """Projects API view. @@ -126,9 +146,9 @@ def get(self, request, root_system=None): ``` """ - query_string = request.GET.get('query_string') - offset = int(request.GET.get('offset', 0)) - limit = int(request.GET.get('limit', 100)) + query_string = request.GET.get("query_string") + offset = int(request.GET.get("offset", 0)) + limit = int(request.GET.get("limit", 100)) METRICS.info( "Projects", @@ -147,24 +167,29 @@ def get(self, request, root_system=None): if query_string: search = IndexedProject.search() - ngram_query = Q("query_string", query=query_string.lower(), - fields=["title", "id"], - minimum_should_match='100%', - default_operator='or') + ngram_query = Q( + "query_string", + query=query_string.lower(), + fields=["title", "id"], + minimum_should_match="100%", + default_operator="or", + ) - wildcard_query = Q("wildcard", title=f'*{query_string.lower()}*') | Q("wildcard", id=f'*{query_string.lower()}*') + wildcard_query = Q("wildcard", title=f"*{query_string.lower()}*") | Q( + "wildcard", id=f"*{query_string.lower()}*" + ) search = search.query(ngram_query | wildcard_query) search = search.extra(from_=int(offset), size=int(limit)) res = search.execute() - hits = [hit.id for hit in res if hasattr(hit, 'id') and hit.id is not None] + hits = [hit.id for hit in res if hasattr(hit, "id") and hit.id is not None] listing = [] # Filter search results to projects specific to user if hits: client = get_project_client(request.user) listing = list_projects(client, root_system) - filtered_list = filter(lambda prj: prj['id'] in hits, listing) + filtered_list = filter(lambda prj: prj["id"] in hits, listing) listing = list(filtered_list) else: client = get_project_client(request.user) @@ -174,9 +199,9 @@ def get(self, request, root_system=None): if settings.PORTAL_PROJECTS_ENABLE_METADATA: for project in listing: try: - project_meta = ProjectMetadata.objects.get(models.Q(value__projectId=project['id'])) + project_meta = ProjectMetadata.objects.get(models.Q(value__projectId=project["id"])) project.update(get_ordered_value(project_meta.name, project_meta.value)) - project["projectId"] = project['id'].split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] + project["projectId"] = project["id"].split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] except Exception as e: LOGGER.exception(f"Failed to retrieve metadata for project {project['id']}: {e}") @@ -187,11 +212,11 @@ def get(self, request, root_system=None): @transaction.atomic def post(self, request): # pylint: disable=no-self-use """POST handler.""" - title = request.POST.get('title') - description = request.POST.get('description') - metadata = request.POST.get('metadata') - cover_image = request.FILES.get('cover_image') - keywords = request.POST.get('keywords') + title = request.POST.get("title") + description = request.POST.get("description") + metadata = request.POST.get("metadata") + cover_image = request.FILES.get("cover_image") + keywords = request.POST.get("keywords") workspace_number = increment_workspace_count() system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" @@ -207,23 +232,34 @@ def post(self, request): # pylint: disable=no-self-use project_metadata["projectId"] = system_id if cover_image: - project_metadata['cover_image'] = f'media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}' + project_metadata["cover_image"] = ( + f"media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}" + ) project_meta = create_project_metadata(project_metadata) initialize_project_graph(project_meta.project_id) client = request.user.tapis_oauth.client - session_key_hash = sha256((request.session.session_key or '').encode()).hexdigest() - system_id = create_shared_workspace(client, title, description, keywords, request.user.username, - workspace_number, tapis_tracking_id=f"portals.{session_key_hash}") + session_key_hash = sha256((request.session.session_key or "").encode()).hexdigest() + system_id = create_shared_workspace( + client, + title, + description, + keywords, + request.user.username, + workspace_number, + tapis_tracking_id=f"portals.{session_key_hash}", + ) # Upload cover image to media folder if cover_image: service_client = service_account() resized_file = resize_cover_image(cover_image) - service_client.files.insert(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, - path=f'media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}', - file=resized_file) + service_client.files.insert( + systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f"media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}", + file=resized_file, + ) METRICS.info( "Projects", @@ -237,15 +273,10 @@ def post(self, request): # pylint: disable=no-self-use }, ) - return JsonResponse( - { - 'status': 200, - 'response': {"id": system_id} - } - ) + return JsonResponse({"status": 200, "response": {"id": system_id}}) -@method_decorator(agave_jwt_login, name='dispatch') +@method_decorator(agave_jwt_login, name="dispatch") class ProjectInstanceApiView(BaseApiView): """Project Instance API view. @@ -267,7 +298,11 @@ def get(self, request, project_id=None, system_id=None): if system_id is not None: project_id = system_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] - if system_id and settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX and system_id.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + if ( + system_id + and settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + and system_id.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX) + ): client = service_account() else: client = get_project_client(request.user) @@ -289,7 +324,9 @@ def get(self, request, project_id=None, system_id=None): # Retrieve project metadata entity for metadata enabled portals if settings.PORTAL_PROJECTS_ENABLE_METADATA: try: - project = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) + project = ProjectMetadata.objects.get( + models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") + ) prj.update(get_ordered_value(project.name, project.value)) prj["projectId"] = project_id @@ -303,29 +340,27 @@ def get(self, request, project_id=None, system_id=None): else: root_system = settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME - postit = service_client.files.createPostIt(systemId=root_system, path=prj['cover_image'], allowedUses=-1, - validSeconds=86400) + postit = service_client.files.createPostIt( + systemId=root_system, path=prj["cover_image"], allowedUses=-1, validSeconds=86400 + ) prj["file_url"] = postit.redeemUrl - if not prj.get('is_review_project', False) and not prj.get('is_published_project', False): - sync_files_without_metadata.delay(client.access_token.access_token, f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") + if not prj.get("is_review_project", False) and not prj.get("is_published_project", False): + sync_files_without_metadata.delay( + client.access_token.access_token, f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" + ) except Exception as e: LOGGER.exception(f"Failed to retrieve metadata for project {project_id}: {e}") return JsonResponse( { - 'status': 200, - 'response': prj, + "status": 200, + "response": prj, } ) @transaction.atomic - def patch( - self, - request, - project_id=None, - system_id=None - ): # pylint: disable=no-self-use + def patch(self, request, project_id=None, system_id=None): # pylint: disable=no-self-use """Update one or multiple fields. This method should be used to update metadata values **mainly**. @@ -352,14 +387,13 @@ def patch( :param request: Request object :param str project_id: Project Id. """ - query_dict, multi_value_dict = MultiPartParser(request.META, request, - request.upload_handlers).parse() + query_dict, multi_value_dict = MultiPartParser(request.META, request, request.upload_handlers).parse() - title = query_dict.get('title') - description = query_dict.get('description') - metadata = query_dict.get('metadata') - cover_image = multi_value_dict.get('cover_image') - keywords = query_dict.get('keywords') + title = query_dict.get("title") + description = query_dict.get("description") + metadata = query_dict.get("metadata") + cover_image = multi_value_dict.get("cover_image") + keywords = query_dict.get("keywords") project_id_full = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" @@ -396,7 +430,7 @@ def patch( project_metadata.setdefault("keywords", keywords) if cover_image: - project_metadata['cover_image'] = f'media/{project_id}/cover_image/{cover_image.name}' + project_metadata["cover_image"] = f"media/{project_id}/cover_image/{cover_image.name}" try: entity = patch_project_entity(project_id_full, project_metadata) @@ -408,34 +442,33 @@ def patch( workspace_def.update(get_ordered_value(entity.name, entity.value)) workspace_def["projectId"] = project_id - if cover_image or workspace_def.get('cover_image') is not None: + if cover_image or workspace_def.get("cover_image") is not None: service_client = service_account() # Upload cover image to media folder if cover_image: resized_file = resize_cover_image(cover_image) - service_client.files.insert(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, - path=f'media/{project_id}/cover_image/{cover_image.name}', - file=resized_file) + service_client.files.insert( + systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f"media/{project_id}/cover_image/{cover_image.name}", + file=resized_file, + ) - if workspace_def.get('cover_image') is not None: + if workspace_def.get("cover_image") is not None: # Get the postit for the cover image - postit = service_client.files.createPostIt(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, - path=f"media/{project_id}/cover_image/{Path(workspace_def['cover_image']).name}", - allowedUses=-1, - validSeconds=86400) + postit = service_client.files.createPostIt( + systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f"media/{project_id}/cover_image/{Path(workspace_def['cover_image']).name}", + allowedUses=-1, + validSeconds=86400, + ) workspace_def["file_url"] = postit.redeemUrl - return JsonResponse( - { - 'status': 200, - 'response': workspace_def - } - ) + return JsonResponse({"status": 200, "response": workspace_def}) -@method_decorator(agave_jwt_login, name='dispatch') -@method_decorator(login_required, name='dispatch') +@method_decorator(agave_jwt_login, name="dispatch") +@method_decorator(login_required, name="dispatch") class ProjectMembersApiView(BaseApiView): """Project Members API view.""" @@ -445,20 +478,12 @@ def patch(self, request, project_id): Process any action on a project """ data = json.loads(request.body) - action = data.get('action') + action = data.get("action") try: operation = getattr(self, action.lower()) except AttributeError: - LOGGER.error( - 'Invalid action.', - extra=request.POST.dict(), - exc_info=True - ) - raise ApiException( - 'Invalid action.', - 403, - request.POST.dict() - ) + LOGGER.error("Invalid action.", extra=request.POST.dict(), exc_info=True) + raise ApiException("Invalid action.", 403, request.POST.dict()) METRICS.info( "Projects", @@ -475,16 +500,11 @@ def patch(self, request, project_id): return operation(request, project_id, **data) def transfer_ownership(self, request, project_id, **data): - old_pi = data.get('oldOwner') - new_pi = data.get('newOwner') + old_pi = data.get("oldOwner") + new_pi = data.get("newOwner") client = request.user.tapis_oauth.client res = transfer_ownership(client, project_id, new_pi, old_pi) - return JsonResponse( - { - 'status': 200, - 'response': res - } - ) + return JsonResponse({"status": 200, "response": res}) # pylint: disable=no-self-use def add_member(self, request, project_id, **data): @@ -492,16 +512,11 @@ def add_member(self, request, project_id, **data): In Shared Workspaces (CEPv2) members can only be added with "edit" access, which translates to co_pi """ - username = data.get('username') + username = data.get("username") client = request.user.tapis_oauth.client resp = add_user_to_workspace(client, project_id, username) - return JsonResponse( - { - 'status': 200, - 'response': resp - } - ) + return JsonResponse({"status": 200, "response": resp}) def remove_member(self, request, project_id, **data): """Remove member from project. @@ -510,51 +525,38 @@ def remove_member(self, request, project_id, **data): :param str project_id: Project id. :param dict data: Data. """ - username = data.get('username') + username = data.get("username") client = request.user.tapis_oauth.client resp = remove_user(client, project_id, username) - return JsonResponse( - { - 'status': 200, - 'response': resp - } - ) + return JsonResponse({"status": 200, "response": resp}) def change_project_role(self, request, project_id, **data): - username = data.get('username') - old_role = data.get('oldRole') - new_role = data.get('newRole') - prj = ProjectsManager(request.user).change_project_role( - project_id, - username, - old_role, - new_role - ) + username = data.get("username") + old_role = data.get("oldRole") + new_role = data.get("newRole") + prj = ProjectsManager(request.user).change_project_role(project_id, username, old_role, new_role) return JsonResponse( { - 'status': 200, - 'response': prj.metadata, + "status": 200, + "response": prj.metadata, }, - encoder=ProjectsManager.meta_serializer_cls + encoder=ProjectsManager.meta_serializer_cls, ) def change_system_role(self, request, project_Id, **data): - username = data.get('username') - new_role = data.get('newRole') + username = data.get("username") + new_role = data.get("newRole") client = request.user.tapis_oauth.client - role_map = { - "GUEST": "reader", - "USER": "writer" - } + role_map = {"GUEST": "reader", "USER": "writer"} change_user_role(client, project_Id, username, role_map[new_role]) return JsonResponse( { - 'status': 200, - 'response': 'OK', + "status": 200, + "response": "OK", } ) @@ -578,7 +580,7 @@ def get_project_role(request, project_id, username): role = get_workspace_role(client, project_id, username) - return JsonResponse({'username': username, 'role': role}) + return JsonResponse({"username": username, "role": role}) @login_required @@ -599,11 +601,10 @@ def get_system_role(request, project_id, username): role = get_workspace_role(client, project_id, username) - return JsonResponse({'username': username, 'role': role}) + return JsonResponse({"username": username, "role": role}) class ProjectEntityView(BaseApiView): - def patch(self, request: HttpRequest, project_id: str): if not request.user.is_authenticated: @@ -613,9 +614,7 @@ def patch(self, request: HttpRequest, project_id: str): try: get_project_for_user(project_id, request.user) except ProjectMetadata.DoesNotExist as exc: - raise ApiException( - "User does not have access to the requested project", status=403 - ) from exc + raise ApiException("User does not have access to the requested project", status=403) from exc req_body = json.loads(request.body) value = req_body.get("value", {}) @@ -623,7 +622,7 @@ def patch(self, request: HttpRequest, project_id: str): path = req_body.get("path", "") updated_path = req_body.get("updatedPath", "") - if value['data_type'] == 'file': + if value["data_type"] == "file": try: patch_file_obj_entity(client, project_id, value, path) except Exception as exc: @@ -647,26 +646,28 @@ def post(self, request: HttpRequest, project_id: str): try: get_project_for_user(project_id, request.user) except ProjectMetadata.DoesNotExist as exc: - raise ApiException( - "User does not have access to the requested project", status=403 - ) from exc + raise ApiException("User does not have access to the requested project", status=403) from exc req_body = json.loads(request.body) value = req_body.get("value", {}) name = req_body.get("name", "") path = req_body.get("path", "") - new_meta = create_entity_metadata(project_id, getattr(constants, name.upper()), { - **value, - }) + new_meta = create_entity_metadata( + project_id, + getattr(constants, name.upper()), + { + **value, + }, + ) # FOR CREATING GRAPH parent_node = get_node_from_path(project_id, path) - add_node_to_project(project_id, parent_node['id'], new_meta.uuid, new_meta.name, value['name']) + add_node_to_project(project_id, parent_node["id"], new_meta.uuid, new_meta.name, value["name"]) # FOR CREATING DATA FILE FOLDER - if (value and path): - mkdir(client, project_id, path, value['name']) + if value and path: + mkdir(client, project_id, path, value["name"]) return JsonResponse({"result": "OK"}) @@ -678,16 +679,12 @@ def get(self, request, project_id): if project_id.startswith(settings.PORTAL_PROJECTS_SYSTEM_PREFIX): full_project_id = project_id else: - full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + full_project_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" try: tree = build_project_tree(full_project_id) except ProjectMetadata.DoesNotExist: - LOGGER.error( - f'Project metadata does not exist for project ID: {full_project_id}' - ) - return JsonResponse( - {'error': 'Project metadata does not exist'}, status=404 - ) + LOGGER.error(f"Project metadata does not exist for project ID: {full_project_id}") + return JsonResponse({"error": "Project metadata does not exist"}, status=404) - return JsonResponse({'tree': tree}) + return JsonResponse({"tree": tree}) diff --git a/server/portal/apps/projects/views_unit_test.py b/server/portal/apps/projects/views_unit_test.py index 90cddd890a..a90e5b87a9 100644 --- a/server/portal/apps/projects/views_unit_test.py +++ b/server/portal/apps/projects/views_unit_test.py @@ -32,9 +32,7 @@ def mock_project_mgr(mocker): @pytest.fixture() def mock_service_account(mocker): - return mocker.patch( - "portal.apps.projects.workspace_operations.shared_workspace_operations.service_account" - ) + return mocker.patch("portal.apps.projects.workspace_operations.shared_workspace_operations.service_account") @pytest.fixture @@ -114,7 +112,7 @@ def project_list(authenticated_user): def test_get_project_client_uses_service_account_for_project_admin(authenticated_user, mocker): group = Group.objects.create(name=settings.PROJECT_ADMIN_GROUP) authenticated_user.groups.add(group) - mock_service_account = mocker.patch('portal.apps.projects.views.service_account') + mock_service_account = mocker.patch("portal.apps.projects.views.service_account") client = get_project_client(authenticated_user) @@ -127,7 +125,7 @@ def test_get_project_for_user_allows_project_admin(authenticated_user): authenticated_user.groups.add(group) project = ProjectMetadata.objects.create( name=constants.PROJECT, - value={'projectId': f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123', 'users': []}, + value={"projectId": f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123", "users": []}, ) result = get_project_for_user(project.project_id, authenticated_user) @@ -138,26 +136,24 @@ def test_get_project_for_user_allows_project_admin(authenticated_user): def test_get_project_for_user_allows_tapis_write_role(authenticated_user, mocker): project = ProjectMetadata.objects.create( name=constants.PROJECT, - value={'projectId': f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123', 'users': []}, - ) - mock_get_workspace_role = mocker.patch( - 'portal.apps.projects.views.get_workspace_role', return_value='USER' + value={"projectId": f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123", "users": []}, ) + mock_get_workspace_role = mocker.patch("portal.apps.projects.views.get_workspace_role", return_value="USER") result = get_project_for_user(project.project_id, authenticated_user) assert result == project mock_get_workspace_role.assert_called_once_with( - authenticated_user.tapis_oauth.client, 'PRJ-123', authenticated_user.username + authenticated_user.tapis_oauth.client, "PRJ-123", authenticated_user.username ) def test_get_project_for_user_denies_tapis_guest_role(authenticated_user, mocker): project = ProjectMetadata.objects.create( name=constants.PROJECT, - value={'projectId': f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123', 'users': []}, + value={"projectId": f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.PRJ-123", "users": []}, ) - mocker.patch('portal.apps.projects.views.get_workspace_role', return_value='GUEST') + mocker.patch("portal.apps.projects.views.get_workspace_role", return_value="GUEST") with pytest.raises(ProjectMetadata.DoesNotExist): get_project_for_user(project.project_id, authenticated_user) @@ -185,12 +181,8 @@ def test_projects_get( } fields = "id,host,description,notes,updated,owner,rootDir" query = f"(id.like.{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.*)" - mock_tapis_client.systems.getSystems.assert_any_call( - listType="ALL", search=query, select=fields, limit=-1 - ) - mock_project_search_indexer.delay.assert_called_with( - [project_list["api_response"][0]] - ) + mock_tapis_client.systems.getSystems.assert_any_call(listType="ALL", search=query, select=fields, limit=-1) + mock_project_search_indexer.delay.assert_called_with([project_list["api_response"][0]]) def test_projects_search( @@ -216,9 +208,7 @@ def test_projects_search( "status": 200, "response": [project_list["api_response"][1]], } - mock_project_search_indexer.delay.assert_called_with( - [project_list["api_response"][1]] - ) + mock_project_search_indexer.delay.assert_called_with([project_list["api_response"][1]]) def test_projects_search_result_not_in_tapis( @@ -237,9 +227,7 @@ def test_projects_search_result_not_in_tapis( mock_project_search_indexer.delay.assert_called_with([]) -def test_projects_post( - authenticated_user, client, mock_service_account, mock_tapis_client -): +def test_projects_post(authenticated_user, client, mock_service_account, mock_tapis_client): response = client.post( "/api/projects/", @@ -263,9 +251,7 @@ def test_projects_post( mock_service_account().files.mkdir.assert_called_with( systemId="projects.system.name", path="test.project-2", - headers={ - "X-Tapis-Tracking-ID": f"portals.{sha256(client.session.session_key.encode()).hexdigest()}" - }, + headers={"X-Tapis-Tracking-ID": f"portals.{sha256(client.session.session_key.encode()).hexdigest()}"}, ) mock_service_account().files.setFacl.assert_called_with( systemId="projects.system.name", @@ -275,15 +261,11 @@ def test_projects_post( aclString=f"d:u:{authenticated_user.username}:rwX,u:{authenticated_user.username}:rwX", ) mock_tapis_client.systems.createSystem.assert_called() - assert mock_tapis_client.systems.createSystem.call_args_list[0].contains( - "test.project.test.project-2" - ) + assert mock_tapis_client.systems.createSystem.call_args_list[0].contains("test.project.test.project-2") @override_settings(PORTAL_PROJECTS_USE_SET_FACL_JOB=True) -def test_projects_post_setfacl_job( - authenticated_user, client, mock_service_account, mock_tapis_client -): +def test_projects_post_setfacl_job(authenticated_user, client, mock_service_account, mock_tapis_client): response = client.post( "/api/projects/", { @@ -305,9 +287,7 @@ def test_projects_post_setfacl_job( mock_service_account().files.mkdir.assert_called_with( systemId="projects.system.name", path="test.project-2", - headers={ - "X-Tapis-Tracking-ID": f"portals.{sha256(client.session.session_key.encode()).hexdigest()}" - }, + headers={"X-Tapis-Tracking-ID": f"portals.{sha256(client.session.session_key.encode()).hexdigest()}"}, ) mock_service_account().files.setFacl.assert_not_called() mock_service_account().jobs.submitJob.assert_called_with( @@ -332,18 +312,12 @@ def test_projects_post_setfacl_job( tags=["portalName:test"], ) mock_tapis_client.systems.createSystem.assert_called() - assert mock_tapis_client.systems.createSystem.call_args_list[0].contains( - "test.project.test.project-2" - ) + assert mock_tapis_client.systems.createSystem.call_args_list[0].contains("test.project.test.project-2") -def test_project_instance_get_by_id( - authenticated_user, client, mock_tapis_client, project_list -): +def test_project_instance_get_by_id(authenticated_user, client, mock_tapis_client, project_list): mock_tapis_client.systems.getSystem.return_value = project_list["tapis_response"][0] - mock_tapis_client.systems.getShareInfo.return_value = TapisResult( - **{"users": [authenticated_user.username]} - ) + mock_tapis_client.systems.getShareInfo.return_value = TapisResult(**{"users": [authenticated_user.username]}) response = client.get("/api/projects/PRJ-123/") assert response.status_code == 200 @@ -370,13 +344,9 @@ def test_project_instance_get_by_id( } -def test_project_instance_get_by_system( - authenticated_user, client, mock_tapis_client, project_list -): +def test_project_instance_get_by_system(authenticated_user, client, mock_tapis_client, project_list): mock_tapis_client.systems.getSystem.return_value = project_list["tapis_response"][0] - mock_tapis_client.systems.getShareInfo.return_value = TapisResult( - **{"users": [authenticated_user.username]} - ) + mock_tapis_client.systems.getShareInfo.return_value = TapisResult(**{"users": [authenticated_user.username]}) response = client.get("/api/projects/system/test.project.PRJ-123/") assert response.status_code == 200 @@ -404,16 +374,12 @@ def test_project_instance_get_by_system( } -def test_project_instance_patch( - authenticated_user, client, mock_tapis_client, project_list -): +def test_project_instance_patch(authenticated_user, client, mock_tapis_client, project_list): updated_project = project_list["tapis_response"][0] updated_project.notes.title = "New Title" updated_project.notes.description = "new description" mock_tapis_client.systems.getSystem.return_value = updated_project - mock_tapis_client.systems.getShareInfo.return_value = TapisResult( - **{"users": [authenticated_user.username]} - ) + mock_tapis_client.systems.getShareInfo.return_value = TapisResult(**{"users": [authenticated_user.username]}) response = client.patch( "/api/projects/PRJ-123/", @@ -454,9 +420,7 @@ def test_project_instance_patch( def test_project_change_role(client, mock_project_mgr, project_list): - mock_project_mgr.change_project_role.return_value = MagicMock( - metadata={"projectId": "PRJ-123"} - ) + mock_project_mgr.change_project_role.return_value = MagicMock(metadata={"projectId": "PRJ-123"}) patch_body = { "action": "change_project_role", @@ -467,16 +431,12 @@ def test_project_change_role(client, mock_project_mgr, project_list): response = client.patch("/api/projects/PRJ-123/members/", json.dumps(patch_body)) - mock_project_mgr.change_project_role.assert_called_with( - "PRJ-123", "test_user", "co_pi", "team_member" - ) + mock_project_mgr.change_project_role.assert_called_with("PRJ-123", "test_user", "co_pi", "team_member") assert response.status_code == 200 assert response.json() == {"status": 200, "response": {"projectId": "PRJ-123"}} -def test_project_change_system_role( - client, mock_service_account, mock_tapis_client, project_list -): +def test_project_change_system_role(client, mock_service_account, mock_tapis_client, project_list): # USER translates to writer role patch_body = { "action": "change_system_role", @@ -510,9 +470,7 @@ def test_project_change_system_role( @override_settings(PORTAL_PROJECTS_USE_SET_FACL_JOB=True) -def test_project_change_system_role_setfacl_job( - client, mock_service_account, mock_tapis_client, project_list -): +def test_project_change_system_role_setfacl_job(client, mock_service_account, mock_tapis_client, project_list): mock_rootDir = mock_tapis_client.systems.getSystem().rootDir # USER translates to writer role @@ -559,16 +517,12 @@ def test_project_change_system_role_setfacl_job( ) -def test_members_view_add( - authenticated_user, client, mock_tapis_client, project_list, mock_service_account -): +def test_members_view_add(authenticated_user, client, mock_tapis_client, project_list, mock_service_account): mock_tapis_client.systems.getSystem.return_value = project_list["tapis_response"][0] mock_tapis_client.systems.getShareInfo.return_value = TapisResult( **{"users": [authenticated_user.username, "test_user"]} ) - mock_tapis_client.files.getPermissions.return_value = TapisResult( - **{"permission": "MODIFY"} - ) + mock_tapis_client.files.getPermissions.return_value = TapisResult(**{"permission": "MODIFY"}) patch_body = {"action": "add_member", "username": "test_user"} @@ -616,9 +570,7 @@ def test_members_view_add( recursionMethod="PHYSICAL", aclString="d:u:test_user:rwX,u:test_user:rwX", ) - mock_tapis_client.systems.shareSystem.assert_called_with( - systemId="test.project.PRJ-123", users=["test_user"] - ) + mock_tapis_client.systems.shareSystem.assert_called_with(systemId="test.project.PRJ-123", users=["test_user"]) mock_tapis_client.systems.grantUserPerms.assert_called_with( systemId="test.project.PRJ-123", userName="test_user", @@ -640,9 +592,7 @@ def test_members_view_add_setfacl_job( mock_tapis_client.systems.getShareInfo.return_value = TapisResult( **{"users": [authenticated_user.username, "test_user"]} ) - mock_tapis_client.files.getPermissions.return_value = TapisResult( - **{"permission": "MODIFY"} - ) + mock_tapis_client.files.getPermissions.return_value = TapisResult(**{"permission": "MODIFY"}) patch_body = {"action": "add_member", "username": "test_user"} @@ -701,9 +651,7 @@ def test_members_view_add_setfacl_job( }, tags=["portalName:test"], ) - mock_tapis_client.systems.shareSystem.assert_called_with( - systemId="test.project.PRJ-123", users=["test_user"] - ) + mock_tapis_client.systems.shareSystem.assert_called_with(systemId="test.project.PRJ-123", users=["test_user"]) mock_tapis_client.systems.grantUserPerms.assert_called_with( systemId="test.project.PRJ-123", userName="test_user", @@ -717,9 +665,7 @@ def test_members_view_add_setfacl_job( ) -def test_members_view_remove( - client, mock_service_account, mock_tapis_client, project_list -): +def test_members_view_remove(client, mock_service_account, mock_tapis_client, project_list): mock_tapis_client.systems.getSystem.return_value = project_list["tapis_response"][0] patch_body = {"action": "remove_member", "username": "test_user"} @@ -753,9 +699,7 @@ def test_members_view_remove( recursionMethod="PHYSICAL", aclString="d:u:test_user,u:test_user", ) - mock_tapis_client.systems.unShareSystem.assert_called_with( - systemId="test.project.PRJ-123", users=["test_user"] - ) + mock_tapis_client.systems.unShareSystem.assert_called_with(systemId="test.project.PRJ-123", users=["test_user"]) mock_tapis_client.systems.revokeUserPerms.assert_called_with( systemId="test.project.PRJ-123", userName="test_user", @@ -767,9 +711,7 @@ def test_members_view_remove( @override_settings(PORTAL_PROJECTS_USE_SET_FACL_JOB=True) -def test_members_view_remove_setfacl_job( - client, mock_service_account, mock_tapis_client, project_list -): +def test_members_view_remove_setfacl_job(client, mock_service_account, mock_tapis_client, project_list): mock_tapis_client.systems.getSystem.return_value = project_list["tapis_response"][0] patch_body = {"action": "remove_member", "username": "test_user"} @@ -815,9 +757,7 @@ def test_members_view_remove_setfacl_job( }, tags=["portalName:test"], ) - mock_tapis_client.systems.unShareSystem.assert_called_with( - systemId="test.project.PRJ-123", users=["test_user"] - ) + mock_tapis_client.systems.unShareSystem.assert_called_with(systemId="test.project.PRJ-123", users=["test_user"]) mock_tapis_client.systems.revokeUserPerms.assert_called_with( systemId="test.project.PRJ-123", userName="test_user", diff --git a/server/portal/apps/projects/workspace_operations/datacite_operations.py b/server/portal/apps/projects/workspace_operations/datacite_operations.py index 3255ea8595..4435d632f9 100644 --- a/server/portal/apps/projects/workspace_operations/datacite_operations.py +++ b/server/portal/apps/projects/workspace_operations/datacite_operations.py @@ -108,9 +108,7 @@ def get_datacite_json(pub_graph: nx.DiGraph): identifier = {} if {"publicationLink"} <= r_data.keys(): publication_type = r_data.get("publicationType", None) - identifier["relationType"] = relation_mapping.get( - publication_type, "References" - ) + identifier["relationType"] = relation_mapping.get(publication_type, "References") identifier["relatedIdentifier"] = r_data["publicationLink"] identifier["relatedIdentifierType"] = "URL" if "publicationDoi" in r_data: @@ -132,9 +130,7 @@ def upsert_datacite_json(datacite_json: dict, doi: Optional[str] = None): datacite_payload = { "data": { "type": "dois", - "relationships": { - "client": {"data": {"type": "clients", "id": settings.DATACITE_USER}} - }, + "relationships": {"client": {"data": {"type": "clients", "id": settings.DATACITE_USER}}}, "attributes": datacite_json, } } diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index 21c36b9a2d..a9a2ac5114 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -22,9 +22,7 @@ def _add_node_to_graph( raise nx.exception.NodeNotFound # no-op if metadata with this UUID is already associated. - if meta_uuid in ( - graph.nodes[node]["uuid"] for node in graph.successors(parent_node_id) - ): + if meta_uuid in (graph.nodes[node]["uuid"] for node in graph.successors(parent_node_id)): return (graph, None) _graph: nx.DiGraph = copy.deepcopy(graph) @@ -54,15 +52,13 @@ def initialize_project_graph(project_id: str): "name": project_model.name, "projectType": project_type, "order": 0, - "label": project_model.value.get("title") + "label": project_model.value.get("title"), } if project_type == "other": # type Other projects have a "null" parent node above the project root, to # support multiple versions. - project_graph.add_node( - root_node_id, **{"uuid": None, "name": None, "projectType": "other"} - ) + project_graph.add_node(root_node_id, **{"uuid": None, "name": None, "projectType": "other"}) base_node_id = f"NODE_project_{uuid.uuid4()}" project_graph.add_node(base_node_id, **base_node_data) project_graph.add_edge(root_node_id, base_node_id) @@ -83,7 +79,7 @@ def traverse_graph(project_graph, root_node, path_components): for component in path_components: found = False for successor in project_graph.successors(current_node): - name = project_graph.nodes[successor]['label'] + name = project_graph.nodes[successor]["label"] if name == component: current_node = successor found = True @@ -96,9 +92,7 @@ def traverse_graph(project_graph, root_node, path_components): def get_node_from_path(project_id: str, path: str) -> Dict[str, Any]: """Return the node ID for the parent of a node with the given path.""" - graph_model = ProjectMetadata.objects.get( - name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id - ) + graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) project_graph = nx.node_link_graph(graph_model.value) path_parts = path.strip("/").split("/") @@ -113,9 +107,7 @@ def get_node_from_path(project_id: str, path: str) -> Dict[str, Any]: def get_root_node(project_id: str) -> Dict[str, Any]: """Return the root node for a project graph.""" - graph_model = ProjectMetadata.objects.get( - name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id - ) + graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) project_graph = nx.node_link_graph(graph_model.value) return {"id": "NODE_ROOT", **project_graph.nodes["NODE_ROOT"]} @@ -136,9 +128,7 @@ def update_node_in_project(project_id: str, node_id: str, new_parent: str = None parent_node = new_parent if not project_graph.has_node(parent_node): raise nx.exception.NodeNotFound - project_graph.remove_edge( - next(project_graph.predecessors(node_id)), node_id - ) + project_graph.remove_edge(next(project_graph.predecessors(node_id)), node_id) project_graph.add_edge(parent_node, node_id) if new_name: @@ -194,9 +184,7 @@ def add_node_to_project(project_id: str, parent_node: str, meta_uuid: str, name: ) project_graph = nx.node_link_graph(graph_model.value) - (updated_graph, new_node_id) = _add_node_to_graph( - project_graph, parent_node, meta_uuid, name, label - ) + (updated_graph, new_node_id) = _add_node_to_graph(project_graph, parent_node, meta_uuid, name, label) graph_model.value = nx.node_link_data(updated_graph) graph_model.save() @@ -205,9 +193,7 @@ def add_node_to_project(project_id: str, parent_node: str, meta_uuid: str, name: def get_node_from_uuid(project_id: str, uuid: str): """Get a node from the project graph using its UUID.""" - graph_model = ProjectMetadata.objects.get( - name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id - ) + graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) project_graph = nx.node_link_graph(graph_model.value) for node_id in project_graph.nodes: @@ -251,30 +237,28 @@ def build_project_tree(full_project_id: str): node = graph.nodes[node_id] # Build the node's path from the labels of its ancestors (excluding root). - if nx.has_path(graph, 'NODE_ROOT', node_id): - path_nodes = nx.shortest_path(graph, 'NODE_ROOT', node_id)[1:] - node['path'] = '/'.join( - graph.nodes[parent]['label'] - for parent in path_nodes - if 'label' in graph.nodes[parent] + if nx.has_path(graph, "NODE_ROOT", node_id): + path_nodes = nx.shortest_path(graph, "NODE_ROOT", node_id)[1:] + node["path"] = "/".join( + graph.nodes[parent]["label"] for parent in path_nodes if "label" in graph.nodes[parent] ) else: - node['path'] = "" + node["path"] = "" - if node.get('value'): - metadata = get_ordered_value(node['name'], node['value']) - file_objs = node['value'].get('fileObjs', []) + if node.get("value"): + metadata = get_ordered_value(node["name"], node["value"]) + file_objs = node["value"].get("fileObjs", []) else: - entity = ProjectMetadata.objects.get(uuid=node.get('uuid')) + entity = ProjectMetadata.objects.get(uuid=node.get("uuid")) metadata = get_ordered_value(entity.name, entity.value) - file_objs = entity.value.get('fileObjs', []) + file_objs = entity.value.get("fileObjs", []) - node['metadata'] = metadata - node['fileObjs'] = [ + node["metadata"] = metadata + node["fileObjs"] = [ { **file_obj, - 'id': file_obj.get('uuid'), - 'metadata': get_ordered_value(constants.FILE, file_obj.get('value')), + "id": file_obj.get("uuid"), + "metadata": get_ordered_value(constants.FILE, file_obj.get("value")), } for file_obj in file_objs ] @@ -284,13 +268,13 @@ def build_project_tree(full_project_id: str): def get_path_uuid_mapping(project_id: str): """Return a mapping of node paths to UUIDs for a project graph.""" - graph_model = ProjectMetadata.objects.get( - name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id - ) + graph_model = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) project_graph = nx.node_link_graph(graph_model.value) path_uuid_mapping = {} for node_id in project_graph.nodes: - path_nodes = nx.shortest_path(project_graph, 'NODE_ROOT', node_id)[1:] - path = '/'.join(project_graph.nodes[parent]['label'] for parent in path_nodes if 'label' in project_graph.nodes[parent]) + path_nodes = nx.shortest_path(project_graph, "NODE_ROOT", node_id)[1:] + path = "/".join( + project_graph.nodes[parent]["label"] for parent in path_nodes if "label" in project_graph.nodes[parent] + ) path_uuid_mapping[path] = project_graph.nodes[node_id]["uuid"] return path_uuid_mapping diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index 47a241b321..16e040f217 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -10,14 +10,19 @@ from portal.apps.projects.schema_models import constants from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps.projects.schema_models.base_metadata import PartialEntityWithFiles, FileObj -from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, get_node_from_uuid, get_root_node, update_node_in_project +from portal.apps.projects.workspace_operations.graph_operations import ( + get_node_from_path, + get_node_from_uuid, + get_root_node, + update_node_in_project, +) portal = settings.PORTAL_NAMESPACE.lower() def snake_to_camel(snake_str): - components = snake_str.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) + components = snake_str.split("_") + return components[0] + "".join(x.title() for x in components[1:]) def create_project_metadata(value): @@ -25,9 +30,7 @@ def create_project_metadata(value): schema_model = SCHEMA_MAPPING[constants.PROJECT] validated_model = schema_model.model_validate(value) - project_db_model = ProjectMetadata( - name=constants.PROJECT, value=validated_model.model_dump() - ) + project_db_model = ProjectMetadata(name=constants.PROJECT, value=validated_model.model_dump()) project_db_model.save() return project_db_model @@ -50,15 +53,15 @@ def get_value(project_id, path): try: node = get_node_from_path(project_id, path) - if not node or node['id'] == 'NODE_ROOT': + if not node or node["id"] == "NODE_ROOT": return None - if node.get('value'): - return get_ordered_value(node['name'], node['value']) + if node.get("value"): + return get_ordered_value(node["name"], node["value"]) - entity = ProjectMetadata.objects.get(uuid=node['uuid']) + entity = ProjectMetadata.objects.get(uuid=node["uuid"]) value = get_ordered_value(entity.name, entity.value) - value['uuid'] = node['uuid'] + value["uuid"] = node["uuid"] return value except ProjectMetadata.DoesNotExist: @@ -67,9 +70,9 @@ def get_value(project_id, path): def get_ordered_value(name, value): """ - Return the metadata in the order defined in the Pydantic model. - Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be updated to use camelCase. - """ + Return the metadata in the order defined in the Pydantic model. + Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be updated to use camelCase. + """ schema = SCHEMA_MAPPING.get(name) if not schema: @@ -89,15 +92,19 @@ def get_ordered_value(name, value): # if the fiels is a list, then we need to get the model of the list and process it if isinstance(field_value, list): field_annotation = schema.model_fields[field].annotation - item_type = get_args(field_annotation)[0] if get_args(field_annotation) else None # returns the model class of the list + item_type = ( + get_args(field_annotation)[0] if get_args(field_annotation) else None + ) # returns the model class of the list # Check if the item type is a Pydantic model if item_type and hasattr(item_type, "model_fields"): # Re-order each item in the list if it's a list of Pydantic models ordered_value[field] = [ - {k: item.get(snake_to_camel(k)) - for k in item_type.model_fields.keys() - if item.get(snake_to_camel(k)) is not None} + { + k: item.get(snake_to_camel(k)) + for k in item_type.model_fields.keys() + if item.get(snake_to_camel(k)) is not None + } for item in field_value ] else: @@ -113,10 +120,10 @@ def get_entity(project_id, path): try: node = get_node_from_path(project_id, path) - if not node or node['id'] == 'NODE_ROOT': + if not node or node["id"] == "NODE_ROOT": return None - return ProjectMetadata.objects.get(uuid=node['uuid']) + return ProjectMetadata.objects.get(uuid=node["uuid"]) except ProjectMetadata.DoesNotExist: return None @@ -130,10 +137,10 @@ def create_file_obj(project_id, name, size, path, value): system=project_id, name=name, path=path, - type='file', + type="file", length=size, value=validated_model.model_dump(exclude_none=True), - uuid=str(uuid.uuid4()) + uuid=str(uuid.uuid4()), ) return file_obj @@ -146,13 +153,13 @@ def get_file_obj(project_id, path): parent_node = _get_valid_node(project_id, parent_path) - if (parent_node.get('value')): - file_objs = parent_node['value'].get('fileObjs', []) + if parent_node.get("value"): + file_objs = parent_node["value"].get("fileObjs", []) else: - parent_entity = ProjectMetadata.objects.get(uuid=parent_node['uuid']) - file_objs = parent_entity.value.get('fileObjs', []) + parent_entity = ProjectMetadata.objects.get(uuid=parent_node["uuid"]) + file_objs = parent_entity.value.get("fileObjs", []) - file_obj = next((f for f in file_objs if f['path'] == path), None) + file_obj = next((f for f in file_objs if f["path"] == path), None) if file_obj: return file_obj else: @@ -172,7 +179,7 @@ def patch_project_entity(project_id, value): entity_value = get_ordered_value(entity.name, entity.value) patched_metadata = {**entity_value, **value} - update_node_in_project(project_id, 'NODE_ROOT', None, value.get('title')) + update_node_in_project(project_id, "NODE_ROOT", None, value.get("title")) validated_model = schema_model.model_validate(patched_metadata) entity.value = validated_model.model_dump(exclude_none=True) @@ -188,16 +195,16 @@ def patch_file_obj_entity(client, project_id, value, path): parent_node = _get_valid_node(project_id, parent_path) - entity = ProjectMetadata.objects.get(uuid=parent_node['uuid']) - file_objs = entity.value.get('fileObjs', []) - file_obj = next((f for f in file_objs if f['path'] == path.strip('/')), None) + entity = ProjectMetadata.objects.get(uuid=parent_node["uuid"]) + file_objs = entity.value.get("fileObjs", []) + file_obj = next((f for f in file_objs if f["path"] == path.strip("/")), None) if not file_obj: return None schema = SCHEMA_MAPPING[constants.FILE] validated_model = schema.model_validate(value) - file_obj['value'] = validated_model.model_dump(exclude_none=True) + file_obj["value"] = validated_model.model_dump(exclude_none=True) entity_file_model = PartialEntityWithFiles.model_validate(entity.value) merged_file_objs = _merge_file_objs(entity_file_model.file_objs, [FileObj(**file_obj)]) @@ -212,36 +219,36 @@ def patch_file_obj_entity(client, project_id, value, path): def patch_entity_and_node(project_id, value, path, new_path, new_name, uuid=None): """Perform an operation on an entity.""" - new_path_full = os.path.join(new_path.strip('/'), new_name) + new_path_full = os.path.join(new_path.strip("/"), new_name) - if (path): + if path: source_node = get_node_from_path(project_id, path) - elif (not path and uuid): + elif not path and uuid: source_node = get_node_from_uuid(project_id, uuid) else: raise ValueError("Invalid parameters: path or uuid must be provided.") - entity = ProjectMetadata.objects.get(uuid=uuid if uuid else source_node['uuid']) + entity = ProjectMetadata.objects.get(uuid=uuid if uuid else source_node["uuid"]) new_parent_node = get_node_from_path(project_id, new_path) - update_node_in_project(project_id, source_node['id'], new_parent_node['id'], new_name) + update_node_in_project(project_id, source_node["id"], new_parent_node["id"], new_name) schema_model = SCHEMA_MAPPING[entity.name] - file_objs = entity.value.get('fileObjs', []) + file_objs = entity.value.get("fileObjs", []) updated_file_objs = update_file_paths(file_objs, path, new_path_full) - if value.get('file_objs') is not None: - value.pop('file_objs') + if value.get("file_objs") is not None: + value.pop("file_objs") - patched_metadata = {**value, 'fileObjs': updated_file_objs} + patched_metadata = {**value, "fileObjs": updated_file_objs} validated_model = schema_model.model_validate(patched_metadata) entity.value = validated_model.model_dump(exclude_none=True) entity.save() - update_children_file_paths(project_id, source_node['id'], path, new_path_full) + update_children_file_paths(project_id, source_node["id"], path, new_path_full) return entity @@ -254,24 +261,24 @@ def patch_file_association(project_id, value, source_path_full, dest_path_full, source_node = _get_valid_node(project_id, source_parent_path) dest_node = _get_valid_node(project_id, dest_parent_path) - source_entity = ProjectMetadata.objects.get(uuid=source_node['uuid']) - dest_entity = ProjectMetadata.objects.get(uuid=dest_node['uuid']) + source_entity = ProjectMetadata.objects.get(uuid=source_node["uuid"]) + dest_entity = ProjectMetadata.objects.get(uuid=dest_node["uuid"]) - file_obj_dict = next( - (f for f in source_entity.value.get('fileObjs', []) if f['path'] == source_path_full), None - ) + file_obj_dict = next((f for f in source_entity.value.get("fileObjs", []) if f["path"] == source_path_full), None) if not file_obj_dict: return - if operation == 'move': - file_obj_dict['name'] = new_name - file_obj_dict['path'] = dest_path_full.strip('/') + if operation == "move": + file_obj_dict["name"] = new_name + file_obj_dict["path"] = dest_path_full.strip("/") file_obj = FileObj(**file_obj_dict) remove_file_associations(source_entity.uuid, [source_path_full]) add_file_associations(dest_entity.uuid, [file_obj]) - elif operation == 'copy': - file_obj = create_file_obj(project_id, new_name, file_obj_dict['length'], dest_path_full.strip('/'), file_obj_dict['value']) + elif operation == "copy": + file_obj = create_file_obj( + project_id, new_name, file_obj_dict["length"], dest_path_full.strip("/"), file_obj_dict["value"] + ) add_file_associations(dest_entity.uuid, [file_obj]) @@ -291,20 +298,20 @@ def move_entity(client, project_id, current_path, new_path, value, uuid=None): node = get_node_from_path(project_id, current_path) if current_path else get_root_node(project_id) - entity = ProjectMetadata.objects.get(uuid=uuid if uuid else node['uuid']) + entity = ProjectMetadata.objects.get(uuid=uuid if uuid else node["uuid"]) - current_name = entity.value.get('name') + current_name = entity.value.get("name") current_path_full = current_path - new_name = value.get('name') + new_name = value.get("name") new_path = new_path if all([current_name, new_name, new_path, current_path_full]) and ( current_name != new_name or new_path != str(Path(current_path_full).parent) ): move_result = move(client, project_id, current_path, project_id, new_path, new_name) - move_message = move_result['message'].split('DestinationPath: ', 1)[1] - new_name = ('/' + move_message).rsplit('/', 1)[1] + move_message = move_result["message"].split("DestinationPath: ", 1)[1] + new_name = ("/" + move_message).rsplit("/", 1)[1] return new_name @@ -321,21 +328,15 @@ def clear_entities(project_id): return "OK" -def _merge_file_objs( - prev_file_objs: list[FileObj], new_file_objs: list[FileObj] -) -> list[FileObj]: +def _merge_file_objs(prev_file_objs: list[FileObj], new_file_objs: list[FileObj]) -> list[FileObj]: """Combine two arrays of FileObj models, overwriting the first if there are conflicts.""" new_file_paths = [f.path for f in new_file_objs] deduped_file_objs = [fo for fo in prev_file_objs if fo.path not in new_file_paths] - return sorted( - [*deduped_file_objs, *new_file_objs], key=operator.attrgetter("name", "path") - ) + return sorted([*deduped_file_objs, *new_file_objs], key=operator.attrgetter("name", "path")) -def _filter_file_objs( - prev_file_objs: list[FileObj], paths_to_remove: list[str] -) -> list[FileObj]: +def _filter_file_objs(prev_file_objs: list[FileObj], paths_to_remove: list[str]) -> list[FileObj]: return sorted( [fo for fo in prev_file_objs if fo.path not in paths_to_remove], key=operator.attrgetter("name", "path"), @@ -384,75 +385,77 @@ def remove_file_obj_by_path(project_id, path): Mirrors get_file_obj's parent resolution. No-op if the parent isn't in the graph. """ parent_node = _get_valid_node(project_id, str(Path(path).parent)) - if parent_node and parent_node.get('uuid'): - remove_file_associations(parent_node['uuid'], [path]) + if parent_node and parent_node.get("uuid"): + remove_file_associations(parent_node["uuid"], [path]) def create_file_entity(project_id: str, value: dict, uploaded_file, path: str): - new_meta = create_entity_metadata(project_id, getattr(constants, value.get('data_type').upper()), { - **value, - }) + new_meta = create_entity_metadata( + project_id, + getattr(constants, value.get("data_type").upper()), + { + **value, + }, + ) parent_node = get_node_from_path(project_id, path) file_obj = FileObj( system=project_id, name=uploaded_file.name, - path=f'{path.strip("/")}/{uploaded_file.name}', - type='file', + path=f"{path.strip('/')}/{uploaded_file.name}", + type="file", length=uploaded_file.size, - uuid=new_meta.uuid + uuid=new_meta.uuid, ) - if parent_node and parent_node['id'] != 'NODE_ROOT': - add_file_associations(parent_node['uuid'], [file_obj]) + if parent_node and parent_node["id"] != "NODE_ROOT": + add_file_associations(parent_node["uuid"], [file_obj]) else: # Add file association to root node if no parent node/entity exists root_node = get_root_node(project_id) - add_file_associations(root_node['uuid'], [file_obj]) + add_file_associations(root_node["uuid"], [file_obj]) def _get_valid_node(project_id, path): node = get_node_from_path(project_id, path) - return node if node and node['id'] != 'NODE_ROOT' else get_root_node(project_id) + return node if node and node["id"] != "NODE_ROOT" else get_root_node(project_id) def update_file_paths(file_objs, old_path, new_path): """Update paths for a list of file objects.""" updated_file_objs = [] for file_obj in file_objs: - file_obj['path'] = _update_file_obj_path(file_obj, old_path, new_path) + file_obj["path"] = _update_file_obj_path(file_obj, old_path, new_path) updated_file_objs.append(file_obj) return updated_file_objs def _update_file_obj_path(file_obj, old_path, new_path): - new_file_obj_path = file_obj['path'].replace(old_path.strip('/'), new_path.strip('/'), 1) + new_file_obj_path = file_obj["path"].replace(old_path.strip("/"), new_path.strip("/"), 1) return new_file_obj_path def update_children_file_paths(project_id, source_node_id, old_path, new_path): """Update paths for all descendant file objects.""" - graph = ProjectMetadata.objects.get( - name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id - ) + graph = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id) graph_value = nx.node_link_graph(graph.value) children = list(nx.descendants(graph_value, source_node_id)) for child in children: - child_uuid = graph_value.nodes[child]['uuid'] + child_uuid = graph_value.nodes[child]["uuid"] child_enity = ProjectMetadata.objects.get(uuid=child_uuid) - child_file_objs = child_enity.value.get('fileObjs', []) + child_file_objs = child_enity.value.get("fileObjs", []) updated_child_file_objs = update_file_paths(child_file_objs, old_path, new_path) - child_enity.value['fileObjs'] = updated_child_file_objs + child_enity.value["fileObjs"] = updated_child_file_objs child_enity.save() return "OK" diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index 7182410f47..f49575a043 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -10,7 +10,11 @@ from portal.apps.projects.schema_models import constants from portal.libs.agave.utils import user_account, service_account from portal.apps.publications.models import Publication, PublicationRequest -from portal.apps.projects.workspace_operations.datacite_operations import get_datacite_json, upsert_datacite_json, publish_datacite_doi +from portal.apps.projects.workspace_operations.datacite_operations import ( + get_datacite_json, + upsert_datacite_json, + publish_datacite_doi, +) from django.db import transaction from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes from portal.apps.search.tasks import index_publication @@ -24,17 +28,13 @@ def _transfer_files(client, source_system_id, dest_system_id): service_client = service_account() - source_system_files = client.files.listFiles(systemId=source_system_id, path='/') + source_system_files = client.files.listFiles(systemId=source_system_id, path="/") # Filter out the trash folder filtered_files = [file for file in source_system_files if file.name != settings.TAPIS_DEFAULT_TRASH_NAME] transfer_elements = [ - { - 'sourceURI': file.url, - 'destinationURI': f'tapis://{dest_system_id}/{file.path}' - } - for file in filtered_files + {"sourceURI": file.url, "destinationURI": f"tapis://{dest_system_id}/{file.path}"} for file in filtered_files ] transfer = service_client.files.createTransferTask(elements=transfer_elements) @@ -44,7 +44,7 @@ def _transfer_files(client, source_system_id, dest_system_id): def _transfer_cover_image(source_system_id, dest_system_id, cover_image_path): if not cover_image_path: - logger.info('No cover image found for project, skipping transfer.') + logger.info("No cover image found for project, skipping transfer.") return None service_client = service_account() @@ -52,8 +52,8 @@ def _transfer_cover_image(source_system_id, dest_system_id, cover_image_path): # Transfer the cover image to the destination system transfer_elements = [ { - 'sourceURI': f'tapis://{source_system_id}/{cover_image_path}', - 'destinationURI': f'tapis://{dest_system_id}/{cover_image_path}' + "sourceURI": f"tapis://{source_system_id}/{cover_image_path}", + "destinationURI": f"tapis://{dest_system_id}/{cover_image_path}", } ] @@ -97,10 +97,16 @@ def publish_project_callback(review_project_id, published_project_id, archive_pr archive_publication_files(archive_project_id) -def publication_request_callback(user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id): +def publication_request_callback( + user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id +): service_client = service_account() - publication_reviewers = get_user_model().objects.filter(groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME).values_list('username', flat=True) + publication_reviewers = ( + get_user_model() + .objects.filter(groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME) + .values_list("username", flat=True) + ) with transaction.atomic(): # Commented out cleanup to prevent breaking admin role functionality @@ -121,7 +127,7 @@ def publication_request_callback(user_access_token, source_workspace_id, review_ f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{review_workspace_id}", settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, ) - logger.info(f'Added reviewer {reviewer} to review system {review_system_id}') + logger.info(f"Added reviewer {reviewer} to review system {review_system_id}") if not settings.DEBUG: send_publication_in_review_email_to_authors.apply_async(args=[source_system_id]) @@ -165,14 +171,8 @@ def archive_publication_files(project_id: str): "appArgs": [], "schedulerOptions": [], "envVariables": [ - { - "key": "publishedRootDir", - "value": published_root_dir - }, - { - "key": "projectId", - "value": project_id - }, + {"key": "publishedRootDir", "value": published_root_dir}, + {"key": "projectId", "value": project_id}, { "key": "ranchSystemId", "value": settings.PORTAL_PUBLICATION_RANCH_SYSTEM_ID, @@ -180,7 +180,7 @@ def archive_publication_files(project_id: str): { "key": "ranchArchiveRootDir", "value": "/", - } + }, ], }, "tags": [f"portalName:{settings.PORTAL_NAMESPACE.lower()}"], @@ -189,18 +189,17 @@ def archive_publication_files(project_id: str): return res -@shared_task(bind=True, max_retries=3, queue='default') +@shared_task(bind=True, max_retries=3, queue="default") def publish_project(self, project_id: str, version: Optional[int] = 1): review_system_prefix = settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX published_system_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX published_workspace_id = f"{project_id}{f'v{version}' if version and version > 1 else ''}" - published_system_id = f'{published_system_prefix}.{published_workspace_id}' - review_system_id = f'{review_system_prefix}.{project_id}' + published_system_id = f"{published_system_prefix}.{published_workspace_id}" + review_system_id = f"{review_system_prefix}.{project_id}" with transaction.atomic(): - project_meta = ProjectMetadata.get_project_by_id(review_system_id) publication_tree: nx.DiGraph = nx.node_link_graph(project_meta.project_graph.value) @@ -214,32 +213,32 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): value=nx.node_link_data(publication_tree), ) - source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + source_project_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" source_project = ProjectMetadata.get_project_by_id(source_project_id) try: # Mint a DataCite DOI existing_doi = source_project.value.get("doi", None) - logger.info(f'Attempting to mint DataCite DOI for project {project_id}, existing DOI: {existing_doi}') + logger.info(f"Attempting to mint DataCite DOI for project {project_id}, existing DOI: {existing_doi}") datacite_json = get_datacite_json(publication_tree) datacite_resp = upsert_datacite_json(datacite_json, doi=existing_doi) doi = datacite_resp["data"]["id"] - logger.info(f'Successfully minted DataCite DOI for project {project_id}: {doi}') + logger.info(f"Successfully minted DataCite DOI for project {project_id}: {doi}") except Exception as e: - logger.error(f'Error minting DataCite DOI for project {project_id}: {e}') - raise Exception(f'Error minting DOI for project {project_id}: {e}') + logger.error(f"Error minting DataCite DOI for project {project_id}: {e}") + raise Exception(f"Error minting DOI for project {project_id}: {e}") # Update project metadata with datacite doi - source_project.value['doi'] = doi - source_project.value['publicationDate'] = published_project.created + source_project.value["doi"] = doi + source_project.value["publicationDate"] = published_project.created source_project.save() pub_tree = nx.node_link_graph(published_project.project_graph.value) pub_tree.nodes["NODE_ROOT"]["version"] = version published_project.project_graph.value = nx.node_link_data(pub_tree) - published_project.value['doi'] = doi - published_project.value['publicationDate'] = published_project.created + published_project.value["doi"] = doi + published_project.value["publicationDate"] = published_project.created published_project.save() pub_metadata, _ = Publication.objects.update_or_create( @@ -251,8 +250,8 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): try: publish_datacite_doi(doi) except Exception as e: - logger.error(f'Error publishing DataCite DOI for project {project_id}: {e}') - raise Exception(f'Error publishing DOI for project {project_id}: {e}') + logger.error(f"Error publishing DataCite DOI for project {project_id}: {e}") + raise Exception(f"Error publishing DOI for project {project_id}: {e}") upload_metadata_file(published_workspace_id, pub_metadata.tree) @@ -264,24 +263,29 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): _transfer_cover_image( settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME, - project_meta.value.get("coverImage", None)) + project_meta.value.get("coverImage", None), + ) poll_tapis_file_transfer.apply_async( args=(transfer.uuid, False), kwargs={ - 'review_project_id': review_system_id, - 'published_project_id': published_system_id, - 'archive_project_id': published_workspace_id - }, countdown=30) + "review_project_id": review_system_id, + "published_project_id": published_system_id, + "archive_project_id": published_workspace_id, + }, + countdown=30, + ) if not settings.DEBUG: send_publication_accepted_email_to_authors.apply_async(args=[project_id]) - send_publication_reviewed_email_to_reviewers.apply_async(args=[project_id, 'APPROVED', None]) + send_publication_reviewed_email_to_reviewers.apply_async(args=[project_id, "APPROVED", None]) -@shared_task(bind=True, max_retries=3, queue='default') -def copy_graph_and_files_for_review_system(self, user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id): - logger.info(f'Starting copy task for system {source_system_id} to system {review_system_id}') +@shared_task(bind=True, max_retries=3, queue="default") +def copy_graph_and_files_for_review_system( + self, user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id +): + logger.info(f"Starting copy task for system {source_system_id} to system {review_system_id}") with transaction.atomic(): pub_tree = _add_values_to_tree(source_system_id) @@ -302,24 +306,27 @@ def copy_graph_and_files_for_review_system(self, user_access_token, source_works _transfer_cover_image( settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, - review_project.value.get("coverImage", None)) + review_project.value.get("coverImage", None), + ) - logger.info(f'Transfer task submmited with id {transfer.uuid}') + logger.info(f"Transfer task submmited with id {transfer.uuid}") poll_tapis_file_transfer.apply_async( args=(transfer.uuid, True), kwargs={ - 'user_access_token': user_access_token, - 'source_workspace_id': source_workspace_id, - 'review_workspace_id': review_workspace_id, - 'source_system_id': source_system_id, - 'review_system_id': review_system_id, - }, countdown=30) + "user_access_token": user_access_token, + "source_workspace_id": source_workspace_id, + "review_workspace_id": review_workspace_id, + "source_system_id": source_system_id, + "review_system_id": review_system_id, + }, + countdown=30, + ) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def poll_tapis_file_transfer(self, transfer_task_id, is_review, **kwargs): - logger.info(f'Starting post transfer task for transfer id {transfer_task_id} with arguments: {kwargs}') + logger.info(f"Starting post transfer task for transfer id {transfer_task_id} with arguments: {kwargs}") try: service_client = service_account() @@ -328,14 +335,16 @@ def poll_tapis_file_transfer(self, transfer_task_id, is_review, **kwargs): transfer_status = _check_transfer_status(service_client, transfer_task_id) # Handle pending or in-progress transfer - if transfer_status in ['PENDING', 'IN_PROGRESS']: - logger.info(f'Transfer {transfer_task_id} is still pending with status {transfer_status}, retrying in 30 seconds.') + if transfer_status in ["PENDING", "IN_PROGRESS"]: + logger.info( + f"Transfer {transfer_task_id} is still pending with status {transfer_status}, retrying in 30 seconds." + ) self.apply_async(args=(transfer_task_id, is_review), kwargs=kwargs, countdown=30) return # Handle completed transfer - elif transfer_status == 'COMPLETED': - logger.info(f'Transfer {transfer_task_id} completed successfully with arguments: {kwargs}') + elif transfer_status == "COMPLETED": + logger.info(f"Transfer {transfer_task_id} completed successfully with arguments: {kwargs}") # Call the callback function with any passed arguments if is_review: @@ -344,11 +353,11 @@ def poll_tapis_file_transfer(self, transfer_task_id, is_review, **kwargs): publish_project_callback(**kwargs) else: - logger.error(f'Error processing transfer {transfer_task_id}: Transfer status is {transfer_status}') - raise Exception(f'Transfer {transfer_task_id} failed with status {transfer_status}') + logger.error(f"Error processing transfer {transfer_task_id}: Transfer status is {transfer_status}") + raise Exception(f"Transfer {transfer_task_id} failed with status {transfer_status}") except Exception as e: - logger.error(f'Error processing transfer {transfer_task_id} with arguments {kwargs}: {e}') + logger.error(f"Error processing transfer {transfer_task_id} with arguments {kwargs}: {e}") self.retry(exc=e, countdown=30) @@ -361,30 +370,38 @@ def update_and_cleanup_review_project(review_project_id: str, status: Publicatio # update the publication request review_project = ProjectMetadata.get_project_by_id(review_project_id) - pub_request = PublicationRequest.objects.get(review_project=review_project, status=PublicationRequest.Status.PENDING) + pub_request = PublicationRequest.objects.get( + review_project=review_project, status=PublicationRequest.Status.PENDING + ) pub_request.status = status pub_request.save() - logger.info(f'Updated publication request for review project {review_project_id} to {status}.') + logger.info(f"Updated publication request for review project {review_project_id} to {status}.") # delete the review project and data inside it reviewers = pub_request.reviewers.all() for reviewer in reviewers: try: - remove_user(client, workspace_id, reviewer.username, review_project_id, settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME) - logger.info(f'Removed reviewer {reviewer.username} from review system {review_project_id}') + remove_user( + client, + workspace_id, + reviewer.username, + review_project_id, + settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, + ) + logger.info(f"Removed reviewer {reviewer.username} from review system {review_project_id}") except Exception: - logger.error(f'Error removing reviewer {reviewer.username} from review system {review_project_id}') + logger.error(f"Error removing reviewer {reviewer.username} from review system {review_project_id}") continue - client.files.delete(systemId=review_project_id, path='/') + client.files.delete(systemId=review_project_id, path="/") client.systems.deleteSystem(systemId=review_project_id) review_project_graph = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project=review_project) review_project_graph.delete() review_project.delete() - logger.info(f'Deleted review project {review_project_id} and its associated data.') + logger.info(f"Deleted review project {review_project_id} and its associated data.") def get_project_user_emails(project_id): @@ -399,7 +416,7 @@ def get_reviewer_emails(): return [reviewer.email for reviewer in reviewers if reviewer.email] -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def send_publication_accepted_email_to_authors(self, project_id): """ Alert project authors that their request has been accepted. @@ -430,7 +447,7 @@ def send_publication_accepted_email_to_authors(self, project_id): ) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def send_publication_rejected_email_to_authors(self, project_id: str): """ Alert project authors that their request has been rejected. @@ -461,7 +478,7 @@ def send_publication_rejected_email_to_authors(self, project_id: str): ) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def send_publication_in_review_email_to_authors(self, project_id): """ Alert dataset authors that their dataset is in review. @@ -495,7 +512,7 @@ def send_publication_in_review_email_to_authors(self, project_id): ) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def send_publication_reviewed_email_to_reviewers(self, project_id, status, reviewer): """ Alert dataset reviewers that a dataset has received feedback. @@ -503,7 +520,7 @@ def send_publication_reviewed_email_to_reviewers(self, project_id, status, revie reviewer_emails = get_reviewer_emails() if status == PublicationRequest.Status.REJECTED: - status = 'Revision Required' + status = "Revision Required" logger.info(f"Sending reviewer notification email to {reviewer_emails}") @@ -529,7 +546,7 @@ def send_publication_reviewed_email_to_reviewers(self, project_id, status, revie ) -@shared_task(bind=True, queue='default') +@shared_task(bind=True, queue="default") def send_publication_submitted_for_review_email_to_reviewers(self, project_id): """ Alert dataset reviewers that a dataset has been submitted for review. diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py index 6710748c60..b91dc790e5 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py @@ -1,10 +1,14 @@ """ Migration scripts for bringing Shared Workspaces from V2 to V3. """ + import requests from django.conf import settings from portal.apps.projects.models import LegacyProjectMetadata -from portal.apps.projects.workspace_operations.shared_workspace_operations import create_workspace_system, add_user_to_workspace +from portal.apps.projects.workspace_operations.shared_workspace_operations import ( + create_workspace_system, + add_user_to_workspace, +) from portal.libs.agave.utils import service_account from portal.settings.settings_secret import _AGAVE_SUPER_TOKEN as v2_token @@ -32,13 +36,13 @@ def get_role(project_id, username): def migrate_project(project_id): - print(f'Beginning migration for project: {project_id}') + print(f"Beginning migration for project: {project_id}") client = service_account() system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" try: v2_project = LegacyProjectMetadata.objects.get(project_id=project_id) except MultipleObjectsReturned: - print('FAILURE: more than 1 project with this ID') + print("FAILURE: more than 1 project with this ID") return try: @@ -47,16 +51,16 @@ def migrate_project(project_id): try: owner = v2_project.pi.username except AttributeError: - print('No owner or PI specified') + print("No owner or PI specified") return try: create_workspace_system(client, project_id, v2_project.title, v2_project.description) except BaseTapyException as e: - if 'SYSAPI_SYS_EXISTS' in e.message: - print('A Tapis V3 workspace already exists for this system.') + if "SYSAPI_SYS_EXISTS" in e.message: + print("A Tapis V3 workspace already exists for this system.") else: - print('Error creating workspace system') + print("Error creating workspace system") print(e) return @@ -65,12 +69,12 @@ def migrate_project(project_id): try: v3_role = ROLE_MAP[v2_role] except KeyError: - print(f'ERROR: No role found for: {v2_role}') + print(f"ERROR: No role found for: {v2_role}") v3_role = "reader" try: add_user_to_workspace(client, project_id, co_pi.username, v3_role) except NotFoundError: - print('ERROR: Workspace directory not found') + print("ERROR: Workspace directory not found") return for team_member in v2_project.team_members.all(): @@ -78,17 +82,17 @@ def migrate_project(project_id): try: v3_role = ROLE_MAP[v2_role] except KeyError: - print(f'ERROR: No role found for: {v2_role}') + print(f"ERROR: No role found for: {v2_role}") v3_role = "reader" try: add_user_to_workspace(client, project_id, team_member.username, v3_role) except NotFoundError: - print('ERROR: Workspace directory not found') + print("ERROR: Workspace directory not found") return client.systems.changeSystemOwner(systemId=system_id, userName=owner) - print(f'Successfully migrated project id: {project_id}') + print(f"Successfully migrated project id: {project_id}") def migrate_all_projects(): diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index 44f766f96f..c6bcb1023c 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -10,44 +10,25 @@ from portal.apps.projects.workspace_operations.project_meta_operations import create_project_metadata, get_ordered_value import logging + logger = logging.getLogger(__name__) def set_workspace_permissions(client: Tapis, username: str, system_id: str, role: str): """Apply read/write/execute permissions to a user on a system.""" - system_pems = { - "reader": ["READ", "EXECUTE"], - "writer": ["READ", "EXECUTE"], - "admin": ["READ", "EXECUTE", "MODIFY"] - } + system_pems = {"reader": ["READ", "EXECUTE"], "writer": ["READ", "EXECUTE"], "admin": ["READ", "EXECUTE", "MODIFY"]} - files_pems = { - "reader": "READ", - "writer": "MODIFY", - "admin": "MODIFY" - } + files_pems = {"reader": "READ", "writer": "MODIFY", "admin": "MODIFY"} logger.info(f"Adding {username} permissions to Tapis system {system_id}") - client.systems.grantUserPerms( - systemId=system_id, - userName=username, - permissions=system_pems[role]) + client.systems.grantUserPerms(systemId=system_id, userName=username, permissions=system_pems[role]) if role == "reader": - client.systems.revokeUserPerms(systemId=system_id, - userName=username, - permissions=["MODIFY"]) - client.files.deletePermissions(systemId=system_id, - path="/", - username=username) - - client.files.grantPermissions( - systemId=system_id, - path="/", - username=username, - permission=files_pems[role] - ) + client.systems.revokeUserPerms(systemId=system_id, userName=username, permissions=["MODIFY"]) + client.files.deletePermissions(systemId=system_id, path="/", username=username) + + client.files.grantPermissions(systemId=system_id, path="/", username=username, permission=files_pems[role]) def get_acl_string(usernames: str, role: str) -> str: @@ -60,12 +41,7 @@ def get_acl_string(usernames: str, role: str) -> str: "none": "d:u:{username},u:{username}", } - return ",".join( - [ - acl_string_map[role].format(username=username) - for username in usernames.split(",") - ] - ) + return ",".join([acl_string_map[role].format(username=username) for username in usernames.split(",")]) def set_workspace_acls(client, system_id, path, root_dir, usernames, operation, role): @@ -94,9 +70,7 @@ def set_workspace_acls(client, system_id, path, root_dir, usernames, operation, ) -def submit_workspace_acls_job( - client, usernames, system_id, path, role, action=Literal["add", "remove"] -): +def submit_workspace_acls_job(client, usernames, system_id, path, role, action=Literal["add", "remove"]): """ Submit a job to set ACLs on a project for a list of comma-separated users. This should be used if we are setting ACLs on an existing project, since there might be too many files for @@ -133,13 +107,15 @@ def submit_workspace_acls_job( def create_workspace_dir(workspace_id: str, system_id=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, **kwargs) -> str: client = service_account() path = f"{workspace_id}" - client.files.mkdir(systemId=system_id, - path=path, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + client.files.mkdir( + systemId=system_id, path=path, headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")} + ) return path -def create_workspace_system(client, workspace_id: str, title: str, description: str, keywords: str, owner=None, system_id=None, root_dir=None) -> str: +def create_workspace_system( + client, workspace_id: str, title: str, description: str, keywords: str, owner=None, system_id=None, root_dir=None +) -> str: system_id = system_id or f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" root_dir = root_dir or f"{settings.PORTAL_PROJECTS_ROOT_DIR}/{workspace_id}" @@ -154,9 +130,9 @@ def create_workspace_system(client, workspace_id: str, title: str, description: "effectiveUserId": settings.PORTAL_ADMIN_USERNAME, "authnCredential": { "privateKey": settings.PORTAL_PROJECTS_PRIVATE_KEY, - "publicKey": settings.PORTAL_PROJECTS_PUBLIC_KEY + "publicKey": settings.PORTAL_PROJECTS_PUBLIC_KEY, }, - "notes": {"title": title, "description": description, "keywords": keywords} + "notes": {"title": title, "description": description, "keywords": keywords}, } if owner: system_args["owner"] = owner @@ -174,8 +150,7 @@ def increment_workspace_count(force=None) -> int: if force: new_count = force - client.systems.patchSystem(systemId=root, - notes={"count": new_count}) + client.systems.patchSystem(systemId=root, notes={"count": new_count}) return new_count @@ -184,7 +159,9 @@ def increment_workspace_count(force=None) -> int: ########################################## -def create_shared_workspace(client: Tapis, title: str, description: str, keywords: str, owner: str, predefind_workspace_number=None, **kwargs): +def create_shared_workspace( + client: Tapis, title: str, description: str, keywords: str, owner: str, predefind_workspace_number=None, **kwargs +): """ Create a workspace system owned by user whose client is passed. """ @@ -195,18 +172,18 @@ def create_shared_workspace(client: Tapis, title: str, description: str, keyword # Service client creates directory and gives owner write permissions create_workspace_dir(workspace_id, **kwargs) - set_workspace_acls(service_client, - settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, - workspace_id, - f"{settings.PORTAL_PROJECTS_ROOT_DIR}/{workspace_id}", - owner, - "add", - "writer") + set_workspace_acls( + service_client, + settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + workspace_id, + f"{settings.PORTAL_PROJECTS_ROOT_DIR}/{workspace_id}", + owner, + "add", + "writer", + ) # User creates the system and adds their credential - system_id = create_workspace_system( - client, workspace_id, title, description, keywords, owner=owner - ) + system_id = create_workspace_system(client, workspace_id, title, description, keywords, owner=owner) # Give portal admin full permissions portal_admin = settings.PORTAL_ADMIN_USERNAME @@ -216,12 +193,9 @@ def create_shared_workspace(client: Tapis, title: str, description: str, keyword return system_id -def add_user_to_workspace(client: Tapis, - workspace_id: str, - username: str, - role="writer", - system_id=None, - system_name=None): +def add_user_to_workspace( + client: Tapis, workspace_id: str, username: str, role="writer", system_id=None, system_name=None +): """ Give a user POSIX and Tapis permissions on a workspace system. """ @@ -229,13 +203,7 @@ def add_user_to_workspace(client: Tapis, system_id = system_id or f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" # system_name = system_name or f"{settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME}" prj = client.systems.getSystem(systemId=system_id) - set_workspace_acls(service_client, - system_id, - "/", - prj.rootDir, - username, - "add", - role) + set_workspace_acls(service_client, system_id, "/", prj.rootDir, username, "add", role) # Share system to allow listing of users client.systems.shareSystem(systemId=system_id, users=[username]) @@ -252,13 +220,7 @@ def change_user_role(client, workspace_id: str, username: str, new_role): service_client = service_account() system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" prj = client.systems.getSystem(systemId=system_id) - set_workspace_acls(service_client, - system_id, - "/", - prj.rootDir, - username, - "add", - new_role) + set_workspace_acls(service_client, system_id, "/", prj.rootDir, username, "add", new_role) set_workspace_permissions(client, username, system_id, new_role) @@ -272,20 +234,10 @@ def remove_user(client, workspace_id: str, username: str, system_id=None, system system_name = system_name or f"{settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME}" prj = client.systems.getSystem(systemId=system_id) - set_workspace_acls(service_client, - system_id, - "/", - prj.rootDir, - username, - "remove", - "none") + set_workspace_acls(service_client, system_id, "/", prj.rootDir, username, "remove", "none") client.systems.unShareSystem(systemId=system_id, users=[username]) - client.systems.revokeUserPerms(systemId=system_id, - userName=username, - permissions=["READ", "MODIFY", "EXECUTE"]) - client.files.deletePermissions(systemId=system_id, - username=username, - path="/") + client.systems.revokeUserPerms(systemId=system_id, userName=username, permissions=["READ", "MODIFY", "EXECUTE"]) + client.files.deletePermissions(systemId=system_id, username=username, path="/") return get_project(client, workspace_id, system_id) @@ -297,20 +249,11 @@ def transfer_ownership(client, workspace_id: str, new_owner: str, old_owner: str service_client = service_account() system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" prj = client.systems.getSystem(systemId=system_id) - set_workspace_acls(service_client, - system_id, - "/", - prj.rootDir, - new_owner, - "add", - "writer") + set_workspace_acls(service_client, system_id, "/", prj.rootDir, new_owner, "add", "writer") # Ensure old owner retains access to Tapis system, as `changeSystemOwner` removes access for old owner client.systems.shareSystem(systemId=system_id, users=[old_owner]) - client.systems.grantUserPerms( - systemId=system_id, - userName=old_owner, - permissions=["READ", "EXECUTE"]) + client.systems.grantUserPerms(systemId=system_id, userName=old_owner, permissions=["READ", "EXECUTE"]) client.systems.changeSystemOwner(systemId=system_id, userName=new_owner) return get_project(client, workspace_id) @@ -318,10 +261,9 @@ def transfer_ownership(client, workspace_id: str, new_owner: str, old_owner: str def update_project(client, workspace_id: str, title: str, description: str, keywords: str): system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" - client.systems.patchSystem(systemId=system_id, - notes={"title": title, - "description": description, - "keywords": keywords}) + client.systems.patchSystem( + systemId=system_id, notes={"title": title, "description": description, "keywords": keywords} + ) return get_project(client, workspace_id) @@ -356,13 +298,12 @@ def list_projects(client, root_system_id=None): if root_system_id: root_system = next( - (system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system['system'] == root_system_id), - None + (system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system["system"] == root_system_id), None ) if root_system: query += f"~(rootDir.like.{root_system['rootDir']}*)" - is_review_system = root_system.get('reviewProject', False) - is_publication_system = root_system.get('publicationProject', False) + is_review_system = root_system.get("reviewProject", False) + is_publication_system = root_system.get("publicationProject", False) else: is_review_system = False is_publication_system = False @@ -370,36 +311,39 @@ def list_projects(client, root_system_id=None): is_review_system = False is_publication_system = False - community_system = next((system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system['scheme'] == 'community'), None) + community_system = next( + (system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system["scheme"] == "community"), None + ) if community_system and not is_review_system and not is_publication_system: - community_data_query = f"(id.like.{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.*)~(rootDir.like.{community_system['homeDir']}*)" + community_data_query = ( + f"(id.like.{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.*)~(rootDir.like.{community_system['homeDir']}*)" + ) else: community_data_query = None # use limit as -1 to allow search to corelate with # all projects available to the api user - listing = client.systems.getSystems(listType='ALL', - search=query, - select=fields, - limit=-1) + listing = client.systems.getSystems(listType="ALL", search=query, select=fields, limit=-1) if community_data_query: - community_listing = client.systems.getSystems(listType='ALL', - search=community_data_query, - select=fields, - limit=-1) + community_listing = client.systems.getSystems( + listType="ALL", search=community_data_query, select=fields, limit=-1 + ) listing = community_listing + listing - serialized_listing = map(lambda prj: { - "id": prj.id, - "path": prj.rootDir, - "name": prj.id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1], - "host": prj.host, - "updated": prj.updated, - "owner": get_project_user(prj.owner), - "title": getattr(prj.notes, "title", None), - "description": getattr(prj.notes, "description", None) - }, listing) + serialized_listing = map( + lambda prj: { + "id": prj.id, + "path": prj.rootDir, + "name": prj.id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1], + "host": prj.host, + "updated": prj.updated, + "owner": get_project_user(prj.owner), + "title": getattr(prj.notes, "title", None), + "description": getattr(prj.notes, "description", None), + }, + listing, + ) projects = list(serialized_listing) projects.sort(key=lambda p: p.get("updated") or "", reverse=True) return projects @@ -413,16 +357,16 @@ def get_project(client, workspace_id, system_id=None): users = [{"user": get_project_user(system.owner), "access": "owner"}] share_users = [u for u in shares.users if u not in [system.owner, settings.PORTAL_ADMIN_USERNAME]] for username in share_users: - perms = client.files.getPermissions(systemId=system_id, - path="/", - username=username) - if perms.permission == 'MODIFY': - access = 'edit' - elif perms.permission == 'READ': - access = 'read' + perms = client.files.getPermissions(systemId=system_id, path="/", username=username) + if perms.permission == "MODIFY": + access = "edit" + elif perms.permission == "READ": + access = "read" else: - logger.info(f"System shared to user without proper Tapis file permissions: {system_id}, username: {username}") - access = 'none' + logger.info( + f"System shared to user without proper Tapis file permissions: {system_id}, username: {username}" + ) + access = "none" users.append({"user": get_project_user(username), "access": access}) @@ -433,7 +377,6 @@ def get_project(client, workspace_id, system_id=None): "projectId": workspace_id, "members": users, "keywords": getattr(system.notes, "keywords", None), - } @@ -441,30 +384,42 @@ def get_workspace_role(client, workspace_id, username): system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" system = client.systems.getSystem(systemId=system_id) if system.owner == username: - return 'OWNER' + return "OWNER" - perms = client.files.getPermissions(systemId=system_id, - path="/", - username=username) - if perms.permission == 'MODIFY': - return 'USER' + perms = client.files.getPermissions(systemId=system_id, path="/", username=username) + if perms.permission == "MODIFY": + return "USER" - if perms.permission == 'READ': - return 'GUEST' + if perms.permission == "READ": + return "GUEST" return None @transaction.atomic -def create_publication_workspace(client, source_workspace_id: str, source_system_id: str, target_workspace_id: str, - target_system_id: str, title: str, description="", is_review=False): +def create_publication_workspace( + client, + source_workspace_id: str, + source_system_id: str, + target_workspace_id: str, + target_system_id: str, + title: str, + description="", + is_review=False, +): portal_admin_username = settings.PORTAL_ADMIN_USERNAME service_client = service_account() # Determine workspace and system-specific settings based on the project type - system_prefix = settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX if is_review else settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX - root_system_name = settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME if is_review else settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + system_prefix = ( + settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX if is_review else settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + ) + root_system_name = ( + settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME + if is_review + else settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + ) root_dir = settings.PORTAL_PROJECTS_REVIEW_ROOT_DIR if is_review else settings.PORTAL_PROJECTS_PUBLISHED_ROOT_DIR if is_review: @@ -479,7 +434,7 @@ def create_publication_workspace(client, source_workspace_id: str, source_system **project_value, "project_id": target_system_id, "is_review_project": is_review, - "is_published_project": not is_review + "is_published_project": not is_review, } # Create and save the new project metadata @@ -490,17 +445,23 @@ def create_publication_workspace(client, source_workspace_id: str, source_system create_workspace_dir(target_workspace_id, root_system_name) query = f"(id.eq.{target_system_id})" - listing = service_client.systems.getSystems(listType='ALL', search=query, select="id,deleted", - showDeleted=True, limit=-1) + listing = service_client.systems.getSystems( + listType="ALL", search=query, select="id,deleted", showDeleted=True, limit=-1 + ) if listing and listing[0].deleted: service_client.systems.undeleteSystem(systemId=target_system_id) else: # Create the target workspace system create_workspace_system( - service_client, target_workspace_id, title, description, None, None, + service_client, + target_workspace_id, + title, + description, + None, + None, f"{system_prefix}.{target_workspace_id}", - f"{root_dir}/{target_workspace_id}" + f"{root_dir}/{target_workspace_id}", ) # Configure workspace ACLs diff --git a/server/portal/apps/public_data/apps.py b/server/portal/apps/public_data/apps.py index e5a61c6289..8af8cdd9c9 100644 --- a/server/portal/apps/public_data/apps.py +++ b/server/portal/apps/public_data/apps.py @@ -2,4 +2,4 @@ class PublicDataConfig(AppConfig): - name = 'portal.apps.public_data' + name = "portal.apps.public_data" diff --git a/server/portal/apps/public_data/urls.py b/server/portal/apps/public_data/urls.py index 4560d77d19..96229804d6 100644 --- a/server/portal/apps/public_data/urls.py +++ b/server/portal/apps/public_data/urls.py @@ -2,21 +2,22 @@ .. module:: portal.apps.site_search.urls :synopsis: Site Search URLs """ + import re from django.urls import re_path from django.conf import settings from portal.apps.public_data.views import IndexView -app_name = 'public_data' +app_name = "public_data" published_prefix = re.escape(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX or "") id_prefix = re.escape(settings.PORTAL_PROJECTS_ID_PREFIX or "") urlpatterns = [ re_path( - rf'^{published_prefix}\.(?P{id_prefix}-[0-9]+)(v(?P[0-9]+))?/?$', + rf"^{published_prefix}\.(?P{id_prefix}-[0-9]+)(v(?P[0-9]+))?/?$", IndexView.as_view(), - name='index' + name="index", ), - re_path(r'^.*$', IndexView.as_view(), name='index_fallback'), + re_path(r"^.*$", IndexView.as_view(), name="index_fallback"), ] diff --git a/server/portal/apps/public_data/views.py b/server/portal/apps/public_data/views.py index 9283caf8be..60c651f79a 100644 --- a/server/portal/apps/public_data/views.py +++ b/server/portal/apps/public_data/views.py @@ -16,13 +16,15 @@ def get_google_scholar_context(pub): scholar_meta = {} scholar_meta["keywords"] = ", ".join(pub.value.get("keywords", [])) scholar_meta["citation_keywords"] = pub.value.get("keywords", []) - scholar_meta["entities"] = [{ - "title": pub.value.get("title"), - "description": pub.value.get("description"), - "doi": pub.value.get("doi"), - "authors": pub.value.get("authors", []), - "publication_date": pub.value.get("publicationDate") or pub.value.get("publication_date"), - }] + scholar_meta["entities"] = [ + { + "title": pub.value.get("title"), + "description": pub.value.get("description"), + "doi": pub.value.get("doi"), + "authors": pub.value.get("authors", []), + "publication_date": pub.value.get("publicationDate") or pub.value.get("publication_date"), + } + ] datacite_json_list = [get_datacite_json(pub_tree)] @@ -34,26 +36,28 @@ class IndexView(TemplateView): """ Main workbench view. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - project_id = kwargs.get('project_id') + project_id = kwargs.get("project_id") if project_id: try: pub = Publication.objects.get(project_id=project_id) scholar_context, datacite_context, title = get_google_scholar_context(pub) - context['dc_context'] = [json.dumps(ctx) for ctx in datacite_context] - context['scholar_context'] = scholar_context - context['citation_title'] = f"{project_id} | {title}" - context['publisher'] = settings.PORTAL_PUBLICATION_PUBLISHER + context["dc_context"] = [json.dumps(ctx) for ctx in datacite_context] + context["scholar_context"] = scholar_context + context["citation_title"] = f"{project_id} | {title}" + context["publisher"] = settings.PORTAL_PUBLICATION_PUBLISHER except Publication.DoesNotExist: pass except Exception as e: - logger.exception(f'Failed to build meta tags for project {project_id}: {e}') - context['setup_complete'] = False if self.request.user.is_anonymous \ - else self.request.user.profile.setup_complete - context['DEBUG'] = settings.DEBUG + logger.exception(f"Failed to build meta tags for project {project_id}: {e}") + context["setup_complete"] = ( + False if self.request.user.is_anonymous else self.request.user.profile.setup_complete + ) + context["DEBUG"] = settings.DEBUG return context def dispatch(self, request, *args, **kwargs): diff --git a/server/portal/apps/publications/apps.py b/server/portal/apps/publications/apps.py index a3ade9082f..eb3d14c8c6 100644 --- a/server/portal/apps/publications/apps.py +++ b/server/portal/apps/publications/apps.py @@ -2,4 +2,4 @@ class PublicationsConfig(AppConfig): - name = 'portal.apps.publications' + name = "portal.apps.publications" diff --git a/server/portal/apps/publications/migrations/0001_initial_squashed_0003_publication.py b/server/portal/apps/publications/migrations/0001_initial_squashed_0003_publication.py index f3ae46f313..e39a4293fe 100644 --- a/server/portal/apps/publications/migrations/0001_initial_squashed_0003_publication.py +++ b/server/portal/apps/publications/migrations/0001_initial_squashed_0003_publication.py @@ -8,39 +8,79 @@ class Migration(migrations.Migration): - - replaces = [('publications', '0001_initial'), ('publications', '0002_alter_publicationrequest_review_project_and_more'), ('publications', '0003_publication')] + replaces = [ + ("publications", "0001_initial"), + ("publications", "0002_alter_publicationrequest_review_project_and_more"), + ("publications", "0003_publication"), + ] dependencies = [ - ('projects', '0005_projectsmetadata_created_at_and_more'), - ('projects', '0007_projectmetadata_and_more'), + ("projects", "0005_projectsmetadata_created_at_and_more"), + ("projects", "0007_projectmetadata_and_more"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( - name='PublicationRequest', + name="PublicationRequest", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.CharField(choices=[('PENDING', 'Pending'), ('APPROVED', 'Approved'), ('REJECTED', 'Rejected')], default='PENDING', max_length=255)), - ('comments', models.TextField(blank=True, null=True)), - ('created_at', models.DateTimeField(default=django.utils.timezone.now)), - ('last_updated', models.DateTimeField(auto_now=True)), - ('review_project', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publication_reviews', to='projects.projectmetadata')), - ('reviewers', models.ManyToManyField(related_name='publication_reviewers', to=settings.AUTH_USER_MODEL)), - ('source_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='source_publication_reviews', to='projects.projectmetadata')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "status", + models.CharField( + choices=[("PENDING", "Pending"), ("APPROVED", "Approved"), ("REJECTED", "Rejected")], + default="PENDING", + max_length=255, + ), + ), + ("comments", models.TextField(blank=True, null=True)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("last_updated", models.DateTimeField(auto_now=True)), + ( + "review_project", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="publication_reviews", + to="projects.projectmetadata", + ), + ), + ( + "reviewers", + models.ManyToManyField(related_name="publication_reviewers", to=settings.AUTH_USER_MODEL), + ), + ( + "source_project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="source_publication_reviews", + to="projects.projectmetadata", + ), + ), ], ), migrations.CreateModel( - name='Publication', + name="Publication", fields=[ - ('project_id', models.CharField(editable=False, max_length=100, primary_key=True, serialize=False)), - ('created', models.DateTimeField(default=django.utils.timezone.now)), - ('is_published', models.BooleanField(default=True)), - ('last_updated', models.DateTimeField(auto_now=True)), - ('version', models.IntegerField(default=1)), - ('value', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, help_text="Value for the project's base metadata, including title/description/users")), - ('tree', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, help_text='JSON document containing the serialized publication tree')), + ("project_id", models.CharField(editable=False, max_length=100, primary_key=True, serialize=False)), + ("created", models.DateTimeField(default=django.utils.timezone.now)), + ("is_published", models.BooleanField(default=True)), + ("last_updated", models.DateTimeField(auto_now=True)), + ("version", models.IntegerField(default=1)), + ( + "value", + models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, + help_text="Value for the project's base metadata, including title/description/users", + ), + ), + ( + "tree", + models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, + help_text="JSON document containing the serialized publication tree", + ), + ), ], ), ] diff --git a/server/portal/apps/publications/models.py b/server/portal/apps/publications/models.py index 4a0882cdaf..3c4caf8d56 100644 --- a/server/portal/apps/publications/models.py +++ b/server/portal/apps/publications/models.py @@ -3,6 +3,7 @@ .. :module:: portal.apps.publications.models :synopsis: Metadata model for publications. """ + import logging from django.conf import settings from django.db import models @@ -16,26 +17,28 @@ class PublicationRequest(models.Model): - class Status(models.TextChoices): - PENDING = 'PENDING' - APPROVED = 'APPROVED' - REJECTED = 'REJECTED' + PENDING = "PENDING" + APPROVED = "APPROVED" + REJECTED = "REJECTED" - review_project = models.ForeignKey(ProjectMetadata, related_name='publication_reviews', on_delete=models.SET_NULL, null=True) - source_project = models.ForeignKey(ProjectMetadata, related_name='source_publication_reviews', on_delete=models.CASCADE) - reviewers = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name='publication_reviewers') + review_project = models.ForeignKey( + ProjectMetadata, related_name="publication_reviews", on_delete=models.SET_NULL, null=True + ) + source_project = models.ForeignKey( + ProjectMetadata, related_name="source_publication_reviews", on_delete=models.CASCADE + ) + reviewers = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name="publication_reviewers") status = models.CharField(max_length=255, choices=Status.choices, default=Status.PENDING) comments = models.TextField(blank=True, null=True) created_at = models.DateTimeField(default=timezone.now) last_updated = models.DateTimeField(auto_now=True) def __str__(self): - return f'Review for {self.review_project.project_id}' + return f"Review for {self.review_project.project_id}" class Publication(models.Model): - project_id = models.CharField(max_length=100, primary_key=True, editable=False) created = models.DateTimeField(default=timezone.now) is_published = models.BooleanField(default=True) @@ -43,9 +46,7 @@ class Publication(models.Model): version = models.IntegerField(default=1) value = models.JSONField( encoder=DjangoJSONEncoder, - help_text=( - "Value for the project's base metadata, including title/description/users" - ), + help_text=("Value for the project's base metadata, including title/description/users"), ) tree = models.JSONField( diff --git a/server/portal/apps/publications/urls.py b/server/portal/apps/publications/urls.py index cb4a617423..ee71a5bfcf 100644 --- a/server/portal/apps/publications/urls.py +++ b/server/portal/apps/publications/urls.py @@ -1,14 +1,18 @@ -"""Publications API Urls -""" +"""Publications API Urls""" + from portal.apps.publications import views from django.urls import path -app_name = 'publications_api' +app_name = "publications_api" urlpatterns = [ - path('publication-request/', views.PublicationRequestView.as_view(), name='publication_request'), - path('publication-request//', views.PublicationRequestView.as_view(), name='publication_request_detail'), - path('publish/', views.PublicationPublishView.as_view(), name='publication_publish'), - path('reject/', views.PublicationRejectView.as_view(), name='publication_reject'), - path('version/', views.PublicationVersionView.as_view(), name='publication_version'), - path('', views.PublicationListingView.as_view(), name='publication_listing'), + path("publication-request/", views.PublicationRequestView.as_view(), name="publication_request"), + path( + "publication-request//", + views.PublicationRequestView.as_view(), + name="publication_request_detail", + ), + path("publish/", views.PublicationPublishView.as_view(), name="publication_publish"), + path("reject/", views.PublicationRejectView.as_view(), name="publication_reject"), + path("version/", views.PublicationVersionView.as_view(), name="publication_version"), + path("", views.PublicationListingView.as_view(), name="publication_listing"), ] diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index 1cfafb3371..cf83b194b0 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -3,6 +3,7 @@ .. :module:: apps.publications.views :synopsis: Views to handle Publications """ + import json import logging from django.contrib.auth.decorators import login_required @@ -34,7 +35,6 @@ class PublicationRequestView(BaseApiView): - def get(self, request, project_id: str): if project_id: @@ -47,46 +47,46 @@ def get(self, request, project_id: str): publication_requests_data = [ { - 'id': pub_request.id, - 'status': pub_request.status, - 'comments': pub_request.comments, - 'reviewers': [ + "id": pub_request.id, + "status": pub_request.status, + "comments": pub_request.comments, + "reviewers": [ { - 'username': reviewer.username, - 'email': reviewer.email, - 'first_name': reviewer.first_name, - 'last_name': reviewer.last_name, + "username": reviewer.username, + "email": reviewer.email, + "first_name": reviewer.first_name, + "last_name": reviewer.last_name, } for reviewer in pub_request.reviewers.all() ], - 'created_at': pub_request.created_at, - 'last_updated': pub_request.last_updated + "created_at": pub_request.created_at, + "last_updated": pub_request.last_updated, } for pub_request in publication_requests ] except ProjectMetadata.DoesNotExist: - raise ApiException(f'Project {project_id} not found', status=404) + raise ApiException(f"Project {project_id} not found", status=404) - return JsonResponse({'response': publication_requests_data}) + return JsonResponse({"response": publication_requests_data}) - return JsonResponse({'response': []}) + return JsonResponse({"response": []}) - @method_decorator(login_required, name='dispatch') + @method_decorator(login_required, name="dispatch") def post(self, request): request_body = json.loads(request.body) client = request.user.tapis_oauth.client - full_project_id = request_body.get('project_id') + full_project_id = request_body.get("project_id") if not full_project_id: raise ApiException("Missing project ID", status=400) source_workspace_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] review_workspace_id = f"{source_workspace_id}" - source_system_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{source_workspace_id}' + source_system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{source_workspace_id}" review_system_id = f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{review_workspace_id}" with transaction.atomic(): @@ -95,20 +95,28 @@ def post(self, request): try: source_project = get_project_for_user(source_system_id, request.user) except ProjectMetadata.DoesNotExist as exc: - raise ApiException( - "User does not have access to the requested project", status=403 - ) from exc - source_project.value['authors'] = request_body.get('authors') + raise ApiException("User does not have access to the requested project", status=403) from exc + source_project.value["authors"] = request_body.get("authors") source_project.save() try: - create_publication_workspace(client, source_workspace_id, source_system_id, review_workspace_id, - review_system_id, request_body.get('title'), request_body.get('description'), True) + create_publication_workspace( + client, + source_workspace_id, + source_system_id, + review_workspace_id, + review_system_id, + request_body.get("title"), + request_body.get("description"), + True, + ) # Create publication request review_project = ProjectMetadata.get_project_by_id(review_system_id) source_project = ProjectMetadata.get_project_by_id(source_system_id) - publication_reviewers = get_user_model().objects.filter(groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME) + publication_reviewers = get_user_model().objects.filter( + groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME + ) publication_request = PublicationRequest( review_project=review_project, @@ -124,23 +132,25 @@ def post(self, request): continue publication_request.save() - logger.info(f'Created publication review for system {review_system_id}') + logger.info(f"Created publication review for system {review_system_id}") # Start task to copy files and metadata - copy_graph_and_files_for_review_system.apply_async(kwargs={ - 'user_access_token': client.access_token.access_token, - 'source_workspace_id': source_workspace_id, - 'review_workspace_id': review_workspace_id, - 'source_system_id': source_system_id, - 'review_system_id': review_system_id - }) + copy_graph_and_files_for_review_system.apply_async( + kwargs={ + "user_access_token": client.access_token.access_token, + "source_workspace_id": source_workspace_id, + "review_workspace_id": review_workspace_id, + "source_system_id": source_system_id, + "review_system_id": review_system_id, + } + ) # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.INFO, Notification.USER: request.user.username, - Notification.MESSAGE: f'{source_workspace_id} submitted for review', + Notification.MESSAGE: f"{source_workspace_id} submitted for review", } with transaction.atomic(): @@ -150,25 +160,24 @@ def post(self, request): # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.ERROR, Notification.USER: request.user.username, - Notification.MESSAGE: f'{source_workspace_id} creation failed', + Notification.MESSAGE: f"{source_workspace_id} creation failed", } with transaction.atomic(): Notification.objects.create(**event_data) - return JsonResponse({'response': 'OK'}) + return JsonResponse({"response": "OK"}) class PublicationListingView(BaseApiView): - def get(self, request): - query_string = request.GET.get('query_string') - offset = int(request.GET.get('offset', 0)) - limit = int(request.GET.get('limit', 100)) + query_string = request.GET.get("query_string") + offset = int(request.GET.get("offset", 0)) + limit = int(request.GET.get("limit", 100)) if query_string: query = IndexedPublication.search() @@ -191,19 +200,13 @@ def get(self, request): "nodes.value.authors.username", ], ) - term_query = Q( - { - "term": { - "nodes.value.projectId.keyword": query_string.replace("/", "\\/") - } - } - ) + term_query = Q({"term": {"nodes.value.projectId.keyword": query_string.replace("/", "\\/")}}) query = query.filter(qs_query | term_query) query = query.extra(from_=int(offset), size=int(limit)) res = query.execute() - hits = [hit.meta.id for hit in res if hasattr(hit.meta, 'id') and hit.meta.id is not None] + hits = [hit.meta.id for hit in res if hasattr(hit.meta, "id") and hit.meta.id is not None] if hits: publications = ( @@ -219,39 +222,40 @@ def get(self, request): publications_data = [] for publication in publications: publication_data = { - 'id': publication.value.get('projectId'), - 'title': publication.value.get('title'), - 'description': publication.value.get('description'), - 'keywords': publication.value.get('keywords'), - 'authors': publication.value.get('authors'), - 'publication_date': publication.created, + "id": publication.value.get("projectId"), + "title": publication.value.get("title"), + "description": publication.value.get("description"), + "keywords": publication.value.get("keywords"), + "authors": publication.value.get("authors"), + "publication_date": publication.created, } try: - project_meta = ProjectMetadata.objects.get(models.Q(value__projectId=publication.value.get('projectId'))) + project_meta = ProjectMetadata.objects.get( + models.Q(value__projectId=publication.value.get("projectId")) + ) - if project_meta.value.get('coverImage'): - publication_data['cover_image'] = project_meta.value['coverImage'] + if project_meta.value.get("coverImage"): + publication_data["cover_image"] = project_meta.value["coverImage"] else: - publication_data['cover_image'] = 'media/default/cover_image/default_logo.png' + publication_data["cover_image"] = "media/default/cover_image/default_logo.png" except ProjectMetadata.DoesNotExist: pass publications_data.append(publication_data) - return JsonResponse({'response': publications_data}) + return JsonResponse({"response": publications_data}) class PublicationPublishView(BaseApiView): - def post(self, request): """view for publishing a project""" client = request.user.tapis_oauth.client request_body = json.loads(request.body) - full_project_id = request_body.get('project_id') - is_review = request_body.get('is_review_project', False) + full_project_id = request_body.get("project_id") + is_review = request_body.get("is_review_project", False) if not full_project_id: raise ApiException("Missing project ID", status=400) @@ -264,29 +268,32 @@ def post(self, request): try: get_project_for_user(full_project_id, request.user) except ProjectMetadata.DoesNotExist as exc: - raise ApiException( - "User does not have access to the requested project", status=403 - ) from exc + raise ApiException("User does not have access to the requested project", status=403) from exc - source_system_id = f'{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}' + source_system_id = f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}" published_workspace_id = f"{project_id}" published_system_id = f"{settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX}.{published_workspace_id}" try: - create_publication_workspace(client, project_id, source_system_id, published_workspace_id, published_system_id, - request_body.get('title'), request_body.get('description'), False) + create_publication_workspace( + client, + project_id, + source_system_id, + published_workspace_id, + published_system_id, + request_body.get("title"), + request_body.get("description"), + False, + ) - publish_project.apply_async(kwargs={ - 'project_id': project_id, - 'version': 1 - }) + publish_project.apply_async(kwargs={"project_id": project_id, "version": 1}) # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.INFO, Notification.USER: request.user.username, - Notification.MESSAGE: f'{project_id} submitted for publication', + Notification.MESSAGE: f"{project_id} submitted for publication", } with transaction.atomic(): @@ -296,28 +303,27 @@ def post(self, request): # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.ERROR, Notification.USER: request.user.username, - Notification.MESSAGE: f'{project_id} publication failed', + Notification.MESSAGE: f"{project_id} publication failed", } with transaction.atomic(): Notification.objects.create(**event_data) - return JsonResponse({'response': 'OK'}) + return JsonResponse({"response": "OK"}) class PublicationVersionView(BaseApiView): - def post(self, request): """view for publishing a project""" client = request.user.tapis_oauth.client request_body = json.loads(request.body) - full_project_id = request_body.get('project_id') - is_review = request_body.get('is_review_project', False) + full_project_id = request_body.get("project_id") + is_review = request_body.get("is_review_project", False) if not full_project_id: raise ApiException("Missing project ID", status=400) @@ -330,38 +336,41 @@ def post(self, request): try: get_project_for_user(full_project_id, request.user) except ProjectMetadata.DoesNotExist as exc: - raise ApiException( - "User does not have access to the requested project", status=403 - ) from exc + raise ApiException("User does not have access to the requested project", status=403) from exc - print('project_id:', project_id) + print("project_id:", project_id) publication = Publication.objects.get(project_id=project_id) version = publication.version + 1 print(f"Version: {version}") - source_system_id = f'{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}' + source_system_id = f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}" published_workspace_id = f"{project_id}{f'v{version}' if version and version > 1 else ''}" published_system_id = f"{settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX}.{published_workspace_id}" print(f"Published Workspace ID: {published_workspace_id}") try: - create_publication_workspace(client, project_id, source_system_id, published_workspace_id, published_system_id, - request_body.get('title'), request_body.get('description'), False) + create_publication_workspace( + client, + project_id, + source_system_id, + published_workspace_id, + published_system_id, + request_body.get("title"), + request_body.get("description"), + False, + ) - publish_project.apply_async(kwargs={ - 'project_id': project_id, - 'version': version - }) + publish_project.apply_async(kwargs={"project_id": project_id, "version": version}) # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.INFO, Notification.USER: request.user.username, - Notification.MESSAGE: f'{project_id} submitted for publication', + Notification.MESSAGE: f"{project_id} submitted for publication", } with transaction.atomic(): @@ -371,43 +380,44 @@ def post(self, request): # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.ERROR, Notification.USER: request.user.username, - Notification.MESSAGE: f'{project_id} publication failed', + Notification.MESSAGE: f"{project_id} publication failed", } with transaction.atomic(): Notification.objects.create(**event_data) - return JsonResponse({'response': 'OK'}) + return JsonResponse({"response": "OK"}) class PublicationRejectView(BaseApiView): - def post(self, request): request_body = json.loads(request.body) - full_project_id = request_body.get('project_id') + full_project_id = request_body.get("project_id") if not full_project_id: raise ApiException("Missing project ID", status=400) if not settings.DEBUG: send_publication_rejected_email_to_authors.apply_async(args=[full_project_id]) - send_publication_reviewed_email_to_reviewers.apply_async(args=[full_project_id, PublicationRequest.Status.REJECTED, request.user.username]) + send_publication_reviewed_email_to_reviewers.apply_async( + args=[full_project_id, PublicationRequest.Status.REJECTED, request.user.username] + ) update_and_cleanup_review_project(full_project_id, PublicationRequest.Status.REJECTED) # Create notification event_data = { - Notification.EVENT_TYPE: 'projects', + Notification.EVENT_TYPE: "projects", Notification.STATUS: Notification.INFO, Notification.USER: request.user.username, - Notification.MESSAGE: f'{full_project_id} was rejected', + Notification.MESSAGE: f"{full_project_id} was rejected", } with transaction.atomic(): Notification.objects.create(**event_data) - return JsonResponse({'response': 'OK'}) + return JsonResponse({"response": "OK"}) diff --git a/server/portal/apps/request_access/api/unit_test.py b/server/portal/apps/request_access/api/unit_test.py index ec3cc7e8dd..19a3095d0c 100644 --- a/server/portal/apps/request_access/api/unit_test.py +++ b/server/portal/apps/request_access/api/unit_test.py @@ -6,8 +6,7 @@ @pytest.fixture def mock_rtutil(mocker, mock_rt): - mocker.patch('portal.apps.tickets.utils.rtUtil.DjangoRt', - return_value=mock_rt) + mocker.patch("portal.apps.tickets.utils.rtUtil.DjangoRt", return_value=mock_rt) yield mock_rt @@ -21,60 +20,53 @@ def mock_rt(mocker): @pytest.fixture def get_authenticate(mocker): - mock = mocker.patch( - 'portal.apps.request_access.api.views.TASClient.authenticate') + mock = mocker.patch("portal.apps.request_access.api.views.TASClient.authenticate") mock.return_value = True yield mock @pytest.fixture def get_authenticate_error(mocker): - mock = mocker.patch( - 'portal.apps.request_access.api.views.TASClient.authenticate') + mock = mocker.patch("portal.apps.request_access.api.views.TASClient.authenticate") mock.return_value = False yield mock @pytest.fixture def get_user(mocker): - mock = mocker.patch( - 'portal.apps.request_access.api.views.TASClient.get_user') - with open(os.path.join(settings.BASE_DIR, - 'fixtures/tas/tas_user.json')) as f: + mock = mocker.patch("portal.apps.request_access.api.views.TASClient.get_user") + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_user.json")) as f: tas_user = json.load(f) mock.return_value = tas_user yield mock @pytest.mark.django_db(transaction=True, reset_sequences=True) -def test_request_access_wrong_user_password(client, - regular_user, - get_authenticate_error): - response = client.post('/api/request-access/', - data={"problem_description": - "This is the problem description", - "username": "testUsername", - "password": "testPassword"}) +def test_request_access_wrong_user_password(client, regular_user, get_authenticate_error): + response = client.post( + "/api/request-access/", + data={ + "problem_description": "This is the problem description", + "username": "testUsername", + "password": "testPassword", + }, + ) assert response.status_code == 401 @pytest.mark.django_db(transaction=True, reset_sequences=True) -def test_request_access(client, - regular_user, - mock_rtutil, - get_authenticate, - get_user): - response = client.post('/api/request-access/', - data={"problem_description": "problem_description", - "username": "testUsername", - "password": "testPassword"}) +def test_request_access(client, regular_user, mock_rtutil, get_authenticate, get_user): + response = client.post( + "/api/request-access/", + data={"problem_description": "problem_description", "username": "testUsername", "password": "testPassword"}, + ) assert response.status_code == 200 result = json.loads(response.content) - assert result['ticket_id'] == 1 + assert result["ticket_id"] == 1 _, kwargs = mock_rtutil.create_ticket.call_args - assert kwargs['problem_description'].startswith("problem_description") - assert kwargs['requestor'] == "user@username.com" - assert kwargs['subject'] == "Request Access" + assert kwargs["problem_description"].startswith("problem_description") + assert kwargs["requestor"] == "user@username.com" + assert kwargs["subject"] == "Request Access" # check that some user info is added to metadata in problem_description - assert "first_name" in kwargs['problem_description'] - assert "last_name" in kwargs['problem_description'] + assert "first_name" in kwargs["problem_description"] + assert "last_name" in kwargs["problem_description"] diff --git a/server/portal/apps/request_access/api/urls.py b/server/portal/apps/request_access/api/urls.py index 4a30e44235..3360ba9855 100644 --- a/server/portal/apps/request_access/api/urls.py +++ b/server/portal/apps/request_access/api/urls.py @@ -1,7 +1,7 @@ from django.urls import path from portal.apps.request_access.api import views -app_name = 'request_access_api' +app_name = "request_access_api" urlpatterns = [ - path('', views.RequestAccessView.as_view()), + path("", views.RequestAccessView.as_view()), ] diff --git a/server/portal/apps/request_access/api/views.py b/server/portal/apps/request_access/api/views.py index dc23a30271..651f35b0cd 100644 --- a/server/portal/apps/request_access/api/views.py +++ b/server/portal/apps/request_access/api/views.py @@ -10,44 +10,37 @@ class RequestAccessView(BaseApiView): def post(self, request): - """Post an access request - - """ + """Post an access request""" data = request.POST.copy() - username = data.get('username') - password = data.get('password') - problem_description = data.get('problem_description') - subject = 'Request Access' + username = data.get("username") + password = data.get("password") + problem_description = data.get("problem_description") + subject = "Request Access" tas = TASClient( - baseURL=settings.TAS_URL, - credentials={ - 'username': settings.TAS_CLIENT_KEY, - 'password': settings.TAS_CLIENT_SECRET - } - ) + baseURL=settings.TAS_URL, + credentials={"username": settings.TAS_CLIENT_KEY, "password": settings.TAS_CLIENT_SECRET}, + ) try: auth = tas.authenticate(username, password) if auth: user = tas.get_user(username=username) - email = user['email'] - first_name = user['firstName'] - last_name = user['lastName'] + email = user["email"] + first_name = user["firstName"] + last_name = user["lastName"] else: - return JsonResponse({'message': 'Incorrect password'}, - status=401) + return JsonResponse({"message": "Incorrect password"}, status=401) except Exception as e: - logger.error('Incorrect password for user: {user}. {exc}' - .format(user=username, exc=e)) - return JsonResponse({'message': 'Incorrect password'}, status=401) + logger.error("Incorrect password for user: {user}. {exc}".format(user=username, exc=e)) + return JsonResponse({"message": "Incorrect password"}, status=401) if email is None or problem_description is None: return HttpResponseBadRequest() - info = request.GET.get('info', "None") + info = request.GET.get("info", "None") meta = request.META - return utils.create_ticket(None, first_name, last_name, email, '', - subject, problem_description, None, info, - meta) + return utils.create_ticket( + None, first_name, last_name, email, "", subject, problem_description, None, info, meta + ) diff --git a/server/portal/apps/request_access/apps.py b/server/portal/apps/request_access/apps.py index 1b4deb0ea9..6c19adafb0 100644 --- a/server/portal/apps/request_access/apps.py +++ b/server/portal/apps/request_access/apps.py @@ -2,4 +2,4 @@ class PublicDataConfig(AppConfig): - name = 'portal.apps.request_access' + name = "portal.apps.request_access" diff --git a/server/portal/apps/request_access/unit_test.py b/server/portal/apps/request_access/unit_test.py index dbdf0ea6d5..738afbbaf3 100644 --- a/server/portal/apps/request_access/unit_test.py +++ b/server/portal/apps/request_access/unit_test.py @@ -1,10 +1,9 @@ - def test_request_access_get(client, regular_user): - response = client.get('/request-access/') + response = client.get("/request-access/") assert response.status_code == 200 def test_request_access_authenticated(client, authenticated_user): - response = client.get('/request-access/') + response = client.get("/request-access/") assert response.status_code == 302 - assert response.url == '/workbench/dashboard/' + assert response.url == "/workbench/dashboard/" diff --git a/server/portal/apps/request_access/urls.py b/server/portal/apps/request_access/urls.py index d572d4a9ef..724dbdf2ca 100644 --- a/server/portal/apps/request_access/urls.py +++ b/server/portal/apps/request_access/urls.py @@ -1,7 +1,7 @@ from django.urls import path from portal.apps.request_access.views import IndexView -app_name = 'request_access' +app_name = "request_access" urlpatterns = [ - path('', IndexView.as_view(), name='index'), + path("", IndexView.as_view(), name="index"), ] diff --git a/server/portal/apps/request_access/views.py b/server/portal/apps/request_access/views.py index 7fe66627b1..7001e55b2b 100644 --- a/server/portal/apps/request_access/views.py +++ b/server/portal/apps/request_access/views.py @@ -7,16 +7,17 @@ class IndexView(TemplateView): """ Request Access view. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def dispatch(self, request, *args, **kwargs): if request.user.is_authenticated: - return redirect('/workbench/dashboard/') + return redirect("/workbench/dashboard/") return super(IndexView, self).dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - context['DEBUG'] = settings.DEBUG + context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/search/apps.py b/server/portal/apps/search/apps.py index 3065882365..4e0bc34f29 100644 --- a/server/portal/apps/search/apps.py +++ b/server/portal/apps/search/apps.py @@ -1,19 +1,19 @@ - - from django.apps import AppConfig from django.conf import settings class SearchConfig(AppConfig): - name = 'portal.apps.search' + name = "portal.apps.search" def ready(self): from elasticsearch_dsl.connections import connections + HOSTS = settings.ES_HOSTS - connections.create_connection('default', - hosts=HOSTS, - http_auth=settings.ES_AUTH, - max_retries=3, - retry_on_timeout=True, - ) + connections.create_connection( + "default", + hosts=HOSTS, + http_auth=settings.ES_AUTH, + max_retries=3, + retry_on_timeout=True, + ) diff --git a/server/portal/apps/search/management/commands/reindex-files.py b/server/portal/apps/search/management/commands/reindex-files.py index f14e3f6421..f421d3f75a 100644 --- a/server/portal/apps/search/management/commands/reindex-files.py +++ b/server/portal/apps/search/management/commands/reindex-files.py @@ -23,20 +23,33 @@ class Command(BaseCommand): help = "Reindex all files into a fresh index, then swap aliases with the current default index." def add_arguments(self, parser): - parser.add_argument('--cleanup', help='Remove documents after swapping aliases to save space.', default=False, action='store_true') - parser.add_argument('--swap-only', help='Only swap index aliases without reindexing.', default=False, action='store_true') + parser.add_argument( + "--cleanup", + help="Remove documents after swapping aliases to save space.", + default=False, + action="store_true", + ) + parser.add_argument( + "--swap-only", help="Only swap index aliases without reindexing.", default=False, action="store_true" + ) def handle(self, *args, **options): - es_client = elasticsearch.Elasticsearch([{'host': settings.ES_HOSTS, 'http_auth': settings.ES_AUTH}], timeout=60) - cleanup = options.get('cleanup') - swap_only = options.get('swap-only') - default_index_alias = settings.ES_INDEX_PREFIX.format('files') - reindex_index_alias = settings.ES_INDEX_PREFIX.format('files-reindex') + es_client = elasticsearch.Elasticsearch( + [{"host": settings.ES_HOSTS, "http_auth": settings.ES_AUTH}], timeout=60 + ) + cleanup = options.get("cleanup") + swap_only = options.get("swap-only") + default_index_alias = settings.ES_INDEX_PREFIX.format("files") + reindex_index_alias = settings.ES_INDEX_PREFIX.format("files-reindex") if not swap_only: - confirm = input('This will delete any documents in the index "{}" and recreate the index. Continue? (Y/n) '.format(reindex_index_alias)) - if confirm != 'Y': - self.stdout.write('Aborting reindex.') + confirm = input( + 'This will delete any documents in the index "{}" and recreate the index. Continue? (Y/n) '.format( + reindex_index_alias + ) + ) + if confirm != "Y": + self.stdout.write("Aborting reindex.") raise SystemExit # Set up a fresh reindexing alias. setup_files_index(reindex=True, force=True) @@ -45,7 +58,9 @@ def handle(self, *args, **options): default_index_name = Index(default_index_alias, using=es_client).get_alias().keys()[0] reindex_index_name = Index(reindex_index_alias, using=es_client).get_alias().keys()[0] except Exception: - self.stdout.write('Unable to lookup required indices by alias. Have you set up both a default and a reindexing index?') + self.stdout.write( + "Unable to lookup required indices by alias. Have you set up both a default and a reindexing index?" + ) raise SystemExit if not swap_only: @@ -53,11 +68,11 @@ def handle(self, *args, **options): elasticsearch.helpers.reindex(es_client, default_index_name, reindex_index_name) alias_body = { - 'actions': [ - {'remove': {'index': default_index_name, 'alias': default_index_alias}}, - {'remove': {'index': reindex_index_name, 'alias': reindex_index_alias}}, - {'add': {'index': default_index_name, 'alias': reindex_index_alias}}, - {'add': {'index': reindex_index_name, 'alias': default_index_alias}}, + "actions": [ + {"remove": {"index": default_index_name, "alias": default_index_alias}}, + {"remove": {"index": reindex_index_name, "alias": reindex_index_alias}}, + {"add": {"index": default_index_name, "alias": reindex_index_alias}}, + {"add": {"index": reindex_index_name, "alias": default_index_alias}}, ] } # Swap the aliases of the default and reindexing aliases. diff --git a/server/portal/apps/search/management/commands/unit_test.py b/server/portal/apps/search/management/commands/unit_test.py index 57236ad982..d4e3597a88 100644 --- a/server/portal/apps/search/management/commands/unit_test.py +++ b/server/portal/apps/search/management/commands/unit_test.py @@ -4,11 +4,10 @@ class TestSwapReindex(TestCase): - def setUp(self): - self.patch_setup = patch('portal.apps.search.management.commands.reindex-files.setup_files_index') - self.patch_connections = patch('portal.apps.search.management.commands.reindex-files.connections') - self.patch_elasticsearch = patch('portal.apps.search.management.commands.reindex-files.elasticsearch') + self.patch_setup = patch("portal.apps.search.management.commands.reindex-files.setup_files_index") + self.patch_connections = patch("portal.apps.search.management.commands.reindex-files.connections") + self.patch_elasticsearch = patch("portal.apps.search.management.commands.reindex-files.elasticsearch") self.mock_setup = self.patch_setup.start() self.mock_connections = self.patch_connections.start() @@ -18,64 +17,68 @@ def setUp(self): self.addCleanup(self.patch_connections.stop) self.addCleanup(self.patch_elasticsearch.stop) - @patch('portal.apps.search.management.commands.reindex-files.input') + @patch("portal.apps.search.management.commands.reindex-files.input") def test_raises_when_user_does_not_proceed(self, mock_input): - mock_input.return_value = 'n' + mock_input.return_value = "n" with self.assertRaises(SystemExit): - call_command('reindex-files') + call_command("reindex-files") - @patch('portal.apps.search.management.commands.reindex-files.Index') - @patch('portal.apps.search.management.commands.reindex-files.input') + @patch("portal.apps.search.management.commands.reindex-files.Index") + @patch("portal.apps.search.management.commands.reindex-files.input") def test_raises_exception_when_no_index(self, mock_input, mock_index): - mock_input.return_value = 'Y' + mock_input.return_value = "Y" mock_index.return_value.get_alias.return_value.keys.side_effect = Exception with self.assertRaises(SystemExit): - call_command('reindex-files') + call_command("reindex-files") - @patch('portal.apps.search.management.commands.reindex-files.Index') - @patch('portal.apps.search.management.commands.reindex-files.input') + @patch("portal.apps.search.management.commands.reindex-files.Index") + @patch("portal.apps.search.management.commands.reindex-files.input") def test_performs_reindex_from_default_to_reindex(self, mock_input, mock_index): - mock_input.return_value = 'Y' + mock_input.return_value = "Y" - mock_index.return_value.get_alias.return_value.keys.side_effect = [['DEFAULT_NAME'], ['REINDEX_NAME']] + mock_index.return_value.get_alias.return_value.keys.side_effect = [["DEFAULT_NAME"], ["REINDEX_NAME"]] mock_client = MagicMock() self.mock_elasticsearch.Elasticsearch.return_value = mock_client - call_command('reindex-files') + call_command("reindex-files") - self.mock_elasticsearch.helpers.reindex.assert_called_with(mock_client, 'DEFAULT_NAME', 'REINDEX_NAME') + self.mock_elasticsearch.helpers.reindex.assert_called_with(mock_client, "DEFAULT_NAME", "REINDEX_NAME") - @patch('portal.apps.search.management.commands.reindex-files.Index') - @patch('portal.apps.search.management.commands.reindex-files.input') + @patch("portal.apps.search.management.commands.reindex-files.Index") + @patch("portal.apps.search.management.commands.reindex-files.input") def test_performs_swap_with_correct_args(self, mock_input, mock_index): - mock_input.return_value = 'Y' + mock_input.return_value = "Y" - mock_index.return_value.get_alias.return_value.keys.side_effect = [['DEFAULT_NAME'], ['REINDEX_NAME']] + mock_index.return_value.get_alias.return_value.keys.side_effect = [["DEFAULT_NAME"], ["REINDEX_NAME"]] - call_command('reindex-files') + call_command("reindex-files") mock_alias = { - 'actions': [ - {'remove': {'index': 'DEFAULT_NAME', 'alias': 'test-staging-files'}}, - {'remove': {'index': 'REINDEX_NAME', 'alias': 'test-staging-files-reindex'}}, - {'add': {'index': 'DEFAULT_NAME', 'alias': 'test-staging-files-reindex'}}, - {'add': {'index': 'REINDEX_NAME', 'alias': 'test-staging-files'}}, + "actions": [ + {"remove": {"index": "DEFAULT_NAME", "alias": "test-staging-files"}}, + {"remove": {"index": "REINDEX_NAME", "alias": "test-staging-files-reindex"}}, + {"add": {"index": "DEFAULT_NAME", "alias": "test-staging-files-reindex"}}, + {"add": {"index": "REINDEX_NAME", "alias": "test-staging-files"}}, ] } self.mock_elasticsearch.Elasticsearch().indices.update_aliases.assert_called_with(mock_alias) - @patch('portal.apps.search.management.commands.reindex-files.Index') - @patch('portal.apps.search.management.commands.reindex-files.input') + @patch("portal.apps.search.management.commands.reindex-files.Index") + @patch("portal.apps.search.management.commands.reindex-files.input") def test_cleanup(self, mock_input, mock_index): - mock_input.return_value = 'Y' + mock_input.return_value = "Y" - mock_index.return_value.get_alias.return_value.keys.side_effect = [['DEFAULT_NAME'], ['REINDEX_NAME'], ['REINDEX_NAME']] - opts = {'cleanup': True} + mock_index.return_value.get_alias.return_value.keys.side_effect = [ + ["DEFAULT_NAME"], + ["REINDEX_NAME"], + ["REINDEX_NAME"], + ] + opts = {"cleanup": True} - call_command('reindex-files', **opts) + call_command("reindex-files", **opts) self.assertEqual(mock_index.return_value.delete.call_count, 1) diff --git a/server/portal/apps/search/tasks.py b/server/portal/apps/search/tasks.py index 8992dd0d09..4ec1e44756 100644 --- a/server/portal/apps/search/tasks.py +++ b/server/portal/apps/search/tasks.py @@ -4,20 +4,28 @@ from portal.libs.agave.utils import user_account, service_account from portal.libs.elasticsearch.utils import index_listing, index_project_listing from portal.apps.projects.models.metadata import LegacyProjectMetadata -from portal.libs.elasticsearch.docs.base import (IndexedProject, IndexedPublication) +from portal.libs.elasticsearch.docs.base import IndexedProject, IndexedPublication from elasticsearch.exceptions import NotFoundError logger = logging.getLogger(__name__) # Crawl and index agave files -@shared_task(bind=True, max_retries=3, queue='indexing', retry_backoff=True, rate_limit="12/m") -def tapis_indexer(self, systemId, access_token=None, filePath='/', recurse=True, update_pems=False, ignore_hidden=True, reindex=False): - - if next((sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if sys.get('scheme', None) == 'projects' - and sys.get('hideSearchBar', None) - and systemId.startswith(settings.PORTAL_PROJECTS_SYSTEM_PREFIX)), None): +@shared_task(bind=True, max_retries=3, queue="indexing", retry_backoff=True, rate_limit="12/m") +def tapis_indexer( + self, systemId, access_token=None, filePath="/", recurse=True, update_pems=False, ignore_hidden=True, reindex=False +): + + if next( + ( + sys + for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS + if sys.get("scheme", None) == "projects" + and sys.get("hideSearchBar", None) + and systemId.startswith(settings.PORTAL_PROJECTS_SYSTEM_PREFIX) + ), + None, + ): return from portal.libs.elasticsearch.utils import index_level @@ -25,8 +33,8 @@ def tapis_indexer(self, systemId, access_token=None, filePath='/', recurse=True, client = user_account(access_token) if access_token else service_account() - if not filePath.startswith('/'): - filePath = '/' + filePath + if not filePath.startswith("/"): + filePath = "/" + filePath try: filePath, folders, files = walk_levels(client, systemId, filePath, ignore_hidden=ignore_hidden).__next__() @@ -37,23 +45,25 @@ def tapis_indexer(self, systemId, access_token=None, filePath='/', recurse=True, if recurse: for child in folders: - self.delay(systemId, filePath=child.get('path'), reindex=reindex) + self.delay(systemId, filePath=child.get("path"), reindex=reindex) -@shared_task(bind=True, max_retries=3, queue='default') +@shared_task(bind=True, max_retries=3, queue="default") def tapis_listing_indexer(self, listing): index_listing(listing) -@shared_task(bind=True, queue='indexing') +@shared_task(bind=True, queue="indexing") def index_community_data(self, reindex=False): for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS: - if sys['api'] == 'tapis' and sys['scheme'] in ['community', 'public']: - logger.info('INDEXING {} SYSTEM with file path {}'.format(sys['name'], sys.get("homeDir", "/"))) - tapis_indexer.apply_async(args=[sys['system']], kwargs={'filePath': sys.get("homeDir", "/"), 'reindex': reindex}) + if sys["api"] == "tapis" and sys["scheme"] in ["community", "public"]: + logger.info("INDEXING {} SYSTEM with file path {}".format(sys["name"], sys.get("homeDir", "/"))) + tapis_indexer.apply_async( + args=[sys["system"]], kwargs={"filePath": sys.get("homeDir", "/"), "reindex": reindex} + ) -@shared_task(bind=True, max_retries=3, queue='indexing') +@shared_task(bind=True, max_retries=3, queue="indexing") def index_project(self, project_id): project = LegacyProjectMetadata.objects.get(project_id=project_id) project_dict = project.to_dict() @@ -62,7 +72,7 @@ def index_project(self, project_id): project_doc.save() -@shared_task(bind=True, max_retries=3, queue='default') +@shared_task(bind=True, max_retries=3, queue="default") def tapis_project_listing_indexer(self, projects): index_project_listing(projects) diff --git a/server/portal/apps/signals/apps.py b/server/portal/apps/signals/apps.py index 7f3b0de1bf..14cfa66d00 100644 --- a/server/portal/apps/signals/apps.py +++ b/server/portal/apps/signals/apps.py @@ -2,8 +2,8 @@ class SignalsConfig(AppConfig): - name = 'portal.apps.signals' - verbose_name = 'Portal Signals' + name = "portal.apps.signals" + verbose_name = "Portal Signals" def ready(self): diff --git a/server/portal/apps/signals/receivers.py b/server/portal/apps/signals/receivers.py index dc4468c7ac..383a76a19e 100644 --- a/server/portal/apps/signals/receivers.py +++ b/server/portal/apps/signals/receivers.py @@ -10,6 +10,7 @@ import copy from asgiref.sync import async_to_sync from channels.layers import get_channel_layer + channel_layer = get_channel_layer() logger = logging.getLogger(__name__) @@ -18,32 +19,20 @@ @receiver(portal_event, dispatch_uid=__name__) def portal_event_callback(sender, **kwargs): logger.debug("Received a generic portal event") - users = kwargs.get('event_users', []) + users = kwargs.get("event_users", []) data = copy.copy(kwargs) - data.pop('signal') + data.pop("signal") if users: for user in users: - async_to_sync(channel_layer.group_send)( - str(user.id), - { - 'type': 'portal_notification', - 'body': data - } - ) + async_to_sync(channel_layer.group_send)(str(user.id), {"type": "portal_notification", "body": data}) else: - async_to_sync(channel_layer.group_send)( - 'portal_events', - { - 'type': 'portal_notification', - 'body': data - } - ) + async_to_sync(channel_layer.group_send)("portal_events", {"type": "portal_notification", "body": data}) -@receiver(post_save, sender=Notification, dispatch_uid='notification_msg') +@receiver(post_save, sender=Notification, dispatch_uid="notification_msg") def send_notification_ws(sender, instance, created, **kwargs): # Only send WS message if it's a new notification not if we're updating. logger.info("Received a Notification event") @@ -53,26 +42,18 @@ def send_notification_ws(sender, instance, created, **kwargs): instance_dict = instance.to_dict() logger.info(instance_dict) user = get_user_model().objects.get(username=instance.user) - async_to_sync(channel_layer.group_send)( - str(user.id), - { - 'type': 'portal_notification', - 'body': instance_dict - } - ) + async_to_sync(channel_layer.group_send)(str(user.id), {"type": "portal_notification", "body": instance_dict}) except Exception: - logger.exception( - 'Exception sending message to channel: portal_notification', - extra=instance.to_dict()) + logger.exception("Exception sending message to channel: portal_notification", extra=instance.to_dict()) return -@receiver(post_save, sender=LegacyProjectMetadata, dispatch_uid='index_project') +@receiver(post_save, sender=LegacyProjectMetadata, dispatch_uid="index_project") def index_project_on_save(sender, instance, created, **kwargs): index_project.apply_async(args=[instance.project_id]) -@receiver(post_save, sender=SetupEvent, dispatch_uid='setup_event') +@receiver(post_save, sender=SetupEvent, dispatch_uid="setup_event") def send_setup_event(sender, instance, **kwargs): logger.info("Sending setup event through websocket") setup_event = instance @@ -85,22 +66,10 @@ def send_setup_event(sender, instance, **kwargs): # Add the setup_event's user to the notification list receiving_users.append(setup_event.user) try: - data = { - "event_type": "setup_event", - "setup_event": setup_event.to_dict() - } + data = {"event_type": "setup_event", "setup_event": setup_event.to_dict()} for user in set(receiving_users): - async_to_sync(channel_layer.group_send)( - str(user.id), - { - 'type': 'portal_notification', - 'body': data - } - ) + async_to_sync(channel_layer.group_send)(str(user.id), {"type": "portal_notification", "body": data}) except Exception: - logger.exception( - 'Exception sending message to channel: portal_notification', - extra=setup_event.to_dict() - ) + logger.exception("Exception sending message to channel: portal_notification", extra=setup_event.to_dict()) return diff --git a/server/portal/apps/site_search/api/unit_test.py b/server/portal/apps/site_search/api/unit_test.py index c118c09e0e..ab71870ced 100644 --- a/server/portal/apps/site_search/api/unit_test.py +++ b/server/portal/apps/site_search/api/unit_test.py @@ -22,9 +22,7 @@ def mock_cms_search(mocker): @pytest.fixture def mock_service_account(mocker): - yield mocker.patch( - "portal.apps.site_search.api.views.service_account", autospec=True - ) + yield mocker.patch("portal.apps.site_search.api.views.service_account", autospec=True) @pytest.fixture @@ -126,9 +124,7 @@ def test_search_with_auth(regular_user, client, mock_cms_search, mock_files_sear }, ], ) -def test_search_with_tapis_error( - regular_user, client, mock_cms_search, mocker, tapis_test_config -): +def test_search_with_tapis_error(regular_user, client, mock_cms_search, mocker, tapis_test_config): # Test if does not error out when public or community search fails with SSH related errors. # file search return different error based on the type. def file_search_side_effect(*args, **kwargs): @@ -161,9 +157,7 @@ def file_search_side_effect(*args, **kwargs): } -def test_search_no_auth( - client, mock_cms_search, mock_files_search, mock_service_account -): +def test_search_no_auth(client, mock_cms_search, mock_files_search, mock_service_account): response = client.get("/api/site-search/?page=0&query_string=test") assert response.json() == { @@ -182,9 +176,7 @@ def test_search_no_auth( } -def test_search_public( - client, configure_public, mock_cms_search, mock_files_search, mock_service_account -): +def test_search_public(client, configure_public, mock_cms_search, mock_files_search, mock_service_account): response = client.get("/api/site-search/?page=0&query_string=test") assert response.json() == { @@ -214,9 +206,7 @@ def test_cms_search_util(mock_dsl_search): dummy_result.hits.__iter__.return_value = [dummy_hit] dummy_result.hits.total.value = 1 - mock_dsl_search().query().highlight().highlight().highlight_options().extra().execute.return_value = ( - dummy_result - ) + mock_dsl_search().query().highlight().highlight().highlight_options().extra().execute.return_value = dummy_result res = cms_search("test_query", offset=0, limit=10) assert res == (1, [{"title": "test title", "highlight": {"body": ["highlight 1"]}}]) diff --git a/server/portal/apps/site_search/api/urls.py b/server/portal/apps/site_search/api/urls.py index dee0886a95..843151754c 100644 --- a/server/portal/apps/site_search/api/urls.py +++ b/server/portal/apps/site_search/api/urls.py @@ -2,10 +2,11 @@ .. module:: portal.apps.site_search.api.urls :synopsis: Site Search API URLs """ + from django.urls import re_path from portal.apps.site_search.api.views import SiteSearchApiView -app_name = 'site_search' +app_name = "site_search" urlpatterns = [ - re_path('', SiteSearchApiView.as_view(), name='site_search_api'), + re_path("", SiteSearchApiView.as_view(), name="site_search_api"), ] diff --git a/server/portal/apps/site_search/api/views.py b/server/portal/apps/site_search/api/views.py index fa7fab86c2..e2ae182a2d 100644 --- a/server/portal/apps/site_search/api/views.py +++ b/server/portal/apps/site_search/api/views.py @@ -11,90 +11,91 @@ def cms_search(query_string, offset=0, limit=10): - cms_index = settings.ES_INDEX_PREFIX.format('cms') - cms_search = Search(index=cms_index)\ - .query( - "query_string", - query=query_string, - default_operator="and", - fields=['title', 'body'])\ - .highlight( - 'body', - fragment_size=100)\ - .highlight('title')\ - .highlight_options( - pre_tags=[""], - post_tags=[""], - require_field_match=False)\ + cms_index = settings.ES_INDEX_PREFIX.format("cms") + cms_search = ( + Search(index=cms_index) + .query("query_string", query=query_string, default_operator="and", fields=["title", "body"]) + .highlight("body", fragment_size=100) + .highlight("title") + .highlight_options(pre_tags=[""], post_tags=[""], require_field_match=False) .extra(from_=offset, size=limit) + ) cms_search = cms_search.execute() res = cms_search.hits total = cms_search.hits.total.value - results = list(map(lambda x: {**x.to_dict(), - 'highlight': x.meta.highlight.to_dict()}, - res)) + results = list(map(lambda x: {**x.to_dict(), "highlight": x.meta.highlight.to_dict()}, res)) return total, results def files_search(client, query_string, system, path, filter=None, offset=0, limit=10): - res = search_operation(client, system, path, offset=offset, limit=limit, - query_string=query_string, filter=filter) - return (res['count'], res['listing']) + res = search_operation(client, system, path, offset=offset, limit=limit, query_string=query_string, filter=filter) + return (res["count"], res["listing"]) class SiteSearchApiView(BaseApiView): - def get(self, request, *args, **kwargs): - qs = request.GET.get('query_string', '') - filter = request.GET.get('filter', None) - page = request.GET.get('page', 1) + qs = request.GET.get("query_string", "") + filter = request.GET.get("filter", None) + page = request.GET.get("page", 1) limit = 10 offset = (int(page) - 1) * limit cms_total, cms_results = cms_search(qs, offset, limit) - response = { - 'cms': {'count': cms_total, - 'listing': cms_results, - 'type': 'cms', - 'include': True}} + response = {"cms": {"count": cms_total, "listing": cms_results, "type": "cms", "include": True}} try: - public_conf = next(conf for conf - in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if conf['scheme'] == 'public' - and ('siteSearchPriority' in conf and conf['siteSearchPriority'] is not None)) - client = request.user.tapis_oauth.client if (request.user.is_authenticated and request.user.profile.setup_complete) else service_account() - (public_total, public_results) = \ - files_search(client, qs, public_conf['system'], public_conf.get("homeDir", "/"), filter=filter, - offset=offset, limit=limit) - response['public'] = {'count': public_total, - 'listing': public_results, - 'type': 'file', - 'include': True} + public_conf = next( + conf + for conf in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS + if conf["scheme"] == "public" + and ("siteSearchPriority" in conf and conf["siteSearchPriority"] is not None) + ) + client = ( + request.user.tapis_oauth.client + if (request.user.is_authenticated and request.user.profile.setup_complete) + else service_account() + ) + (public_total, public_results) = files_search( + client, + qs, + public_conf["system"], + public_conf.get("homeDir", "/"), + filter=filter, + offset=offset, + limit=limit, + ) + response["public"] = {"count": public_total, "listing": public_results, "type": "file", "include": True} except StopIteration: pass except BaseTapyException as e: self._handle_tapis_ssh_exception(e) - if request.user.is_authenticated and \ - request.user.profile.setup_complete: + if request.user.is_authenticated and request.user.profile.setup_complete: try: - community_conf = \ - next(conf for conf - in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if conf['scheme'] == 'community' - and ('siteSearchPriority' in conf and conf['siteSearchPriority'] is not None)) + community_conf = next( + conf + for conf in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS + if conf["scheme"] == "community" + and ("siteSearchPriority" in conf and conf["siteSearchPriority"] is not None) + ) client = request.user.tapis_oauth.client - (community_total, community_results) = \ - files_search(client, qs, community_conf['system'], community_conf.get("homeDir", "/"), filter=filter, - offset=offset, - limit=limit) - response['community'] = {'count': community_total, - 'listing': community_results, - 'type': 'file', - 'include': True} + (community_total, community_results) = files_search( + client, + qs, + community_conf["system"], + community_conf.get("homeDir", "/"), + filter=filter, + offset=offset, + limit=limit, + ) + response["community"] = { + "count": community_total, + "listing": community_results, + "type": "file", + "include": True, + } except StopIteration: pass except BaseTapyException as e: @@ -110,10 +111,6 @@ def _handle_tapis_ssh_exception(self, e): # in case of these error types, user is not authenticated # or does not have access do not fail the entire search # request, log the issue. - logger.exception( - "Error retrieving search results due to TAPIS SSH related error: {}".format( - str(e) - ) - ) + logger.exception("Error retrieving search results due to TAPIS SSH related error: {}".format(str(e))) else: raise diff --git a/server/portal/apps/site_search/apps.py b/server/portal/apps/site_search/apps.py index 78b704a7ff..d5d0151e2c 100644 --- a/server/portal/apps/site_search/apps.py +++ b/server/portal/apps/site_search/apps.py @@ -2,4 +2,4 @@ class SiteSearchConfig(AppConfig): - name = 'portal.apps.site_search' + name = "portal.apps.site_search" diff --git a/server/portal/apps/site_search/urls.py b/server/portal/apps/site_search/urls.py index 6604917b29..5a22c7a7de 100644 --- a/server/portal/apps/site_search/urls.py +++ b/server/portal/apps/site_search/urls.py @@ -2,10 +2,11 @@ .. module:: portal.apps.site_search.urls :synopsis: Site Search URLs """ + from django.urls import re_path from portal.apps.site_search.views import IndexView -app_name = 'site_search' +app_name = "site_search" urlpatterns = [ - re_path('', IndexView.as_view(), name='index'), + re_path("", IndexView.as_view(), name="index"), ] diff --git a/server/portal/apps/site_search/views.py b/server/portal/apps/site_search/views.py index 6ad9488333..6d37d651be 100644 --- a/server/portal/apps/site_search/views.py +++ b/server/portal/apps/site_search/views.py @@ -6,14 +6,16 @@ class IndexView(TemplateView): """ Main workbench view. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def dispatch(self, request, *args, **kwargs): return super(IndexView, self).dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - context['setup_complete'] = False if self.request.user.is_anonymous \ - else self.request.user.profile.setup_complete - context['DEBUG'] = settings.DEBUG + context["setup_complete"] = ( + False if self.request.user.is_anonymous else self.request.user.profile.setup_complete + ) + context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/system_monitor/apps.py b/server/portal/apps/system_monitor/apps.py index 970ef9ff46..70cb1cfa60 100644 --- a/server/portal/apps/system_monitor/apps.py +++ b/server/portal/apps/system_monitor/apps.py @@ -2,4 +2,4 @@ class SysmonConfig(AppConfig): - name = 'portal.apps.system_monitor' + name = "portal.apps.system_monitor" diff --git a/server/portal/apps/system_monitor/unit_test.py b/server/portal/apps/system_monitor/unit_test.py index 547d46710f..1bf320ff4b 100644 --- a/server/portal/apps/system_monitor/unit_test.py +++ b/server/portal/apps/system_monitor/unit_test.py @@ -7,56 +7,56 @@ @pytest.fixture def system_status(scope="module"): - with open(os.path.join(settings.BASE_DIR, 'fixtures/system_monitor/index.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/system_monitor/index.json")) as f: yield json.load(f) @pytest.fixture def system_status_missing_frontera(scope="module"): - with open(os.path.join(settings.BASE_DIR, 'fixtures/system_monitor/index_missing_frontera.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/system_monitor/index_missing_frontera.json")) as f: yield json.load(f) @pytest.mark.django_db() def test_system_monitor_get(client, settings, requests_mock, system_status): - settings.SYSTEM_MONITOR_DISPLAY_LIST = ['Frontera'] + settings.SYSTEM_MONITOR_DISPLAY_LIST = ["Frontera"] requests_mock.get(settings.SYSTEM_MONITOR_URL, json=system_status) - response = client.get('/api/system-monitor/') + response = client.get("/api/system-monitor/") assert response.status_code == 200 system = response.json()[0] - assert system['display_name'] == 'Frontera' - assert system['hostname'] == 'frontera.tacc.utexas.edu' - assert system['load_percentage'] == 97 - assert system['jobs'] == {'running': 365, 'queued': 247} - assert system['is_operational'] + assert system["display_name"] == "Frontera" + assert system["hostname"] == "frontera.tacc.utexas.edu" + assert system["load_percentage"] == 97 + assert system["jobs"] == {"running": 365, "queued": 247} + assert system["is_operational"] @pytest.mark.django_db() def test_system_monitor_when_missing_system(client, settings, requests_mock, system_status_missing_frontera): - settings.SYSTEM_MONITOR_DISPLAY_LIST = ['Frontera'] + settings.SYSTEM_MONITOR_DISPLAY_LIST = ["Frontera"] requests_mock.get(settings.SYSTEM_MONITOR_URL, json=system_status_missing_frontera) - response = client.get('/api/system-monitor/') + response = client.get("/api/system-monitor/") assert response.status_code == 200 system = response.json()[0] - assert system['hostname'] == 'frontera.tacc.utexas.edu' - assert system['display_name'] == 'Frontera' - assert not system['is_operational'] - assert system['jobs'] == {'running': 0, 'queued': 0} - assert system['load_percentage'] == 0 + assert system["hostname"] == "frontera.tacc.utexas.edu" + assert system["display_name"] == "Frontera" + assert not system["is_operational"] + assert system["jobs"] == {"running": 0, "queued": 0} + assert system["load_percentage"] == 0 @pytest.mark.django_db() def test_system_monitor_when_display_list_is_empty(client, settings, requests_mock, system_status): settings.SYSTEM_MONITOR_DISPLAY_LIST = [] requests_mock.get(settings.SYSTEM_MONITOR_URL, json=system_status) - response = client.get('/api/system-monitor/') + response = client.get("/api/system-monitor/") assert response.status_code == 200 assert response.json() == [] @pytest.mark.django_db() def test_system_monitor_when_status_endpoint_fails(client, settings, requests_mock): - settings.SYSTEM_MONITOR_DISPLAY_LIST = ['Frontera'] + settings.SYSTEM_MONITOR_DISPLAY_LIST = ["Frontera"] requests_mock.get(settings.SYSTEM_MONITOR_URL, exc=Http404) - response = client.get('/api/system-monitor/') + response = client.get("/api/system-monitor/") assert response.status_code == 404 diff --git a/server/portal/apps/system_monitor/urls.py b/server/portal/apps/system_monitor/urls.py index 109437b56a..9e608820fc 100644 --- a/server/portal/apps/system_monitor/urls.py +++ b/server/portal/apps/system_monitor/urls.py @@ -1,9 +1,8 @@ - from django.urls import path from portal.apps.system_monitor import views -app_name = 'system_monitor' +app_name = "system_monitor" urlpatterns = [ - path('', views.SysmonDataView.as_view(), name='system_monitor'), - path('', views.SysmonDataView.as_view(), name='system_monitor'), + path("", views.SysmonDataView.as_view(), name="system_monitor"), + path("", views.SysmonDataView.as_view(), name="system_monitor"), ] diff --git a/server/portal/apps/system_monitor/views.py b/server/portal/apps/system_monitor/views.py index 12016102f2..b708af2619 100644 --- a/server/portal/apps/system_monitor/views.py +++ b/server/portal/apps/system_monitor/views.py @@ -9,30 +9,30 @@ def _get_unoperational_system(display_name): - return {'display_name': display_name, - 'hostname': display_name.lower() + '.tacc.utexas.edu', - 'is_operational': False, - 'load_percentage': 0, - 'jobs': {'running': 0, 'queued': 0}, - } + return { + "display_name": display_name, + "hostname": display_name.lower() + ".tacc.utexas.edu", + "is_operational": False, + "load_percentage": 0, + "jobs": {"running": 0, "queued": 0}, + } class SysmonDataView(BaseApiView): - def get(self, request, system_name=None): - ''' - Pulls and parses data from TACC User Portal then populates and returns a list of Systems objects - ''' + """ + Pulls and parses data from TACC User Portal then populates and returns a list of Systems objects + """ if system_name: - system_json = requests.get(f'{settings.SYSTEM_MONITOR_URL}{system_name}').json() + system_json = requests.get(f"{settings.SYSTEM_MONITOR_URL}{system_name}").json() requested_systems = settings.SYSTEM_MONITOR_DISPLAY_LIST - if (system_json['display_name'] in requested_systems): + if system_json["display_name"] in requested_systems: system_queues = [] - for queue in system_json['queues'].items(): - system_queues.append({'name': queue[0], **queue[1]}) + for queue in system_json["queues"].items(): + system_queues.append({"name": queue[0], **queue[1]}) return JsonResponse(system_queues, safe=False) else: @@ -41,38 +41,39 @@ def get(self, request, system_name=None): systems_json = requests.get(settings.SYSTEM_MONITOR_URL).json() for sys in requested_systems: if sys not in systems_json: - logger.info('System information for {} is missing. Assuming not operational status.'.format(sys)) + logger.info("System information for {} is missing. Assuming not operational status.".format(sys)) systems.append(_get_unoperational_system(sys)) continue try: system = System(systems_json[sys]).to_dict() systems.append(system) except Exception: - logger.exception('Problem gather system information for {}: Assuming not operational status'.format(sys)) + logger.exception( + "Problem gather system information for {}: Assuming not operational status".format(sys) + ) systems.append(_get_unoperational_system(sys)) return JsonResponse(systems, safe=False) class System: - def __init__(self, system_dict): try: - self.display_name = system_dict.get('display_name') - self.hostname = system_dict.get('hostname') + self.display_name = system_dict.get("display_name") + self.hostname = system_dict.get("hostname") self.resource_type = system_dict.get("system_type") - self.load_percentage = system_dict.get('load') + self.load_percentage = system_dict.get("load") if isinstance(self.load_percentage, (float, int)): self.load_percentage = int(self.load_percentage * 100) else: self.load_percentage = 0 self.jobs = { - 'running': system_dict.get('running'), - 'queued': system_dict.get('waiting'), + "running": system_dict.get("running"), + "queued": system_dict.get("waiting"), } - self.online = system_dict.get('online') - self.reachable = system_dict.get('reachable') - self.queues_down = system_dict.get('queues_down') - self.in_maintenance = system_dict.get('in_maintenance') + self.online = system_dict.get("online") + self.reachable = system_dict.get("reachable") + self.queues_down = system_dict.get("queues_down") + self.in_maintenance = system_dict.get("in_maintenance") self.is_operational = self.is_up() except Exception as exc: logger.error(exc) diff --git a/server/portal/apps/tickets/api/unit_test.py b/server/portal/apps/tickets/api/unit_test.py index 3dfc32c75e..16972b382d 100644 --- a/server/portal/apps/tickets/api/unit_test.py +++ b/server/portal/apps/tickets/api/unit_test.py @@ -7,18 +7,18 @@ @pytest.fixture def rt_tickets(scope="module"): - yield json.load(open(os.path.join(settings.BASE_DIR, 'fixtures/rt/tickets.json'))) + yield json.load(open(os.path.join(settings.BASE_DIR, "fixtures/rt/tickets.json"))) @pytest.fixture def rt_ticket_history(scope="module"): - yield json.load(open(os.path.join(settings.BASE_DIR, 'fixtures/rt/ticket_history.json'))) + yield json.load(open(os.path.join(settings.BASE_DIR, "fixtures/rt/ticket_history.json"))) @pytest.fixture def mock_invalid_recaptcha(requests_mock): - recaptchaSuccess = {'success': False, 'challenge_ts': '2021-11-23T17:58:27Z', 'hostname': 'testkey.google.com'} - requests_mock.post('https://www.google.com/recaptcha/api/siteverify', json=recaptchaSuccess) + recaptchaSuccess = {"success": False, "challenge_ts": "2021-11-23T17:58:27Z", "hostname": "testkey.google.com"} + requests_mock.post("https://www.google.com/recaptcha/api/siteverify", json=recaptchaSuccess) @pytest.fixture @@ -34,133 +34,140 @@ def mock_rt(mocker, rt_tickets, rt_ticket_history): @pytest.fixture def mock_rtutil(mocker, mock_rt): - mocker.patch('portal.apps.tickets.api.views.rtUtil.DjangoRt', return_value=mock_rt) + mocker.patch("portal.apps.tickets.api.views.rtUtil.DjangoRt", return_value=mock_rt) yield mock_rt @pytest.fixture def mock_rtutil_no_access(mocker, mock_rt): mock_rt.hasAccess.return_value = False - mocker.patch('portal.apps.tickets.api.views.rtUtil.DjangoRt', return_value=mock_rt) + mocker.patch("portal.apps.tickets.api.views.rtUtil.DjangoRt", return_value=mock_rt) yield mock_rt @pytest.fixture def mock_get_matching_history_entry(mocker, rt_ticket_history): last_entry = rt_ticket_history[-1] - mocker.patch('portal.apps.tickets.api.views.TicketsHistoryView._get_matching_history_entry', return_value=last_entry) + mocker.patch( + "portal.apps.tickets.api.views.TicketsHistoryView._get_matching_history_entry", return_value=last_entry + ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_get(client, authenticated_user, mock_rtutil): - response = client.get('/api/tickets/') + response = client.get("/api/tickets/") assert response.status_code == 200 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create_missing_required_description_or_subject(client, authenticated_user, mock_rtutil): # missing subject - response = client.post('/api/tickets/', - data={"problem_description": "problem_description"}) + response = client.post("/api/tickets/", data={"problem_description": "problem_description"}) assert response.status_code == 400 # missing problem_description - response = client.post('/api/tickets/', - data={"subject": "subject"}) + response = client.post("/api/tickets/", data={"subject": "subject"}) assert response.status_code == 400 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create_missing_required_email(client, regular_user, mock_rtutil): # missing user email for unauthenticated user - response = client.post('/api/tickets/', - data={"problem_description": "problem_description", - "subject": "subject"}) + response = client.post("/api/tickets/", data={"problem_description": "problem_description", "subject": "subject"}) assert response.status_code == 400 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create(client, authenticated_user, mock_rtutil): - response = client.post('/api/tickets/', - data={"problem_description": "problem_description", - "subject": "subject"}) + response = client.post("/api/tickets/", data={"problem_description": "problem_description", "subject": "subject"}) assert response.status_code == 200 result = json.loads(response.content) - assert result['ticket_id'] == 1 + assert result["ticket_id"] == 1 _, kwargs = mock_rtutil.create_ticket.call_args - assert len(kwargs['attachments']) == 0 - assert kwargs['problem_description'].startswith("problem_description") - assert kwargs['requestor'] == authenticated_user.email - assert kwargs['subject'] == "subject" + assert len(kwargs["attachments"]) == 0 + assert kwargs["problem_description"].startswith("problem_description") + assert kwargs["requestor"] == authenticated_user.email + assert kwargs["subject"] == "subject" # check that some user info is added to metadata in problem_description - assert authenticated_user.first_name in kwargs['problem_description'] - assert authenticated_user.last_name in kwargs['problem_description'] - assert authenticated_user.username in kwargs['problem_description'] + assert authenticated_user.first_name in kwargs["problem_description"] + assert authenticated_user.last_name in kwargs["problem_description"] + assert authenticated_user.username in kwargs["problem_description"] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create_unauthencated(client, regular_user, mock_rtutil): - response = client.post('/api/tickets/', - data={"problem_description": "problem_description", - "email": "email@test.com", - "subject": "subject", - "first_name": "first_name", - "last_name": "last_name"}) + response = client.post( + "/api/tickets/", + data={ + "problem_description": "problem_description", + "email": "email@test.com", + "subject": "subject", + "first_name": "first_name", + "last_name": "last_name", + }, + ) assert response.status_code == 200 result = json.loads(response.content) - assert result['ticket_id'] == 1 + assert result["ticket_id"] == 1 _, kwargs = mock_rtutil.create_ticket.call_args - assert len(kwargs['attachments']) == 0 - assert kwargs['problem_description'].startswith("problem_description") - assert kwargs['requestor'] == "email@test.com" - assert kwargs['subject'] == "subject" + assert len(kwargs["attachments"]) == 0 + assert kwargs["problem_description"].startswith("problem_description") + assert kwargs["requestor"] == "email@test.com" + assert kwargs["subject"] == "subject" # check that some user info is added to metadata in problem_description - assert "first_name" in kwargs['problem_description'] - assert "last_name" in kwargs['problem_description'] + assert "first_name" in kwargs["problem_description"] + assert "last_name" in kwargs["problem_description"] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create_unauthencated_invalid_recaptcha(client, regular_user, mock_rtutil, mock_invalid_recaptcha): - response = client.post('/api/tickets/', - data={"problem_description": "problem_description", - "email": "email@test.com", - "subject": "subject", - "first_name": "first_name", - "last_name": "last_name"} - ) + response = client.post( + "/api/tickets/", + data={ + "problem_description": "problem_description", + "email": "email@test.com", + "subject": "subject", + "first_name": "first_name", + "last_name": "last_name", + }, + ) assert response.status_code == 400 - assert json.loads(response.content) == {'message': 'Invalid reCAPTCHA. Please try again.'} + assert json.loads(response.content) == {"message": "Invalid reCAPTCHA. Please try again."} @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_create_with_attachments(client, authenticated_user, mock_rtutil): - attachment = (io.BytesIO(b"abcdef"), 'test.jpg') - response = client.post('/api/tickets/', - data={"problem_description": "problem_description", - "email": "email@test.com", - "cc": "cc@test.com", - "subject": "subject", - "first_name": "firstName", - "last_name": "lastName", - 'attachments': attachment}, - format="multipart") + attachment = (io.BytesIO(b"abcdef"), "test.jpg") + response = client.post( + "/api/tickets/", + data={ + "problem_description": "problem_description", + "email": "email@test.com", + "cc": "cc@test.com", + "subject": "subject", + "first_name": "firstName", + "last_name": "lastName", + "attachments": attachment, + }, + format="multipart", + ) assert response.status_code == 200 result = json.loads(response.content) - assert result['ticket_id'] == 1 + assert result["ticket_id"] == 1 _, kwargs = mock_rtutil.create_ticket.call_args - assert len(kwargs['attachments']) == 1 - assert kwargs['problem_description'].startswith("problem_description") - assert kwargs['requestor'] == authenticated_user.email - assert kwargs['subject'] == "subject" - assert kwargs['cc'] == "cc@test.com" + assert len(kwargs["attachments"]) == 1 + assert kwargs["problem_description"].startswith("problem_description") + assert kwargs["requestor"] == authenticated_user.email + assert kwargs["subject"] == "subject" + assert kwargs["cc"] == "cc@test.com" # check that some user info is added to metadata in problem_description - assert authenticated_user.first_name in kwargs['problem_description'] - assert authenticated_user.last_name in kwargs['problem_description'] + assert authenticated_user.first_name in kwargs["problem_description"] + assert authenticated_user.last_name in kwargs["problem_description"] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_get_history(client, authenticated_user, mock_rtutil): - response = client.get('/api/tickets/1/history') + response = client.get("/api/tickets/1/history") assert response.status_code == 200 result = json.loads(response.content) assert len(result["ticket_history"]) == 5 @@ -172,13 +179,15 @@ def test_tickets_get_history(client, authenticated_user, mock_rtutil): @pytest.mark.django_db(transaction=True, reset_sequences=True) @pytest.mark.parametrize("service_account", ["portal", "rtdev", "rtprod"]) -def test_tickets_get_history_handle_service_accounts(service_account, client, authenticated_user, mock_rtutil, rt_ticket_history): +def test_tickets_get_history_handle_service_accounts( + service_account, client, authenticated_user, mock_rtutil, rt_ticket_history +): for i in [6, 9]: # two messages from service accounts # set to different service account rt_ticket_history[i]["Creator"] = service_account mock_rtutil.getTicketHistory.return_value = rt_ticket_history - response = client.get('/api/tickets/1/history') + response = client.get("/api/tickets/1/history") assert response.status_code == 200 result = json.loads(response.content) full_name = "{} {}".format(authenticated_user.first_name, authenticated_user.last_name) @@ -189,53 +198,56 @@ def test_tickets_get_history_handle_service_accounts(service_account, client, au @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_get_history_no_access(client, authenticated_user, mock_rtutil_no_access): - response = client.get('/api/tickets/1/history') + response = client.get("/api/tickets/1/history") assert response.status_code == 403 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_post_history_reply_with_text(client, authenticated_user, mock_rtutil, mock_get_matching_history_entry): - response = client.post('/api/tickets/1/history', - data={"reply": "reply text"}) + response = client.post("/api/tickets/1/history", data={"reply": "reply text"}) assert response.status_code == 200 - mock_rtutil.replyToTicket.assert_called_with(ticket_id=1, - files=[], - reply_text="reply text\n[Reply submitted on behalf of {}]".format( - authenticated_user.username)) + mock_rtutil.replyToTicket.assert_called_with( + ticket_id=1, + files=[], + reply_text="reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username), + ) @pytest.mark.django_db(transaction=True, reset_sequences=True) -def test_tickets_post_history_reply_with_multiline_text(client, authenticated_user, mock_rtutil, mock_get_matching_history_entry): - response = client.post('/api/tickets/1/history', - data={"reply": "reply text"}) +def test_tickets_post_history_reply_with_multiline_text( + client, authenticated_user, mock_rtutil, mock_get_matching_history_entry +): + response = client.post("/api/tickets/1/history", data={"reply": "reply text"}) assert response.status_code == 200 - mock_rtutil.replyToTicket.assert_called_with(ticket_id=1, - files=[], - reply_text="reply text\n[Reply submitted on behalf of {}]".format( - authenticated_user.username)) + mock_rtutil.replyToTicket.assert_called_with( + ticket_id=1, + files=[], + reply_text="reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username), + ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_post_history_reply_missing_text(client, authenticated_user, mock_rtutil): - response = client.post('/api/tickets/1/history') + response = client.post("/api/tickets/1/history") assert response.status_code == 400 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_tickets_post_history_reply_no_access(client, authenticated_user, mock_rtutil_no_access): - response = client.post('/api/tickets/1/history') + response = client.post("/api/tickets/1/history") assert response.status_code == 403 @pytest.mark.django_db(transaction=True, reset_sequences=True) -def test_tickets_post_history_reply_with_text_and_attachment(client, authenticated_user, mock_rtutil, mock_get_matching_history_entry): - attachment = (io.BytesIO(b"abcdef"), 'test.jpg') - response = client.post('/api/tickets/1/history', - data={"reply": "reply text", - 'attachments': attachment}, - format="multipart") +def test_tickets_post_history_reply_with_text_and_attachment( + client, authenticated_user, mock_rtutil, mock_get_matching_history_entry +): + attachment = (io.BytesIO(b"abcdef"), "test.jpg") + response = client.post( + "/api/tickets/1/history", data={"reply": "reply text", "attachments": attachment}, format="multipart" + ) assert response.status_code == 200 _, kwargs = mock_rtutil.replyToTicket.call_args - assert kwargs['ticket_id'] == 1 - assert len(kwargs['files']) == 1 - assert kwargs['reply_text'] == "reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username) + assert kwargs["ticket_id"] == 1 + assert len(kwargs["files"]) == 1 + assert kwargs["reply_text"] == "reply text\n[Reply submitted on behalf of {}]".format(authenticated_user.username) diff --git a/server/portal/apps/tickets/api/urls.py b/server/portal/apps/tickets/api/urls.py index 6e88b9ca6c..b9678d5e71 100644 --- a/server/portal/apps/tickets/api/urls.py +++ b/server/portal/apps/tickets/api/urls.py @@ -1,10 +1,10 @@ from django.urls import path from portal.apps.tickets.api import views -app_name = 'portal_tickets_api' +app_name = "portal_tickets_api" urlpatterns = [ - path('', views.TicketsView.as_view()), - path('', views.TicketsView.as_view()), - path('/history', views.TicketsHistoryView.as_view()), - path('/attachment/', views.TicketsAttachmentView.as_view()) + path("", views.TicketsView.as_view()), + path("", views.TicketsView.as_view()), + path("/history", views.TicketsHistoryView.as_view()), + path("/attachment/", views.TicketsAttachmentView.as_view()), ] diff --git a/server/portal/apps/tickets/api/views.py b/server/portal/apps/tickets/api/views.py index a9c8585e1a..47f48d6288 100644 --- a/server/portal/apps/tickets/api/views.py +++ b/server/portal/apps/tickets/api/views.py @@ -22,9 +22,7 @@ class TicketsView(BaseApiView): def get(self, request, ticket_id=None): - """Get a list of all tickets for a user or a single ticket - - """ + """Get a list of all tickets for a user or a single ticket""" if not request.user.is_authenticated: raise PermissionDenied @@ -33,27 +31,24 @@ def get(self, request, ticket_id=None): if not rt.hasAccess(ticket_id, request.user.email): raise PermissionDenied ticket = rt.getTicket(ticket_id) - return JsonResponse({'tickets': [ticket]}) + return JsonResponse({"tickets": [ticket]}) else: user_tickets = rt.getUserTickets(request.user.email) - return JsonResponse({'tickets': user_tickets}) + return JsonResponse({"tickets": user_tickets}) def post(self, request): - """Post a new ticket - - """ + """Post a new ticket""" data = request.POST.copy() - subject = data.get('subject') - problem_description = data.get('problem_description') - cc = data.get('cc', []) - attachments = [(f.name, ContentFile(f.read()), f.content_type) - for f in request.FILES.getlist('attachments')] - info = request.GET.get('info', "None") + subject = data.get("subject") + problem_description = data.get("problem_description") + cc = data.get("cc", []) + attachments = [(f.name, ContentFile(f.read()), f.content_type) for f in request.FILES.getlist("attachments")] + info = request.GET.get("info", "None") meta = request.META is_authenticated = request.user.is_authenticated username = None - if (is_authenticated): + if is_authenticated: username = request.user.username email = request.user.email first_name = request.user.first_name @@ -61,22 +56,15 @@ def post(self, request): else: if settings.RECAPTCHA_SECRET_KEY: recap_result = utils.get_recaptcha_verification(request) - if not recap_result.get('success', False): - raise ApiException('Invalid reCAPTCHA. Please try again.') - email = data.get('email') - first_name = data.get('first_name') - last_name = data.get('last_name') - - return utils.create_ticket(username, - first_name, - last_name, - email, - cc, - subject, - problem_description, - attachments, - info, - meta) + if not recap_result.get("success", False): + raise ApiException("Invalid reCAPTCHA. Please try again.") + email = data.get("email") + first_name = data.get("first_name") + last_name = data.get("last_name") + + return utils.create_ticket( + username, first_name, last_name, email, cc, subject, problem_description, attachments, info, meta + ) def has_access_to_ticket(function): @@ -93,11 +81,10 @@ def wrapper(*args, **kwargs): return wrapper -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class TicketsHistoryView(BaseApiView): - def _get_ticket_history(self, rt, requesting_username, ticket_id): - """ Get ticket history + """Get ticket history Returns history of ticket. Complete RT history is not returned rather a subset of history types (see `ALLOWED_HISTORY_TYPES`). @@ -108,42 +95,47 @@ def _get_ticket_history(self, rt, requesting_username, ticket_id): :return: return ticket history """ ticket_history = rt.getTicketHistory(ticket_id) - ticket_history = list(filter(lambda h: h['Type'] in ALLOWED_HISTORY_TYPES, ticket_history)) + ticket_history = list(filter(lambda h: h["Type"] in ALLOWED_HISTORY_TYPES, ticket_history)) for entry in ticket_history: - if entry['Type'] == "Status": - entry['Content'] = entry['Description'] + if entry["Type"] == "Status": + entry["Content"] = entry["Description"] # Determine who created this message using portal - if entry['Creator'] in SERVICE_ACCOUNTS: + if entry["Creator"] in SERVICE_ACCOUNTS: # Check if its a reply submitted on behalf of a user - submitted_for_user = re.search(r'\[Reply submitted on behalf of (.*?)\]', - entry['Content'].splitlines()[-1]) if entry['Content'] else False + submitted_for_user = ( + re.search(r"\[Reply submitted on behalf of (.*?)\]", entry["Content"].splitlines()[-1]) + if entry["Content"] + else False + ) if submitted_for_user: - entry['Creator'] = submitted_for_user.group(1) - entry["Content"] = entry['Content'][:entry['Content'].rfind('\n')] + entry["Creator"] = submitted_for_user.group(1) + entry["Content"] = entry["Content"][: entry["Content"].rfind("\n")] # if user info is in the ticket metadata - if not submitted_for_user and entry['Type'] == "Create": - submitted_for_user = re.findall(r'authenticated_user:[\r\n]+([^\r\n]+)', - entry['Content'], re.MULTILINE) if entry['Content'] else False + if not submitted_for_user and entry["Type"] == "Create": + submitted_for_user = ( + re.findall(r"authenticated_user:[\r\n]+([^\r\n]+)", entry["Content"], re.MULTILINE) + if entry["Content"] + else False + ) if submitted_for_user: - entry['Creator'] = submitted_for_user[-1] + entry["Creator"] = submitted_for_user[-1] - if entry['Type'] == "Create": - entry["Content"] = entry['Content'][:entry['Content'].rfind(METADATA_HEADER)] + if entry["Type"] == "Create": + entry["Content"] = entry["Content"][: entry["Content"].rfind(METADATA_HEADER)] - entry['Creator'] = "RT System" if entry['Creator'] == "RT_System" else entry['Creator'] + entry["Creator"] = "RT System" if entry["Creator"] == "RT_System" else entry["Creator"] - entry["IsCreator"] = True if requesting_username == entry['Creator'] else False + entry["IsCreator"] = True if requesting_username == entry["Creator"] else False - known_user = get_user_model().objects.filter(username=entry['Creator']).first() + known_user = get_user_model().objects.filter(username=entry["Creator"]).first() if known_user: - entry['Creator'] = "{} {}".format(known_user.first_name, known_user.last_name) + entry["Creator"] = "{} {}".format(known_user.first_name, known_user.last_name) return ticket_history def _get_matching_history_entry(self, ticket_history, content): - """ Find most-recent ticket history entry that matches certain content - """ + """Find most-recent ticket history entry that matches certain content""" for entry in reversed(ticket_history): if entry["IsCreator"] and entry["Content"] == content: return entry @@ -151,18 +143,16 @@ def _get_matching_history_entry(self, ticket_history, content): @has_access_to_ticket def post(self, request, ticket_id): - """ Post reply to ticket - - """ + """Post reply to ticket""" data = request.POST.copy() - reply = data.get('reply') + reply = data.get("reply") if reply is None: return HttpResponseBadRequest() # Add information on which user submitted this reply (as this is being done by a service account) modified_reply = reply + "\n[Reply submitted on behalf of {}]".format(request.user.username) - attachments = [(f.name, ContentFile(f.read()), f.content_type) for f in request.FILES.getlist('attachments')] + attachments = [(f.name, ContentFile(f.read()), f.content_type) for f in request.FILES.getlist("attachments")] rt = rtUtil.DjangoRt() result = rt.replyToTicket(ticket_id=ticket_id, reply_text=modified_reply, files=attachments) @@ -176,15 +166,14 @@ def post(self, request, ticket_id): if not history_reply: raise ApiException("Unable to reply to ticket.") - return JsonResponse({'ticket_history_reply': history_reply}) + return JsonResponse({"ticket_history_reply": history_reply}) @has_access_to_ticket def get(self, request, ticket_id): - """Get ticket history - """ + """Get ticket history""" rt = rtUtil.DjangoRt() ticket_history = self._get_ticket_history(rt, request.user.username, ticket_id) - return JsonResponse({'ticket_history': ticket_history}) + return JsonResponse({"ticket_history": ticket_history}) class TicketsAttachmentView(BaseApiView): diff --git a/server/portal/apps/tickets/rtUtil.py b/server/portal/apps/tickets/rtUtil.py index 13467d466b..a9d871d4bc 100644 --- a/server/portal/apps/tickets/rtUtil.py +++ b/server/portal/apps/tickets/rtUtil.py @@ -6,31 +6,32 @@ class DjangoRt: def __init__(self): - self.rtHost = getattr(settings, 'RT_HOST') - self.rtUn = getattr(settings, 'RT_UN') - self.rtPw = getattr(settings, 'RT_PW') - self.rtQueue = getattr(settings, 'RT_QUEUE', '') + self.rtHost = getattr(settings, "RT_HOST") + self.rtUn = getattr(settings, "RT_UN") + self.rtPw = getattr(settings, "RT_PW") + self.rtQueue = getattr(settings, "RT_QUEUE", "") self.tracker = rt.Rt(self.rtHost, self.rtUn, self.rtPw, http_auth=HTTPBasicAuth(self.rtUn, self.rtPw)) self.tracker.login() def getUserTickets(self, userEmail, status="ALL"): if not status == "ALL": - ticket_list = self.tracker.search(Queue=rt.ALL_QUEUES, Requestors__exact=userEmail, Status__exact=status, - order='-LastUpdated') + ticket_list = self.tracker.search( + Queue=rt.ALL_QUEUES, Requestors__exact=userEmail, Status__exact=status, order="-LastUpdated" + ) else: - ticket_list = self.tracker.search(Queue=rt.ALL_QUEUES, Requestors__exact=userEmail, order='-LastUpdated') + ticket_list = self.tracker.search(Queue=rt.ALL_QUEUES, Requestors__exact=userEmail, order="-LastUpdated") for ticket in ticket_list: - ticket['id'] = ticket['id'].replace('ticket/', '') - ticket['LastUpdated'] = datetime.strptime(ticket['LastUpdated'], '%a %b %d %X %Y') + ticket["id"] = ticket["id"].replace("ticket/", "") + ticket["LastUpdated"] = datetime.strptime(ticket["LastUpdated"], "%a %b %d %X %Y") return ticket_list def getTicket(self, ticket_id): ticket = self.tracker.get_ticket(ticket_id) - ticket['id'] = ticket['id'].replace('ticket/', '') + ticket["id"] = ticket["id"].replace("ticket/", "") return ticket @@ -38,18 +39,20 @@ def getTicketHistory(self, ticket_id): ticketHistory = self.tracker.get_history(ticket_id) for ticket in ticketHistory: - ticket['Created'] = datetime.strptime(ticket['Created'], '%Y-%m-%d %X') + ticket["Created"] = datetime.strptime(ticket["Created"], "%Y-%m-%d %X") return ticketHistory def create_ticket(self, attachments, subject, problem_description, requestor, cc): - return self.tracker.create_ticket(Queue=self.rtQueue, - files=attachments, - Subject=subject, - Text=problem_description, - Requestor=requestor, - Cc=cc, - CF_resource=settings.RT_TAG) + return self.tracker.create_ticket( + Queue=self.rtQueue, + files=attachments, + Subject=subject, + Text=problem_description, + Requestor=requestor, + Cc=cc, + CF_resource=settings.RT_TAG, + ) def replyToTicket(self, ticket_id, reply_text, files=[]): return self.tracker.reply(ticket_id=ticket_id, text=reply_text, files=files) @@ -60,7 +63,9 @@ def replyToTicket(self, ticket_id, reply_text, files=[]): def hasAccess(self, ticket_id, user=None): if user and ticket_id: ticket = self.tracker.get_ticket(ticket_id) - if DjangoRt.contains_user(ticket.get('Requestors', ''), user) or DjangoRt.contains_user(ticket.get('Cc', ''), user): + if DjangoRt.contains_user(ticket.get("Requestors", ""), user) or DjangoRt.contains_user( + ticket.get("Cc", ""), user + ): return True return False diff --git a/server/portal/apps/tickets/unit_test.py b/server/portal/apps/tickets/unit_test.py index e7049d4ec4..eacd97a0a0 100644 --- a/server/portal/apps/tickets/unit_test.py +++ b/server/portal/apps/tickets/unit_test.py @@ -7,24 +7,23 @@ @pytest.fixture(autouse=True) def mock_render(mocker): - yield mocker.patch('portal.apps.tickets.views.render', return_value=HttpResponse("OK")) + yield mocker.patch("portal.apps.tickets.views.render", return_value=HttpResponse("OK")) def test_tickets_get(client, authenticated_user): - response = client.get('/tickets/') + response = client.get("/tickets/") assert response.status_code == 302 - assert response.url == '/workbench/dashboard/' + assert response.url == "/workbench/dashboard/" def test_ticket_create_authenticated(client, regular_user): - """Users who are setup_complete may use workbench/dashboard routes - """ + """Users who are setup_complete may use workbench/dashboard routes""" regular_user.profile.setup_complete = True regular_user.profile.save() client.force_login(regular_user) - response = client.get('/tickets/new/') + response = client.get("/tickets/new/") assert response.status_code == 302 - assert response.url == '/workbench/dashboard/tickets/create/' + assert response.url == "/workbench/dashboard/tickets/create/" def test_ticket_create_authenticated_setup_incomplete(client, regular_user): @@ -35,31 +34,31 @@ def test_ticket_create_authenticated_setup_incomplete(client, regular_user): regular_user.profile.setup_complete = False regular_user.profile.save() client.force_login(regular_user) - response = client.get('/tickets/new/') + response = client.get("/tickets/new/") assert response.status_code == 200 def test_get_recaptcha_verification(mocker, requests_mock, regular_user): - recaptchaSuccess = {'success': True, 'challenge_ts': '2021-11-23T17:58:27Z', 'hostname': 'testkey.google.com'} - requests_mock.post('https://www.google.com/recaptcha/api/siteverify', json=recaptchaSuccess) + recaptchaSuccess = {"success": True, "challenge_ts": "2021-11-23T17:58:27Z", "hostname": "testkey.google.com"} + requests_mock.post("https://www.google.com/recaptcha/api/siteverify", json=recaptchaSuccess) request = HttpRequest() - request.method = 'POST' - request.POST['recaptchaResponse'] = 'string' + request.method = "POST" + request.POST["recaptchaResponse"] = "string" result = get_recaptcha_verification(request) - assert result['success'] == recaptchaSuccess['success'] + assert result["success"] == recaptchaSuccess["success"] class RtUtilTestable(rtUtil.DjangoRt): - ''' + """ Tester for rtUtil.DjangoRt. - ''' + """ def __init__(self, tracker): # Set the attributes directly - self.rtHost = 'mock_host' - self.rtUn = 'mock_rt_user' - self.rtPw = 'mock_pw' - self.rtQueue = '' + self.rtHost = "mock_host" + self.rtUn = "mock_rt_user" + self.rtPw = "mock_pw" + self.rtQueue = "" self.tracker = tracker @@ -75,33 +74,48 @@ def mock_tracker(mocker, rt_ticket): yield mock_tracker -@pytest.mark.parametrize('rt_ticket', [ - {'id': 1, 'Requestors': ["UserName1@Example.COM", "Username2@Example.com"], 'Cc': []}, - {'id': 1, 'Requestors': "UserName1@Example.COM,Username2@Example.com", 'Cc': []}, - {'id': 1, 'Requestors': ["username1@example.com", "username2@example.com"], 'Cc': []}], indirect=True) +@pytest.mark.parametrize( + "rt_ticket", + [ + {"id": 1, "Requestors": ["UserName1@Example.COM", "Username2@Example.com"], "Cc": []}, + {"id": 1, "Requestors": "UserName1@Example.COM,Username2@Example.com", "Cc": []}, + {"id": 1, "Requestors": ["username1@example.com", "username2@example.com"], "Cc": []}, + ], + indirect=True, +) def test_rt_hasaccess_requestors_or_cc(mock_tracker): rtTester = RtUtilTestable(mock_tracker) - assert rtTester.hasAccess(1, 'Username1@Example.com') is True - assert rtTester.hasAccess(1, 'Username2@Example.com') is True - - -@pytest.mark.parametrize('rt_ticket', [ - {'id': 1, 'Requestors': ["Foo@example.com"], 'Cc': ["UserName1@Example.COM", "username2@example.com"]}, - {'id': 1, 'Requestors': ["Foo@example.com"], 'Cc': "UserName1@Example.COM,username2@example.com"}, - {'id': 1, 'Requestors': ["Foo@example.com"], 'Cc': ["username1@example.com", "username2@example.com"]}, - {'id': 1, 'Cc': ["username1@example.com", "username2@example.com"]}, - {'id': 1, 'Requestors': [], 'Cc': ["username1@example.com", "username2@example.com"]}], indirect=True) + assert rtTester.hasAccess(1, "Username1@Example.com") is True + assert rtTester.hasAccess(1, "Username2@Example.com") is True + + +@pytest.mark.parametrize( + "rt_ticket", + [ + {"id": 1, "Requestors": ["Foo@example.com"], "Cc": ["UserName1@Example.COM", "username2@example.com"]}, + {"id": 1, "Requestors": ["Foo@example.com"], "Cc": "UserName1@Example.COM,username2@example.com"}, + {"id": 1, "Requestors": ["Foo@example.com"], "Cc": ["username1@example.com", "username2@example.com"]}, + {"id": 1, "Cc": ["username1@example.com", "username2@example.com"]}, + {"id": 1, "Requestors": [], "Cc": ["username1@example.com", "username2@example.com"]}, + ], + indirect=True, +) def test_rt_hasaccess_cc(mock_tracker): rtTester = RtUtilTestable(mock_tracker) - assert rtTester.hasAccess(1, 'Username1@Example.com') is True - assert rtTester.hasAccess(1, 'Username2@Example.com') is True - - -@pytest.mark.parametrize('rt_ticket', [ - {'id': 1, 'Requestors': ["foo@example.com"], 'Cc': ["baz@example.com"]}, - {'id': 1, 'Requestors': ["FOO@example.com"], 'Cc': ["BAZ@example.com"]}, - {'id': 1}, - {'id': 1, 'Requestors': [], 'Cc': []}], indirect=True) + assert rtTester.hasAccess(1, "Username1@Example.com") is True + assert rtTester.hasAccess(1, "Username2@Example.com") is True + + +@pytest.mark.parametrize( + "rt_ticket", + [ + {"id": 1, "Requestors": ["foo@example.com"], "Cc": ["baz@example.com"]}, + {"id": 1, "Requestors": ["FOO@example.com"], "Cc": ["BAZ@example.com"]}, + {"id": 1}, + {"id": 1, "Requestors": [], "Cc": []}, + ], + indirect=True, +) def test_rt_hasnoaccess(mock_tracker): rtTester = RtUtilTestable(mock_tracker) - assert rtTester.hasAccess(1, 'Username1@Example.com') is False + assert rtTester.hasAccess(1, "Username1@Example.com") is False diff --git a/server/portal/apps/tickets/urls.py b/server/portal/apps/tickets/urls.py index 42313f5459..10812fa59f 100644 --- a/server/portal/apps/tickets/urls.py +++ b/server/portal/apps/tickets/urls.py @@ -1,8 +1,8 @@ from django.urls import path from portal.apps.tickets import views -app_name = 'tickets' +app_name = "tickets" urlpatterns = [ - path('', views.tickets, name='mytickets'), - path('new/', views.ticket_create, name='create'), + path("", views.tickets, name="mytickets"), + path("new/", views.ticket_create, name="create"), ] diff --git a/server/portal/apps/tickets/utils.py b/server/portal/apps/tickets/utils.py index ae3b0ba2d0..e09166da4c 100644 --- a/server/portal/apps/tickets/utils.py +++ b/server/portal/apps/tickets/utils.py @@ -6,8 +6,7 @@ METADATA_HEADER = "*** Ticket Metadata ***" -def create_ticket(username, first_name, last_name, email, cc, subject, - problem_description, attachments, info, meta): +def create_ticket(username, first_name, last_name, email, cc, subject, problem_description, attachments, info, meta): rt = rtUtil.DjangoRt() if subject is None or email is None or problem_description is None: @@ -16,7 +15,7 @@ def create_ticket(username, first_name, last_name, email, cc, subject, metadata = "{}\n\n".format(METADATA_HEADER) metadata += "Client info:\n{}\n\n".format(info) - for key in ['HTTP_REFERER', 'HTTP_USER_AGENT', 'HTTP_HOST']: + for key in ["HTTP_REFERER", "HTTP_USER_AGENT", "HTTP_HOST"]: metadata += "{}:\n{}\n\n".format(key, meta.get(key, "None")) if username: @@ -30,22 +29,17 @@ def create_ticket(username, first_name, last_name, email, cc, subject, problem_description += "\n\n" + metadata - ticket_id = rt.create_ticket(subject=subject, - problem_description=problem_description, - requestor=email, - cc=cc, - attachments=attachments) + ticket_id = rt.create_ticket( + subject=subject, problem_description=problem_description, requestor=email, cc=cc, attachments=attachments + ) - return JsonResponse({'ticket_id': ticket_id}) + return JsonResponse({"ticket_id": ticket_id}) def get_recaptcha_verification(request): - recaptcha_response = request.POST.get('recaptchaResponse') + recaptcha_response = request.POST.get("recaptchaResponse") secret_key = settings.RECAPTCHA_SECRET_KEY - data = { - 'secret': secret_key, - 'response': recaptcha_response - } - r = requests.post('https://www.google.com/recaptcha/api/siteverify', data=data) + data = {"secret": secret_key, "response": recaptcha_response} + r = requests.post("https://www.google.com/recaptcha/api/siteverify", data=data) recap_result = r.json() return recap_result diff --git a/server/portal/apps/tickets/views.py b/server/portal/apps/tickets/views.py index e1440f19ae..93b6d51e9e 100644 --- a/server/portal/apps/tickets/views.py +++ b/server/portal/apps/tickets/views.py @@ -6,14 +6,13 @@ @login_required def tickets(request): - response = redirect('/workbench/dashboard/') + response = redirect("/workbench/dashboard/") return response @ensure_csrf_cookie def ticket_create(request): if request.user.is_authenticated and request.user.profile.setup_complete: - response = redirect('/workbench/dashboard/tickets/create/') + response = redirect("/workbench/dashboard/tickets/create/") return response - return render(request, 'portal/apps/workbench/index.html', - context={'DEBUG': settings.DEBUG}) + return render(request, "portal/apps/workbench/index.html", context={"DEBUG": settings.DEBUG}) diff --git a/server/portal/apps/users/tasks.py b/server/portal/apps/users/tasks.py index 598d481888..1c67807ea4 100644 --- a/server/portal/apps/users/tasks.py +++ b/server/portal/apps/users/tasks.py @@ -59,10 +59,7 @@ def get_tas_allocations(username): # Separate active and inactive allocations and make single entry for each project if resource["allocation"]["status"] == "Active": - if ( - resource["host"] in hosts - and charge_code not in hosts[resource["host"]] - ): + if resource["host"] in hosts and charge_code not in hosts[resource["host"]]: hosts[resource["host"]].append(charge_code) elif resource["host"] not in hosts: hosts[resource["host"]] = [charge_code] @@ -74,9 +71,7 @@ def get_tas_allocations(username): active_allocations[charge_code] = { "title": tas_proj["title"], "projectId": tas_proj["id"], - "pi": "{} {}".format( - tas_proj["pi"]["firstName"], tas_proj["pi"]["lastName"] - ), + "pi": "{} {}".format(tas_proj["pi"]["firstName"], tas_proj["pi"]["lastName"]), "projectName": tas_proj["chargeCode"], "systems": [resource], } @@ -87,9 +82,7 @@ def get_tas_allocations(username): inactive_allocations[charge_code] = { "title": tas_proj["title"], "projectId": tas_proj["id"], - "pi": "{} {}".format( - tas_proj["pi"]["firstName"], tas_proj["pi"]["lastName"] - ), + "pi": "{} {}".format(tas_proj["pi"]["firstName"], tas_proj["pi"]["lastName"]), "projectName": tas_proj["chargeCode"], "systems": [resource], } diff --git a/server/portal/apps/users/unit_test.py b/server/portal/apps/users/unit_test.py index 3642294cb8..f3a8a72819 100644 --- a/server/portal/apps/users/unit_test.py +++ b/server/portal/apps/users/unit_test.py @@ -14,7 +14,6 @@ class AttrDict(dict): - def __getattr__(self, key): return self[key] @@ -25,7 +24,7 @@ def __setattr__(self, key, value): class TestUserApiViews(TestCase): @classmethod def setUpClass(cls): - cls.mock_client_patcher = patch('portal.apps.auth.models.TapisOAuthToken.client') + cls.mock_client_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client") cls.mock_client = cls.mock_client_patcher.start() @classmethod @@ -34,12 +33,8 @@ def tearDownClass(cls): def setUp(self): User = get_user_model() - user = User.objects.create_user('test', 'test@test.com', 'test') - token = TapisOAuthToken( - access_token="1234fsf", - refresh_token="123123123", - expires_in=14400, - created=1523633447) + user = User.objects.create_user("test", "test@test.com", "test") + token = TapisOAuthToken(access_token="1234fsf", refresh_token="123123123", expires_in=14400, created=1523633447) token.user = user token.save() user.is_staff = False @@ -47,12 +42,12 @@ def setUp(self): user.save() def test_auth_view(self): - self.client.login(username='test', password='test') + self.client.login(username="test", password="test") resp = self.client.get("/api/users/auth/", follow=True) data = resp.json() self.assertEqual(resp.status_code, 200) # should only return user data system and community - self.assertTrue(data["username"] == 'test') + self.assertTrue(data["username"] == "test") self.assertTrue(data["email"] == "test@test.com") self.assertFalse(data["isStaff"]) @@ -61,16 +56,14 @@ def test_auth_view_noauth(self): self.assertEqual(resp.status_code, 401) # should only return user data system and community - @patch('portal.apps.users.views.IndexedFile') + @patch("portal.apps.users.views.IndexedFile") def test_usage_view(self, mocked_file): # TODO: this is hideous, there must be a better way to write that or # re-write the route to be less disgusting. mocked_file.search.return_value.filter.return_value.extra.return_value.execute.return_value.to_dict.return_value = { - "aggregations": { - "total_storage_bytes": {"value": 10} - } + "aggregations": {"total_storage_bytes": {"value": 10}} } - self.client.login(username='test', password='test') + self.client.login(username="test", password="test") resp = self.client.get("/api/users/usage/systemId", follow=True) data = resp.json() self.assertTrue(data["total_storage_bytes"] == 10) @@ -84,18 +77,15 @@ def test_usage_view_noauth(self): class TestGetAllocations(TestCase): def setUp(self): super(TestGetAllocations, self).setUp() - self.mock_tas_patcher = patch( - 'portal.apps.users.tasks.TASClient', - spec=TASClient - ) + self.mock_tas_patcher = patch("portal.apps.users.tasks.TASClient", spec=TASClient) self.mock_tas = self.mock_tas_patcher.start() def tearDown(self): super(TestGetAllocations, self).tearDown() self.mock_tas_patcher.stop() - @patch('portal.apps.users.utils.IndexedAllocation') - @patch('portal.apps.users.utils.get_tas_allocations') + @patch("portal.apps.users.utils.IndexedAllocation") + @patch("portal.apps.users.utils.get_tas_allocations") def test_force_get_allocations(self, mock_get, mock_idx): mock_get.return_value = [] get_allocations("username", force=True) @@ -107,47 +97,23 @@ def test_allocations_returned(self): "title": "Big Project", "chargeCode": "Big-Proj", "id": "Test-ID", - "pi": { - "firstName": "Test", - "lastName": "User" - }, - "allocations": [ - { - "status": "Active", - "resource": "Frontera" - } - ] + "pi": {"firstName": "Test", "lastName": "User"}, + "allocations": [{"status": "Active", "resource": "Frontera"}], }, { "title": "Old Proj", "chargeCode": "Proj-Old", "id": "Test-ID", - "pi": { - "firstName": "Old", - "lastName": "User" - }, - "allocations": [ - { - "status": "Inactive", - "resource": "Stampede4" - } - ] + "pi": {"firstName": "Old", "lastName": "User"}, + "allocations": [{"status": "Inactive", "resource": "Stampede4"}], }, { "title": "A Proj", "chargeCode": "Proj-Code", "id": "Test-ID", - "pi": { - "firstName": "Another", - "lastName": "User" - }, - "allocations": [ - { - "status": "Active", - "resource": "Rodeo2" - } - ] - } + "pi": {"firstName": "Another", "lastName": "User"}, + "allocations": [{"status": "Active", "resource": "Rodeo2"}], + }, ] active_expected = [ { @@ -155,29 +121,29 @@ def test_allocations_returned(self): "projectId": "Test-ID", "systems": [ { - 'allocation': {'resource': 'Frontera', 'status': 'Active'}, - 'host': 'frontera.tacc.utexas.edu', - 'name': 'Frontera', - 'type': 'HPC' + "allocation": {"resource": "Frontera", "status": "Active"}, + "host": "frontera.tacc.utexas.edu", + "name": "Frontera", + "type": "HPC", } ], "title": "Big Project", - "pi": "Test User" + "pi": "Test User", }, { "projectName": "Proj-Code", "projectId": "Test-ID", "systems": [ { - 'allocation': {'resource': 'Rodeo2', 'status': 'Active'}, - 'host': 'rodeo.tacc.utexas.edu', - 'name': 'Rodeo', - 'type': 'STORAGE' + "allocation": {"resource": "Rodeo2", "status": "Active"}, + "host": "rodeo.tacc.utexas.edu", + "name": "Rodeo", + "type": "STORAGE", } ], "title": "A Proj", - "pi": "Another User" - } + "pi": "Another User", + }, ] inactive_expected = [ @@ -186,27 +152,24 @@ def test_allocations_returned(self): "projectId": "Test-ID", "systems": [ { - 'allocation': {'resource': 'Stampede4', 'status': 'Inactive'}, - 'host': 'stampede2.tacc.utexas.edu', - 'name': 'Stampede2', - 'type': 'HPC' + "allocation": {"resource": "Stampede4", "status": "Inactive"}, + "host": "stampede2.tacc.utexas.edu", + "name": "Stampede2", + "type": "HPC", } ], "title": "Old Proj", - "pi": "Old User" + "pi": "Old User", } ] - hosts_expected = { - 'rodeo.tacc.utexas.edu': ['Proj-Code'], - 'frontera.tacc.utexas.edu': ['Big-Proj'] - } + hosts_expected = {"rodeo.tacc.utexas.edu": ["Proj-Code"], "frontera.tacc.utexas.edu": ["Big-Proj"]} data_expected = { - 'active': active_expected, - 'inactive': inactive_expected, - 'hosts': hosts_expected, - 'portal_alloc': 'test' + "active": active_expected, + "inactive": inactive_expected, + "hosts": hosts_expected, + "portal_alloc": "test", } data = get_tas_allocations("username") @@ -214,82 +177,79 @@ def test_allocations_returned(self): class TestGetIndexedAllocations(TestCase): - - @patch('portal.apps.users.utils.index_allocations') - @patch('portal.apps.users.utils.IndexedAllocation') + @patch("portal.apps.users.utils.index_allocations") + @patch("portal.apps.users.utils.IndexedAllocation") def test_checks_allocations(self, mock_idx, mock_index_allocations): - get_allocations('testuser') - mock_index_allocations.apply_async.assert_called_once_with( - args=['testuser'] - ) - mock_idx.from_username.assert_called_with('testuser') - - @patch('portal.apps.users.utils.IndexedAllocation') - @patch('portal.apps.users.utils.get_tas_allocations') + get_allocations("testuser") + mock_index_allocations.apply_async.assert_called_once_with(args=["testuser"]) + mock_idx.from_username.assert_called_with("testuser") + + @patch("portal.apps.users.utils.IndexedAllocation") + @patch("portal.apps.users.utils.get_tas_allocations") def test_allocation_fallback(self, mock_get_alloc, mock_idx): mock_idx.from_username.side_effect = NotFoundError - get_allocations('testuser') - mock_get_alloc.assert_called_with('testuser') + get_allocations("testuser") + mock_get_alloc.assert_called_with("testuser") mock_idx().save.assert_called_with() @pytest.fixture def tas_add_user_response(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_add_user_to_project.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_add_user_to_project.json")) as f: yield json.load(f) @pytest.fixture def tas_delete_user_response(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_delete_user_from_project.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_delete_user_from_project.json")) as f: yield json.load(f) @pytest.fixture def tas_add_user_error_response(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_add_user_to_project_error.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_add_user_to_project_error.json")) as f: yield json.load(f) @pytest.fixture def tas_delete_user_error_response(): - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_delete_user_from_project_error.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_delete_user_from_project_error.json")) as f: yield json.load(f) def test_add_user(client, requests_mock, authenticated_user, tas_add_user_response): requests_mock.post("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_add_user_response) - response = client.post('/api/users/team/manage/1234/5678') + response = client.post("/api/users/team/manage/1234/5678") assert response.status_code == 200 - assert response.json() == {"response": 'ok'} + assert response.json() == {"response": "ok"} def test_add_user_unauthenticated(client): - response = client.post('/api/users/team/manage/1234/5678') + response = client.post("/api/users/team/manage/1234/5678") assert response.status_code == 302 def test_add_user_failure(client, requests_mock, authenticated_user, tas_add_user_error_response): requests_mock.post("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_add_user_error_response) - response = client.post('/api/users/team/manage/1234/5678') + response = client.post("/api/users/team/manage/1234/5678") assert response.status_code == 400 def test_delete_user(client, requests_mock, authenticated_user, tas_delete_user_response): requests_mock.delete("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_delete_user_response) - response = client.delete('/api/users/team/manage/1234/5678') + response = client.delete("/api/users/team/manage/1234/5678") assert response.status_code == 200 - assert response.json() == {"response": 'ok'} + assert response.json() == {"response": "ok"} def test_delete_user_unauthenticated(client): - response = client.delete('/api/users/team/manage/1234/5678') + response = client.delete("/api/users/team/manage/1234/5678") assert response.status_code == 302 def test_delete_user_failure(client, requests_mock, authenticated_user, tas_delete_user_error_response): requests_mock.delete("{}/v1/projects/1234/users/5678".format(settings.TAS_URL), json=tas_delete_user_error_response) - response = client.delete('/api/users/team/manage/1234/5678') + response = client.delete("/api/users/team/manage/1234/5678") assert response.status_code == 400 @@ -315,7 +275,7 @@ def mock_tas_account2(mocker): @pytest.fixture def mock_tas_zeep_client(mocker): - zeep_client = mocker.patch('portal.apps.users.views.Client', autospec=True) + zeep_client = mocker.patch("portal.apps.users.views.Client", autospec=True) zeep_client.return_value.service.GetAccountsByLastName.return_value = [] zeep_client.return_value.service.GetAccountsByEmail.return_value = [] zeep_client.return_value.service.GetAccountByLogin.side_effect = Fault("None") @@ -323,12 +283,12 @@ def mock_tas_zeep_client(mocker): def test_search_tas_user_unauthenticated(client): - response = client.get('/api/users/tas-users/', {"search": "foo"}) + response = client.get("/api/users/tas-users/", {"search": "foo"}) assert response.status_code == 302 def test_search_tas_empty_response(client, authenticated_user, mock_tas_zeep_client): - response = client.get('/api/users/tas-users/', {"search": "foo"}) + response = client.get("/api/users/tas-users/", {"search": "foo"}) assert response.status_code == 200 assert response.json() == {"result": []} @@ -336,9 +296,21 @@ def test_search_tas_empty_response(client, authenticated_user, mock_tas_zeep_cli def test_search_tas(client, authenticated_user, mock_tas_zeep_client, mock_tas_account1, mock_tas_account2): mock_tas_zeep_client.service.GetAccountsByLastName.return_value = [mock_tas_account1, mock_tas_account2] mock_tas_zeep_client.service.GetAccountsByEmail.return_value = [mock_tas_account1, mock_tas_account2] - response = client.get('/api/users/tas-users/', {"search": "foo"}) + response = client.get("/api/users/tas-users/", {"search": "foo"}) assert response.status_code == 200 - assert response.json() == {"result": [{"username": "username1", "email": "user1@user.com", - "firstName": "firstName1", "lastName": "commonLastName"}, - {"username": "username2", "email": "user2@user.com", - "firstName": "firstName2", "lastName": "commonLastName"}]} + assert response.json() == { + "result": [ + { + "username": "username1", + "email": "user1@user.com", + "firstName": "firstName1", + "lastName": "commonLastName", + }, + { + "username": "username2", + "email": "user2@user.com", + "firstName": "firstName2", + "lastName": "commonLastName", + }, + ] + } diff --git a/server/portal/apps/users/urls.py b/server/portal/apps/users/urls.py index 96ebe19e10..447d956522 100644 --- a/server/portal/apps/users/urls.py +++ b/server/portal/apps/users/urls.py @@ -1,16 +1,27 @@ from django.urls import path, re_path -from portal.apps.users.views import (SearchView, AuthenticatedView, UsageView, AllocationsView, TeamView, - UserDataView, TasUsersView, AllocationUsageView, AllocationManagementView) +from portal.apps.users.views import ( + SearchView, + AuthenticatedView, + UsageView, + AllocationsView, + TeamView, + UserDataView, + TasUsersView, + AllocationUsageView, + AllocationManagementView, +) -app_name = 'users' +app_name = "users" urlpatterns = [ - re_path(r'^$', SearchView.as_view(), name='user_search'), - re_path(r'^auth/$', AuthenticatedView.as_view(), name='user_authenticated'), - path('usage/', UsageView.as_view(), name='user_usage'), - re_path(r'^allocations/$', AllocationsView.as_view(), name='user_allocations'), - path('tas-users/', TasUsersView.as_view(), name='tas_users'), - path('team/', TeamView.as_view(), name='user_team'), - path('team/user/', UserDataView.as_view(), name='user_data'), - path('team/usage/', AllocationUsageView.as_view(), name='allocation_usage'), - path('team/manage//', AllocationManagementView.as_view(), name='allocation_management') + re_path(r"^$", SearchView.as_view(), name="user_search"), + re_path(r"^auth/$", AuthenticatedView.as_view(), name="user_authenticated"), + path("usage/", UsageView.as_view(), name="user_usage"), + re_path(r"^allocations/$", AllocationsView.as_view(), name="user_allocations"), + path("tas-users/", TasUsersView.as_view(), name="tas_users"), + path("team/", TeamView.as_view(), name="user_team"), + path("team/user/", UserDataView.as_view(), name="user_data"), + path("team/usage/", AllocationUsageView.as_view(), name="allocation_usage"), + path( + "team/manage//", AllocationManagementView.as_view(), name="allocation_management" + ), ] diff --git a/server/portal/apps/users/utils.py b/server/portal/apps/users/utils.py index 839c23cb4f..d7a236cafb 100644 --- a/server/portal/apps/users/utils.py +++ b/server/portal/apps/users/utils.py @@ -16,10 +16,10 @@ def list_to_model_queries(q_comps): query = None if len(q_comps) > 2: - query = Q(first_name__icontains=' '.join(q_comps[:1])) - query |= Q(first_name__icontains=' '.join(q_comps[:2])) - query |= Q(last_name__icontains=' '.join(q_comps[1:])) - query |= Q(last_name__icontains=' '.join(q_comps[2:])) + query = Q(first_name__icontains=" ".join(q_comps[:1])) + query |= Q(first_name__icontains=" ".join(q_comps[:2])) + query |= Q(last_name__icontains=" ".join(q_comps[1:])) + query |= Q(last_name__icontains=" ".join(q_comps[2:])) else: query = Q(first_name__icontains=q_comps[0]) query |= Q(last_name__icontains=q_comps[1]) @@ -31,7 +31,7 @@ def q_to_model_queries(q): return None query = None - if ' ' in q: + if " " in q: q_comps = q.split() query = list_to_model_queries(q_comps) else: @@ -59,12 +59,7 @@ def get_allocations(username, force=False): if force: logger.info("Forcing TAS allocation retrieval for user:{}".format(username)) raise NotFoundError - result = { - 'hosts': {}, - 'portal_alloc': None, - 'active': [], - 'inactive': [] - } + result = {"hosts": {}, "portal_alloc": None, "active": [], "inactive": []} result.update(IndexedAllocation.from_username(username).value.to_dict()) index_allocations.apply_async(args=[username]) return result @@ -84,12 +79,12 @@ def get_project_users_from_name(project_name): : rtype: list """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/projects/name/{1}/users'.format(settings.TAS_URL, project_name), auth=auth) + r = requests.get("{0}/v1/projects/name/{1}/users".format(settings.TAS_URL, project_name), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise ApiException('Failed to get project users', resp['message']) + raise ApiException("Failed to get project users", resp["message"]) def get_project_users_from_id(project_id): @@ -99,12 +94,12 @@ def get_project_users_from_id(project_id): : rtype: list """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/projects/{1}/users'.format(settings.TAS_URL, project_id), auth=auth) + r = requests.get("{0}/v1/projects/{1}/users".format(settings.TAS_URL, project_id), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise ApiException('Failed to get project users', resp['message']) + raise ApiException("Failed to get project users", resp["message"]) def get_project_from_name(project_name): @@ -114,12 +109,12 @@ def get_project_from_name(project_name): : rtype: dict """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/projects/name/{1}'.format(settings.TAS_URL, project_name), auth=auth) + r = requests.get("{0}/v1/projects/name/{1}".format(settings.TAS_URL, project_name), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise ApiException('Failed to get project', resp['message']) + raise ApiException("Failed to get project", resp["message"]) def get_project_from_id(project_id): @@ -129,12 +124,12 @@ def get_project_from_id(project_id): : rtype: dict """ auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/projects/{1}'.format(settings.TAS_URL, project_id), auth=auth) + r = requests.get("{0}/v1/projects/{1}".format(settings.TAS_URL, project_id), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise ApiException('Failed to get project', resp['message']) + raise ApiException("Failed to get project", resp["message"]) def get_user_data(username): @@ -145,10 +140,7 @@ def get_user_data(username): """ tas_client = TASClient( baseURL=settings.TAS_URL, - credentials={ - 'username': settings.TAS_CLIENT_KEY, - 'password': settings.TAS_CLIENT_SECRET - } + credentials={"username": settings.TAS_CLIENT_KEY, "password": settings.TAS_CLIENT_SECRET}, ) user_data = tas_client.get_user(username=username) return user_data @@ -156,39 +148,38 @@ def get_user_data(username): def get_per_user_allocation_usage(allocation_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.get('{0}/v1/allocations/{1}/usage'.format(settings.TAS_URL, allocation_id), auth=auth) + r = requests.get("{0}/v1/allocations/{1}/usage".format(settings.TAS_URL, allocation_id), auth=auth) resp = r.json() - if resp['status'] == 'success': - return resp['result'] + if resp["status"] == "success": + return resp["result"] else: - raise ApiException('Failed to get project users: {}'.format(resp['message'])) + raise ApiException("Failed to get project users: {}".format(resp["message"])) def add_user(project_id, user_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - uri = '{0}/v1/projects/{1}/users/{2}'.format(settings.TAS_URL, project_id, user_id) + uri = "{0}/v1/projects/{1}/users/{2}".format(settings.TAS_URL, project_id, user_id) r = requests.post(uri, auth=auth) resp = r.json() - if resp['status'] != 'success': - raise ApiException("Failed to add user: '{}'".format(resp['message'])) - return resp['result'] + if resp["status"] != "success": + raise ApiException("Failed to add user: '{}'".format(resp["message"])) + return resp["result"] def remove_user(project_id, user_id): auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) - r = requests.delete('{0}/v1/projects/{1}/users/{2}'.format(settings.TAS_URL, project_id, user_id), auth=auth) + r = requests.delete("{0}/v1/projects/{1}/users/{2}".format(settings.TAS_URL, project_id, user_id), auth=auth) resp = r.json() - if resp['status'] != 'success': - raise ApiException("Failed to delete user: '{}'".format(resp['message'])) - return resp['result'] + if resp["status"] != "success": + raise ApiException("Failed to delete user: '{}'".format(resp["message"])) + return resp["result"] def check_user_groups(username, groups): try: - return any( - user['username'] == str(username) - for group in groups for user in get_project_users_from_name(group) - ) + return any(user["username"] == str(username) for group in groups for user in get_project_users_from_name(group)) except Exception as e: - logger.error("Issue checking user groups for user:{} which failed with the following exception:{}".format(username, e)) + logger.error( + "Issue checking user groups for user:{} which failed with the following exception:{}".format(username, e) + ) return False diff --git a/server/portal/apps/users/views.py b/server/portal/apps/users/views.py index a25d06199d..5374ec15d7 100644 --- a/server/portal/apps/users/views.py +++ b/server/portal/apps/users/views.py @@ -18,14 +18,20 @@ from elasticsearch_dsl import Q from portal.libs.elasticsearch.docs.base import IndexedFile from pytas.http import TASClient -from portal.apps.users.utils import (get_allocations, get_user_data, get_per_user_allocation_usage, - add_user, remove_user, get_project_from_id, get_project_users_from_id) +from portal.apps.users.utils import ( + get_allocations, + get_user_data, + get_per_user_allocation_usage, + add_user, + remove_user, + get_project_from_id, + get_project_users_from_id, +) logger = logging.getLogger(__name__) class AuthenticatedView(BaseApiView): - def get(self, request): if request.user.is_authenticated: u = request.user @@ -41,26 +47,25 @@ def get(self, request): "expires_in": u.tapis_oauth.expires_in, }, "isStaff": u.is_staff, - "groups": groups + "groups": groups, } return JsonResponse(out) - return JsonResponse({'message': 'Unauthorized'}, status=401) + return JsonResponse({"message": "Unauthorized"}, status=401) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class UsageView(BaseApiView): - def get(self, request, system_id): default_sys = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM if not system_id and default_sys: - system_id = default_sys['system'] + system_id = default_sys["system"] search = IndexedFile.search() # search = search.filter(Q({'nested': {'path': 'pems', 'query': {'term': {'pems.username': username} }} })) - search = search.filter(Q('term', **{"system._exact": system_id})) + search = search.filter(Q("term", **{"system._exact": system_id})) search = search.extra(size=0) - search.aggs.metric('total_storage_bytes', 'sum', field="length") + search.aggs.metric("total_storage_bytes", "sum", field="length") resp = search.execute() resp = resp.to_dict() aggs = resp["aggregations"]["total_storage_bytes"] @@ -69,37 +74,34 @@ def get(self, request, system_id): return JsonResponse(out, safe=False) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class SearchView(BaseApiView): - def get(self, request): - resp_fields = ['first_name', 'last_name', 'email', 'username'] + resp_fields = ["first_name", "last_name", "email", "username"] model = get_user_model() - q = request.GET.get('username') + q = request.GET.get("username") if q: try: user = model.objects.get(username=q) except ObjectDoesNotExist: return HttpResponseNotFound() res_dict = { - 'first_name': user.first_name, - 'last_name': user.last_name, - 'email': user.email, - 'username': user.username, + "first_name": user.first_name, + "last_name": user.last_name, + "email": user.email, + "username": user.username, } try: user_tas = TASClient().get_user(username=q) - res_dict['profile'] = { - 'institution': user_tas['institution'] - } + res_dict["profile"] = {"institution": user_tas["institution"]} except Exception: - logger.info('No Profile.') + logger.info("No Profile.") return JsonResponse(res_dict) - q = request.GET.get('q') - role = request.GET.get('role') + q = request.GET.get("q") + role = request.GET.get("role") user_rs = model.objects.filter() if q: query = users_utils.q_to_model_queries(q) @@ -122,9 +124,8 @@ def get(self, request): return HttpResponseNotFound() -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class AllocationsView(BaseApiView): - def get(self, request): """Returns active user allocations on TACC resources @@ -134,16 +135,17 @@ def get(self, request): data = get_allocations(request.user.username) # This line iterates through all the projects in the active allocations and filters out the ones that are excluded in settings. - filtered_projects = [project for project in data["active"] if project.get("projectName") not in settings.ALLOCATIONS_TO_EXCLUDE] + filtered_projects = [ + project for project in data["active"] if project.get("projectName") not in settings.ALLOCATIONS_TO_EXCLUDE + ] data["active"] = filtered_projects return JsonResponse({"response": data}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class TeamView(BaseApiView): - def get(self, request, project_id): """Returns usernames for project team @@ -152,40 +154,39 @@ def get(self, request, project_id): """ usernames = get_project_users_from_id(project_id) - return JsonResponse({'response': usernames}, safe=False) + return JsonResponse({"response": usernames}, safe=False) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class UserDataView(BaseApiView): - def get(self, request, username): user_data = get_user_data(username) return JsonResponse({username: user_data}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class TasUsersView(BaseApiView): """SOAP actions for TAS""" def _getSOAPTASClient(self): - """SOAP client via zeep - """ + """SOAP client via zeep""" session = requests.Session() session.auth = requests.auth.HTTPBasicAuth(settings.TAS_CLIENT_KEY, settings.TAS_CLIENT_SECRET) try: - client = Client("https://tas.tacc.utexas.edu/TASWebService/PortalService.asmx?WSDL", - transport=Transport(session=session, cache=InMemoryCache())) + client = Client( + "https://tas.tacc.utexas.edu/TASWebService/PortalService.asmx?WSDL", + transport=Transport(session=session, cache=InMemoryCache()), + ) except Exception: raise Exception("Error instantiating TAS SOAP Client") return client def get(self, request): - """SOAP search endpoint for TAS users - """ - search_term = request.GET.get('search') + """SOAP search endpoint for TAS users""" + search_term = request.GET.get("search") if search_term is None: - raise HttpResponseBadRequest('No search term provided') + raise HttpResponseBadRequest("No search term provided") client = self._getSOAPTASClient() @@ -207,60 +208,63 @@ def get(self, request): result = [] for r in combined_results: - entry = {"username": r.Login, - "email": r.Person.Email, - "firstName": r.Person.FirstName, - "lastName": r.Person.LastName} + entry = { + "username": r.Login, + "email": r.Person.Email, + "firstName": r.Person.FirstName, + "lastName": r.Person.LastName, + } if entry not in result: result.append(entry) - return JsonResponse({'result': result}) + return JsonResponse({"result": result}) def put(self, request): - """SOAP endpoint to update TAS project user roles - """ + """SOAP endpoint to update TAS project user roles""" body = json.loads(request.body) - project_id = body.get('projectId', None) + project_id = body.get("projectId", None) if project_id is None: - return HttpResponseBadRequest('No project ID provided') - user_role = body.get('role', None) + return HttpResponseBadRequest("No project ID provided") + user_role = body.get("role", None) if user_role is None: - return HttpResponseBadRequest('No user role provided') - user_id = body.get('userId', None) + return HttpResponseBadRequest("No user role provided") + user_id = body.get("userId", None) if user_id is None: - return HttpResponseBadRequest('No user id provided') + return HttpResponseBadRequest("No user id provided") tas_project = get_project_from_id(project_id) - project_name = tas_project['title'] - is_pi = tas_project['pi']['username'] == request.user.username + project_name = tas_project["title"] + is_pi = tas_project["pi"]["username"] == request.user.username if not is_pi: - return JsonResponse({'message': 'Forbidden: Project roles can only be assigned by the Project PI.'}, status=403) + return JsonResponse( + {"message": "Forbidden: Project roles can only be assigned by the Project PI."}, status=403 + ) tas_client = self._getSOAPTASClient() try: tas_client.service.EditProjectUser(user_id, user_role) except Exception: - raise Exception(f"Error assigning user: {user_id} new role: {user_role} to project name:id : {project_name}:{project_id}") + raise Exception( + f"Error assigning user: {user_id} new role: {user_role} to project name:id : {project_name}:{project_id}" + ) - return JsonResponse({'response': 'ok'}) + return JsonResponse({"response": "ok"}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class AllocationUsageView(BaseApiView): - def get(self, request, allocation_id): usage = get_per_user_allocation_usage(allocation_id) - return JsonResponse({'response': usage}) + return JsonResponse({"response": usage}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class AllocationManagementView(BaseApiView): - def post(self, request, project_id, user_id): - logger.info('Adding {} to TAS project {}'.format(user_id, project_id)) + logger.info("Adding {} to TAS project {}".format(user_id, project_id)) add_user(project_id, user_id) - return JsonResponse({'response': 'ok'}) + return JsonResponse({"response": "ok"}) def delete(self, request, project_id, user_id): - logger.info('Deleting {} to TAS project {}'.format(user_id, project_id)) + logger.info("Deleting {} to TAS project {}".format(user_id, project_id)) remove_user(project_id, user_id) - return JsonResponse({'response': 'ok'}) + return JsonResponse({"response": "ok"}) diff --git a/server/portal/apps/webhooks/apps.py b/server/portal/apps/webhooks/apps.py index f788da8b33..b90d0f78b7 100644 --- a/server/portal/apps/webhooks/apps.py +++ b/server/portal/apps/webhooks/apps.py @@ -1,10 +1,8 @@ - - from django.apps import AppConfig class WebhookConfig(AppConfig): - name = 'portal.apps.webhooks' - label = 'webhooks' - verbose_name = 'Portal Webhooks' - app_label = 'webhooks' + name = "portal.apps.webhooks" + label = "webhooks" + verbose_name = "Portal Webhooks" + app_label = "webhooks" diff --git a/server/portal/apps/webhooks/callback.py b/server/portal/apps/webhooks/callback.py index f979e134e8..e627346657 100644 --- a/server/portal/apps/webhooks/callback.py +++ b/server/portal/apps/webhooks/callback.py @@ -7,6 +7,7 @@ class WebhookCallback(object): An abstract base class for executing callback functions upon receiving a validated webhook. """ + def __init__(self): pass diff --git a/server/portal/apps/webhooks/conftest.py b/server/portal/apps/webhooks/conftest.py index 46e10940c2..5be6a7ab96 100644 --- a/server/portal/apps/webhooks/conftest.py +++ b/server/portal/apps/webhooks/conftest.py @@ -3,6 +3,6 @@ @pytest.fixture def mock_webhook_id(mocker): - mock_get_webhook_id = mocker.patch('portal.apps.webhooks.utils.get_webhook_id') + mock_get_webhook_id = mocker.patch("portal.apps.webhooks.utils.get_webhook_id") mock_get_webhook_id.return_value = "MOCK_WEBHOOK_ID" yield mock_get_webhook_id diff --git a/server/portal/apps/webhooks/fields.py b/server/portal/apps/webhooks/fields.py index 10809fc076..f44929bae0 100644 --- a/server/portal/apps/webhooks/fields.py +++ b/server/portal/apps/webhooks/fields.py @@ -1,9 +1,7 @@ import json from django.conf import settings -from django.contrib.postgres.fields import ( - JSONField as DjangoJSONField -) +from django.contrib.postgres.fields import JSONField as DjangoJSONField from django.db.models import Field @@ -11,10 +9,11 @@ # from https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1 -if 'sqlite' in settings.DATABASES['default']['ENGINE']: +if "sqlite" in settings.DATABASES["default"]["ENGINE"]: + class JSONField(Field): def db_type(self, connection): - return 'text' + return "text" def from_db_value(self, value, expression, connection): if value is not None: @@ -37,5 +36,6 @@ def get_prep_value(self, value): def value_to_string(self, obj): return self.value_from_object(obj) else: + class JSONField(DjangoJSONField): pass diff --git a/server/portal/apps/webhooks/migrations/0001_initial.py b/server/portal/apps/webhooks/migrations/0001_initial.py index bcbdb17661..dd7f654449 100644 --- a/server/portal/apps/webhooks/migrations/0001_initial.py +++ b/server/portal/apps/webhooks/migrations/0001_initial.py @@ -8,7 +8,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -17,14 +16,22 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='ExternalCall', + name="ExternalCall", fields=[ - ('webhook_id', models.CharField(max_length=16, primary_key=True, serialize=False)), - ('time', models.DateTimeField(default=datetime.datetime.now)), - ('callback', models.CharField(max_length=300, null=True)), - ('callback_data', portal.apps.webhooks.fields.JSONField(null=True)), - ('accepting', models.BooleanField(default=True)), - ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("webhook_id", models.CharField(max_length=16, primary_key=True, serialize=False)), + ("time", models.DateTimeField(default=datetime.datetime.now)), + ("callback", models.CharField(max_length=300, null=True)), + ("callback_data", portal.apps.webhooks.fields.JSONField(null=True)), + ("accepting", models.BooleanField(default=True)), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py b/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py index c77bb31ecd..a120000022 100644 --- a/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py +++ b/server/portal/apps/webhooks/migrations/0001_squashed_0003_alter_externalcall_callback_data.py @@ -7,8 +7,11 @@ class Migration(migrations.Migration): - - replaces = [('webhooks', '0001_initial'), ('webhooks', '0002_auto_20221221_2114'), ('webhooks', '0003_alter_externalcall_callback_data')] + replaces = [ + ("webhooks", "0001_initial"), + ("webhooks", "0002_auto_20221221_2114"), + ("webhooks", "0003_alter_externalcall_callback_data"), + ] initial = True @@ -18,14 +21,22 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='ExternalCall', + name="ExternalCall", fields=[ - ('webhook_id', models.CharField(max_length=16, primary_key=True, serialize=False)), - ('time', models.DateTimeField(default=datetime.datetime.now)), - ('callback', models.CharField(max_length=300, null=True)), - ('callback_data', models.JSONField(null=True)), - ('accepting', models.BooleanField(default=True)), - ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("webhook_id", models.CharField(max_length=16, primary_key=True, serialize=False)), + ("time", models.DateTimeField(default=datetime.datetime.now)), + ("callback", models.CharField(max_length=300, null=True)), + ("callback_data", models.JSONField(null=True)), + ("accepting", models.BooleanField(default=True)), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py b/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py index a2d82e0e8d..528a2948b6 100644 --- a/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py +++ b/server/portal/apps/webhooks/migrations/0002_auto_20221221_2114.py @@ -5,15 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('webhooks', '0001_initial'), + ("webhooks", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='externalcall', - name='callback_data', + model_name="externalcall", + name="callback_data", field=portal.utils.fields.JSONField(null=True), ), ] diff --git a/server/portal/apps/webhooks/migrations/0003_alter_externalcall_callback_data.py b/server/portal/apps/webhooks/migrations/0003_alter_externalcall_callback_data.py index 58593e1438..1dfb65852a 100644 --- a/server/portal/apps/webhooks/migrations/0003_alter_externalcall_callback_data.py +++ b/server/portal/apps/webhooks/migrations/0003_alter_externalcall_callback_data.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('webhooks', '0002_auto_20221221_2114'), + ("webhooks", "0002_auto_20221221_2114"), ] operations = [ migrations.AlterField( - model_name='externalcall', - name='callback_data', + model_name="externalcall", + name="callback_data", field=models.JSONField(null=True), ), ] diff --git a/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py b/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py index 6bc28eaba6..b59163d643 100644 --- a/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py +++ b/server/portal/apps/webhooks/migrations/0004_alter_externalcall_time.py @@ -5,15 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('webhooks', '0001_squashed_0003_alter_externalcall_callback_data'), + ("webhooks", "0001_squashed_0003_alter_externalcall_callback_data"), ] operations = [ migrations.AlterField( - model_name='externalcall', - name='time', + model_name="externalcall", + name="time", field=models.DateTimeField(default=django.utils.timezone.now), ), ] diff --git a/server/portal/apps/webhooks/models.py b/server/portal/apps/webhooks/models.py index cbd42d3d7e..60a78a87da 100644 --- a/server/portal/apps/webhooks/models.py +++ b/server/portal/apps/webhooks/models.py @@ -13,12 +13,7 @@ class ExternalCall(models.Model): webhook_id = models.CharField(max_length=16, primary_key=True) # Associated user for webhook events - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name="+", - on_delete=models.CASCADE, - null=True - ) + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="+", on_delete=models.CASCADE, null=True) # Timestamp for outbound external call time = models.DateTimeField(default=timezone.now) @@ -34,9 +29,8 @@ class ExternalCall(models.Model): accepting = models.BooleanField(default=True) def __unicode__(self): - return '{webhook_id} ({accepting})'.format( - webhook_id=self.webhook_id, - accepting="Accepting Webhooks" if self.accepting else "Not accepting webhooks" + return "{webhook_id} ({accepting})".format( + webhook_id=self.webhook_id, accepting="Accepting Webhooks" if self.accepting else "Not accepting webhooks" ) def __str__(self): diff --git a/server/portal/apps/webhooks/unit_test.py b/server/portal/apps/webhooks/unit_test.py index 7b58bff58a..1668f2f9aa 100644 --- a/server/portal/apps/webhooks/unit_test.py +++ b/server/portal/apps/webhooks/unit_test.py @@ -15,7 +15,7 @@ class TestValidateTapisJob(TestCase): def setUp(self): - job_status_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_staging.json'))) + job_status_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_staging.json"))) self.tapis_event = TapisResult(**job_status_event) mock_client = MagicMock() mock_client.jobs.getJob.return_value = self.tapis_event @@ -23,10 +23,7 @@ def setUp(self): mock_user.tapis_oauth.client = mock_client mock_user_model = MagicMock() mock_user_model.objects.get.return_value = mock_user - self.user_model_patcher = patch( - 'portal.apps.webhooks.views.get_user_model', - return_value=mock_user_model - ) + self.user_model_patcher = patch("portal.apps.webhooks.views.get_user_model", return_value=mock_user_model) self.user_model = self.user_model_patcher.start() def tearDown(self): @@ -42,11 +39,10 @@ def test_valid_job_invalid_user(self): validate_tapis_job("id", "wronguser") def test_invalid_state(self): - self.assertEqual(validate_tapis_job("id", "username", disallowed_states=['STAGING_INPUTS']), None) + self.assertEqual(validate_tapis_job("id", "username", disallowed_states=["STAGING_INPUTS"]), None) class TestJobsWebhookView(TransactionTestCase): - def setUp(self): signals.post_save.disconnect(sender=Notification, dispatch_uid="notification_msg") mock_client = MagicMock() @@ -54,10 +50,7 @@ def setUp(self): mock_user.tapis_oauth.client = mock_client mock_user_model = MagicMock() mock_user_model.objects.get.return_value = mock_user - self.user_model_patcher = patch( - 'portal.apps.webhooks.views.get_user_model', - return_value=mock_user_model - ) + self.user_model_patcher = patch("portal.apps.webhooks.views.get_user_model", return_value=mock_user_model) self.user_model = self.user_model_patcher.start() def tearDown(self): @@ -65,36 +58,38 @@ def tearDown(self): self.user_model_patcher.stop() @override_settings(PORTAL_JOB_NOTIFICATION_STATES=["STAGING_INPUTS"]) - @patch('portal.apps.webhooks.views.validate_tapis_job') + @patch("portal.apps.webhooks.views.validate_tapis_job") def test_webhook_job_post(self, mock_validate_tapis_job): - job_notification_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_event.json'))) - job_status_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_staging.json'))) + job_notification_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_event.json"))) + job_status_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_staging.json"))) mock_validate_tapis_job.return_value = TapisResult(**job_status_event) - response = self.client.post(reverse('webhooks:jobs_wh_handler'), - json.dumps(job_notification_event), content_type='application/json') + response = self.client.post( + reverse("webhooks:jobs_wh_handler"), json.dumps(job_notification_event), content_type="application/json" + ) self.assertEqual(response.status_code, 200) n = Notification.objects.last() - n_status = n.to_dict()['extra']['status'] - job_data = json.loads(job_notification_event['event']['data']) - self.assertEqual(n_status, job_data['newJobStatus']) + n_status = n.to_dict()["extra"]["status"] + job_data = json.loads(job_notification_event["event"]["data"]) + self.assertEqual(n_status, job_data["newJobStatus"]) @override_settings(PORTAL_JOB_NOTIFICATION_STATES=["RUNNING"]) - @patch('portal.apps.webhooks.views.validate_tapis_job') + @patch("portal.apps.webhooks.views.validate_tapis_job") def test_webhook_job_post_invalid_state(self, mock_validate_tapis_job): - job_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_event.json'))) + job_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_event.json"))) mock_validate_tapis_job.return_value = TapisResult(**job_event) - response = self.client.post(reverse('webhooks:jobs_wh_handler'), - json.dumps(job_event), content_type='application/json') + response = self.client.post( + reverse("webhooks:jobs_wh_handler"), json.dumps(job_event), content_type="application/json" + ) self.assertEqual(response.status_code, 200) self.assertEqual(len(Notification.objects.all()), 0) class TestInteractiveWebhookView(TestCase): - fixtures = ['users', 'auth'] + fixtures = ["users", "auth"] def setUp(self): - self.mock_tapis_patcher = patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) + self.mock_tapis_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) self.mock_tapis_client = self.mock_tapis_patcher.start() self.client.force_login(get_user_model().objects.get(username="username")) @@ -105,7 +100,7 @@ def setUp(self): "event_type": "interactive_session_ready", "address": "https://frontera.tacc.utexas.edu:1234", "job_uuid": "e8a57f35-b4a7-4e17-9aea-a6e55564db4d-007", - "owner": "username" + "owner": "username", } def tearDown(self): @@ -113,34 +108,40 @@ def tearDown(self): signals.post_save.connect(send_notification_ws, sender=Notification, dispatch_uid="notification_msg") def test_unsupported_event_type(self): - response = self.client.post(reverse('webhooks:interactive_wh_handler'), - urlencode({'event_type': 'DUMMY'}), - content_type='application/x-www-form-urlencoded') + response = self.client.post( + reverse("webhooks:interactive_wh_handler"), + urlencode({"event_type": "DUMMY"}), + content_type="application/x-www-form-urlencoded", + ) self.assertTrue(response.status_code == 400) def test_webhook_web_post(self): - job_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_running.json'))) + job_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_running.json"))) self.mock_tapis_client.jobs.getJob.return_value = TapisResult(**job_event) - response = self.client.post(reverse('webhooks:interactive_wh_handler'), - urlencode(self.web_event), - content_type='application/x-www-form-urlencoded') + response = self.client.post( + reverse("webhooks:interactive_wh_handler"), + urlencode(self.web_event), + content_type="application/x-www-form-urlencoded", + ) self.assertEqual(response.status_code, 200) self.assertFalse(self.mock_tapis_client.meta.addMetadata.called) self.assertEqual(Notification.objects.count(), 1) n = Notification.objects.last() - action_link = n.to_dict()['action_link'] + action_link = n.to_dict()["action_link"] self.assertEqual(action_link, "https://frontera.tacc.utexas.edu:1234") def test_webhook_web_post_no_matching_job(self): - job_event = json.load(open(os.path.join(os.path.dirname(__file__), 'fixtures/job_failed.json'))) + job_event = json.load(open(os.path.join(os.path.dirname(__file__), "fixtures/job_failed.json"))) self.mock_tapis_client.jobs.get.return_value = TapisResult(**job_event) - response = self.client.post(reverse('webhooks:interactive_wh_handler'), - urlencode(self.web_event), - content_type='application/x-www-form-urlencoded') + response = self.client.post( + reverse("webhooks:interactive_wh_handler"), + urlencode(self.web_event), + content_type="application/x-www-form-urlencoded", + ) # no matching running job so it fails self.assertEqual(response.status_code, 400) self.assertEqual(Notification.objects.count(), 0) diff --git a/server/portal/apps/webhooks/urls.py b/server/portal/apps/webhooks/urls.py index 805753d9dc..4828be8b07 100644 --- a/server/portal/apps/webhooks/urls.py +++ b/server/portal/apps/webhooks/urls.py @@ -1,12 +1,12 @@ -"""Webhooks URLs -""" +"""Webhooks URLs""" + from django.urls import path from portal.apps.webhooks import views -app_name = 'webhooks' +app_name = "webhooks" urlpatterns = [ - path('jobs/', views.JobsWebhookView.as_view(), name='jobs_wh_handler'), - path('interactive/', views.InteractiveWebhookView.as_view(), name='interactive_wh_handler'), - path('callbacks//', views.CallbackWebhookView.as_view(), name='callback_wh_handler') + path("jobs/", views.JobsWebhookView.as_view(), name="jobs_wh_handler"), + path("interactive/", views.InteractiveWebhookView.as_view(), name="interactive_wh_handler"), + path("callbacks//", views.CallbackWebhookView.as_view(), name="callback_wh_handler"), ] diff --git a/server/portal/apps/webhooks/utils.py b/server/portal/apps/webhooks/utils.py index 80af9510a9..53486ec920 100644 --- a/server/portal/apps/webhooks/utils.py +++ b/server/portal/apps/webhooks/utils.py @@ -13,7 +13,7 @@ def get_webhook_id(): chars = string.ascii_letters + string.digits - return ''.join(random.choice(chars) for i in range(16)) + return "".join(random.choice(chars) for i in range(16)) def register_webhook(callback=None, callback_data=None, user=None): @@ -22,10 +22,7 @@ def register_webhook(callback=None, callback_data=None, user=None): Create an instance of ExternalCall and return the associated callback URL """ external_call = ExternalCall.objects.create( - callback=callback, - callback_data=callback_data, - user=user, - webhook_id=get_webhook_id() + callback=callback, callback_data=callback_data, user=user, webhook_id=get_webhook_id() ) return "{}/webhooks/callbacks/{}/".format(settings.VANITY_BASE_URL, external_call.webhook_id) @@ -44,25 +41,15 @@ def validate_webhook(webhook_id): def load_callback(callback_name): - """load_callback - - """ - module_str, callable_str = callback_name.rsplit('.', 1) + """load_callback""" + module_str, callable_str = callback_name.rsplit(".", 1) module = import_module(module_str) call = getattr(module, callable_str) if not isclass(call): - raise ValueError( - "{callback_name} is not a class".format( - callback_name=callback_name - ) - ) + raise ValueError("{callback_name} is not a class".format(callback_name=callback_name)) callback_instance = call() if not isinstance(callback_instance, WebhookCallback): - raise ValueError( - "{callback_name} is not a subclass of WebhookCallback".format( - callback_name=callback_name - ) - ) + raise ValueError("{callback_name} is not a subclass of WebhookCallback".format(callback_name=callback_name)) return callback_instance diff --git a/server/portal/apps/webhooks/utils_unit_test.py b/server/portal/apps/webhooks/utils_unit_test.py index 060cf05950..0cabee9330 100644 --- a/server/portal/apps/webhooks/utils_unit_test.py +++ b/server/portal/apps/webhooks/utils_unit_test.py @@ -1,12 +1,6 @@ - from portal.apps.webhooks.models import ExternalCall from portal.apps.webhooks.callback import WebhookCallback -from portal.apps.webhooks.utils import ( - load_callback, - register_webhook, - validate_webhook, - execute_callback -) +from portal.apps.webhooks.utils import load_callback, register_webhook, validate_webhook, execute_callback import pytest @@ -28,12 +22,12 @@ def mock_invalid_function(): def test_load_callback(): - result = load_callback('portal.apps.webhooks.utils_unit_test.MockCallback') + result = load_callback("portal.apps.webhooks.utils_unit_test.MockCallback") assert isinstance(result, MockCallback) with pytest.raises(ValueError): - load_callback('portal.apps.webhooks.utils_unit_test.InvalidCallback') + load_callback("portal.apps.webhooks.utils_unit_test.InvalidCallback") with pytest.raises(ValueError): - load_callback('portal.apps.webhooks.utils_unit_test.mock_invalid_function') + load_callback("portal.apps.webhooks.utils_unit_test.mock_invalid_function") def test_register_webhook(mock_webhook_id, regular_user): @@ -60,9 +54,6 @@ def test_validate_webhook(mock_webhook_id): def test_execute_callback(): - register_webhook( - callback='portal.apps.webhooks.utils_unit_test.MockCallback', - callback_data={"key": "value"} - ) + register_webhook(callback="portal.apps.webhooks.utils_unit_test.MockCallback", callback_data={"key": "value"}) external_callback = ExternalCall.objects.all()[0] execute_callback(external_callback, "mock_request") diff --git a/server/portal/apps/webhooks/views.py b/server/portal/apps/webhooks/views.py index f38430c1bf..c7a3595b98 100644 --- a/server/portal/apps/webhooks/views.py +++ b/server/portal/apps/webhooks/views.py @@ -16,10 +16,7 @@ from portal.views.base import BaseApiView from portal.libs.exceptions import PortalLibException from portal.exceptions.api import ApiException -from portal.apps.webhooks.utils import ( - validate_webhook, - execute_callback -) +from portal.apps.webhooks.utils import validate_webhook, execute_callback from portal.apps.workspace.api.utils import check_job_for_timeout from django.conf import settings @@ -46,9 +43,10 @@ def validate_tapis_job(job_uuid, job_owner, disallowed_states=[]): # Validate the job UUID against the owner if job_data.owner != job_owner: logger.error( - "Tapis job (owner='{}', status='{}) for this event (owner='{}') is not valid".format(job_data.owner, - job_data.status, - job_owner)) + "Tapis job (owner='{}', status='{}) for this event (owner='{}') is not valid".format( + job_data.owner, job_data.status, job_owner + ) + ) raise PortalLibException("Unable to find a related valid job for this notification.") # Check to see if the job state should generate a notification @@ -60,7 +58,7 @@ def validate_tapis_job(job_uuid, job_owner, disallowed_states=[]): return job_data -@method_decorator(csrf_exempt, name='dispatch') +@method_decorator(csrf_exempt, name="dispatch") class JobsWebhookView(BaseApiView): """ Dispatches notifications when receiving a POST request from the Tapis @@ -81,17 +79,17 @@ def post(self, request, *args, **kwargs): """ subscription = json.loads(request.body) - job = json.loads(subscription['event']['data']) + job = json.loads(subscription["event"]["data"]) - user = get_user_model().objects.get(username=job['jobOwner']) + user = get_user_model().objects.get(username=job["jobOwner"]) client = user.tapis_oauth.client try: - username = job['jobOwner'] - job_uuid = job['jobUuid'] - job_status = job['newJobStatus'] - job_name = job['jobName'] - job_old_status = job['oldJobStatus'] + username = job["jobOwner"] + job_uuid = job["jobUuid"] + job_status = job["newJobStatus"] + job_name = job["jobName"] + job_old_status = job["oldJobStatus"] # Do nothing on job status not in portal notification states if job_status not in settings.PORTAL_JOB_NOTIFICATION_STATES: @@ -106,47 +104,46 @@ def post(self, request, *args, **kwargs): if job_status == job_old_status: return HttpResponse("OK") - logger.info('JOB STATUS CHANGE: UUID={} status={}'.format(job_uuid, job_status)) + logger.info("JOB STATUS CHANGE: UUID={} status={}".format(job_uuid, job_status)) event_data = { - Notification.EVENT_TYPE: 'job', + Notification.EVENT_TYPE: "job", Notification.STATUS: Notification.INFO, Notification.USER: username, - Notification.EXTRA: { - "name": job_name, - "owner": username, - "status": job_status, - "uuid": job_uuid - } + Notification.EXTRA: {"name": job_name, "owner": username, "status": job_status, "uuid": job_uuid}, } # get additional job information only after the job has reached a terminal state non_terminal_states = list(set(settings.PORTAL_JOB_NOTIFICATION_STATES) - set(TERMINAL_JOB_STATES)) job_details = validate_tapis_job(job_uuid, username, disallowed_states=non_terminal_states) if job_details: - event_data[Notification.EXTRA]['remoteOutcome'] = job_details.remoteOutcome - event_data[Notification.EXTRA]['status'] = job_details.status + event_data[Notification.EXTRA]["remoteOutcome"] = job_details.remoteOutcome + event_data[Notification.EXTRA]["status"] = job_details.status try: - logger.info('Indexing job output for job={}'.format(job_uuid)) - - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': job_details.archiveSystemId, - 'filePath': job_details.archiveSystemDir}) + logger.info("Indexing job output for job={}".format(job_uuid)) + + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": job_details.archiveSystemId, + "filePath": job_details.archiveSystemDir, + } + ) except Exception as e: - logger.exception('Error starting async task to index job output: {}'.format(e)) + logger.exception("Error starting async task to index job output: {}".format(e)) with transaction.atomic(): Notification.objects.create(**event_data) - return HttpResponse('OK') + return HttpResponse("OK") except (ObjectDoesNotExist, BaseTapyException, PortalLibException) as e: logger.exception(e) return HttpResponseBadRequest("ERROR") -@method_decorator(csrf_exempt, name='dispatch') +@method_decorator(csrf_exempt, name="dispatch") class InteractiveWebhookView(BaseApiView): """ Dispatches notifications when receiving a POST request from interactive jobs @@ -157,11 +154,11 @@ def post(self, request, *args, **kwargs): Creates a notification with a link to the interactive job event. """ - event_type = request.POST.get('event_type', None) - job_uuid = request.POST.get('job_uuid', None) - job_owner = request.POST.get('owner', None) - address = request.POST.get('address', None) - message = request.POST.get('message', None) + event_type = request.POST.get("event_type", None) + job_uuid = request.POST.get("job_uuid", None) + job_owner = request.POST.get("owner", None) + address = request.POST.get("address", None) + message = request.POST.get("message", None) if not address: msg = "Missing required interactive webhook parameter: address" @@ -172,7 +169,7 @@ def post(self, request, *args, **kwargs): Notification.EVENT_TYPE: event_type, Notification.STATUS: Notification.INFO, Notification.USER: job_owner, - Notification.ACTION_LINK: address + Notification.ACTION_LINK: address, } if message: @@ -183,14 +180,12 @@ def post(self, request, *args, **kwargs): valid_state = validate_tapis_job(job_uuid, job_owner, TERMINAL_JOB_STATES) if not valid_state: raise PortalLibException( - "Interactive Job UUID {} for user {} was in invalid state".format( - job_uuid, job_owner - ) + "Interactive Job UUID {} for user {} was in invalid state".format(job_uuid, job_owner) ) event_data[Notification.EXTRA] = { "name": valid_state.name, "status": valid_state.status, - "uuid": valid_state.uuid + "uuid": valid_state.uuid, } except (HTTPError, BaseTapyException, PortalLibException) as e: @@ -199,10 +194,10 @@ def post(self, request, *args, **kwargs): Notification.objects.create(**event_data) - return HttpResponse('OK') + return HttpResponse("OK") -@method_decorator(csrf_exempt, name='dispatch') +@method_decorator(csrf_exempt, name="dispatch") class CallbackWebhookView(BaseApiView): """ Validates incoming webhook and executes registered callbacks @@ -213,4 +208,4 @@ def post(self, request, webhook_id): if external_call is None: raise ApiException execute_callback(external_call, request) - return HttpResponse('OK') + return HttpResponse("OK") diff --git a/server/portal/apps/webhooks/views_unit_test.py b/server/portal/apps/webhooks/views_unit_test.py index cc8d100260..ac138459a0 100644 --- a/server/portal/apps/webhooks/views_unit_test.py +++ b/server/portal/apps/webhooks/views_unit_test.py @@ -11,10 +11,9 @@ @pytest.fixture def webhook_url(): webhook_url = register_webhook( - callback="portal.apps.webhooks.views_unit_test.MockCallback", - callback_data={"key": "value"} + callback="portal.apps.webhooks.views_unit_test.MockCallback", callback_data={"key": "value"} ) - yield webhook_url[len(settings.VANITY_BASE_URL):] + yield webhook_url[len(settings.VANITY_BASE_URL) :] class MockCallback(WebhookCallback): @@ -26,9 +25,9 @@ def callback(self, external_call, webhook_request): def test_callback(client, webhook_url): - response = client.post(webhook_url, {"incoming": "data"}, content_type='application/json') + response = client.post(webhook_url, {"incoming": "data"}, content_type="application/json") assert response.status_code == 200 - response = client.post(webhook_url, {"incoming": "data"}, content_type='application/json') + response = client.post(webhook_url, {"incoming": "data"}, content_type="application/json") assert response.status_code == 400 response = client.post("/webhooks/callbacks/invalid/") assert response.status_code == 400 diff --git a/server/portal/apps/workbench/api/unit_test.py b/server/portal/apps/workbench/api/unit_test.py index d2f649a303..b60650a8ff 100644 --- a/server/portal/apps/workbench/api/unit_test.py +++ b/server/portal/apps/workbench/api/unit_test.py @@ -3,8 +3,8 @@ def test_workbench(client, authenticated_user): - response = client.get('/api/workbench/') + response = client.get("/api/workbench/") assert response.status_code == 200 result = json.loads(response.content) - assert result['response']['config']['debug'] == settings.DEBUG - assert result['response']['portalName'] == settings.PORTAL_NAMESPACE + assert result["response"]["config"]["debug"] == settings.DEBUG + assert result["response"]["portalName"] == settings.PORTAL_NAMESPACE diff --git a/server/portal/apps/workbench/api/urls.py b/server/portal/apps/workbench/api/urls.py index b3588d3bf6..3c41256e3e 100644 --- a/server/portal/apps/workbench/api/urls.py +++ b/server/portal/apps/workbench/api/urls.py @@ -1,7 +1,7 @@ from django.urls import path from portal.apps.workbench.api import views -app_name = 'workbench_api' +app_name = "workbench_api" urlpatterns = [ - path('', views.workbench_state, name='state'), + path("", views.workbench_state, name="state"), ] diff --git a/server/portal/apps/workbench/api/views.py b/server/portal/apps/workbench/api/views.py index 6e6f21ad2d..15d3a365be 100644 --- a/server/portal/apps/workbench/api/views.py +++ b/server/portal/apps/workbench/api/views.py @@ -4,15 +4,15 @@ def workbench_state(request): data = { - 'config': { + "config": { **settings.WORKBENCH_SETTINGS, - 'projectsEnableMetadata': settings.PORTAL_PROJECTS_ENABLE_METADATA, - 'publisher': settings.PORTAL_PUBLICATION_PUBLISHER, + "projectsEnableMetadata": settings.PORTAL_PROJECTS_ENABLE_METADATA, + "publisher": settings.PORTAL_PUBLICATION_PUBLISHER, }, - 'portalName': settings.PORTAL_NAMESPACE, - 'recaptchaSiteKey': settings.RECAPTCHA_SITE_KEY, - 'isTACCPortal': settings.IS_TACC_PORTAL, + "portalName": settings.PORTAL_NAMESPACE, + "recaptchaSiteKey": settings.RECAPTCHA_SITE_KEY, + "isTACCPortal": settings.IS_TACC_PORTAL, } if request.user.is_authenticated: - data['setupComplete'] = request.user.profile.setup_complete - return JsonResponse({'response': data}) + data["setupComplete"] = request.user.profile.setup_complete + return JsonResponse({"response": data}) diff --git a/server/portal/apps/workbench/apps.py b/server/portal/apps/workbench/apps.py index 9b81bd6e57..c70181e223 100644 --- a/server/portal/apps/workbench/apps.py +++ b/server/portal/apps/workbench/apps.py @@ -2,4 +2,4 @@ class WorkbenchConfig(AppConfig): - name = 'portal.apps.workbench' + name = "portal.apps.workbench" diff --git a/server/portal/apps/workbench/urls.py b/server/portal/apps/workbench/urls.py index 0a0617cfb1..6079bbbafa 100644 --- a/server/portal/apps/workbench/urls.py +++ b/server/portal/apps/workbench/urls.py @@ -2,13 +2,14 @@ .. module:: portal.apps.accounts.urls :synopsis: Accounts URLs """ + from django.urls import re_path from portal.apps.workbench.views import IndexView -app_name = 'workbench' +app_name = "workbench" urlpatterns = [ - re_path('account', IndexView.as_view(), name='account'), - re_path('dashboard', IndexView.as_view(), name='dashboard'), - re_path('onboarding/admin', IndexView.as_view(), name='onboarding_admin'), - re_path('', IndexView.as_view(), name='index'), + re_path("account", IndexView.as_view(), name="account"), + re_path("dashboard", IndexView.as_view(), name="dashboard"), + re_path("onboarding/admin", IndexView.as_view(), name="onboarding_admin"), + re_path("", IndexView.as_view(), name="index"), ] diff --git a/server/portal/apps/workbench/views.py b/server/portal/apps/workbench/views.py index eb5b855d85..3835c2fa3a 100644 --- a/server/portal/apps/workbench/views.py +++ b/server/portal/apps/workbench/views.py @@ -4,18 +4,19 @@ from django.conf import settings -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class IndexView(TemplateView): """ Main workbench view. """ - template_name = 'portal/apps/workbench/index.html' + + template_name = "portal/apps/workbench/index.html" def dispatch(self, request, *args, **kwargs): return super(IndexView, self).dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): context = super(IndexView, self).get_context_data(**kwargs) - context['setup_complete'] = self.request.user.profile.setup_complete - context['DEBUG'] = settings.DEBUG + context["setup_complete"] = self.request.user.profile.setup_complete + context["DEBUG"] = settings.DEBUG return context diff --git a/server/portal/apps/workspace/admin.py b/server/portal/apps/workspace/admin.py index 3ae87c361d..35507b699a 100644 --- a/server/portal/apps/workspace/admin.py +++ b/server/portal/apps/workspace/admin.py @@ -1,25 +1,19 @@ from django.contrib import admin -from portal.apps.workspace.models import ( - AppTrayEntry, - AppTrayCategory -) +from portal.apps.workspace.models import AppTrayEntry, AppTrayCategory @admin.register(AppTrayCategory) class AppTrayCategoryAdmin(admin.ModelAdmin): - fields = ('category', 'priority', ) + fields = ( + "category", + "priority", + ) @admin.register(AppTrayEntry) class AppTrayEntryAdmin(admin.ModelAdmin): fieldsets = ( - ('Display Options', { - 'fields': ('label', 'category', 'icon', 'appType', 'available') - }), - ('Tapis App Specification', { - 'fields': ('appId', 'version') - }), - ('HTML - all fields required', { - 'fields': ['html'] - }) + ("Display Options", {"fields": ("label", "category", "icon", "appType", "available")}), + ("Tapis App Specification", {"fields": ("appId", "version")}), + ("HTML - all fields required", {"fields": ["html"]}), ) diff --git a/server/portal/apps/workspace/api/lookups.py b/server/portal/apps/workspace/api/lookups.py index 2af22d1639..b3fa1ebd3f 100644 --- a/server/portal/apps/workspace/api/lookups.py +++ b/server/portal/apps/workspace/api/lookups.py @@ -1,5 +1,5 @@ """ - :synopsis: Function to lookup manager classes +:synopsis: Function to lookup manager classes """ from importlib import import_module @@ -18,7 +18,7 @@ def lookup_manager(name): if name not in manager_names: raise ApiException("Invalid file manager.") - module_str, class_str = settings.PORTAL_WORKSPACE_MANAGERS[name].rsplit('.', 1) + module_str, class_str = settings.PORTAL_WORKSPACE_MANAGERS[name].rsplit(".", 1) module = import_module(module_str) cls = getattr(module, class_str) return cls diff --git a/server/portal/apps/workspace/api/unit_test.py b/server/portal/apps/workspace/api/unit_test.py index 43d8b0ff99..14cb57f6ea 100644 --- a/server/portal/apps/workspace/api/unit_test.py +++ b/server/portal/apps/workspace/api/unit_test.py @@ -7,10 +7,10 @@ @pytest.mark.django_db(transaction=True) class TestJobHistoryView(TestCase): - fixtures = ['users', 'auth'] + fixtures = ["users", "auth"] def setUp(self): - self.mock_tapis_patcher = patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) + self.mock_tapis_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) self.mock_tapis_client = self.mock_tapis_patcher.start() self.client.force_login(get_user_model().objects.get(username="username")) @@ -23,9 +23,7 @@ def test_job_history_get(self): response = self.client.get("/api/workspace/jobs/{}/history".format(job_uuid)) self.mock_tapis_client.jobs.getJobHistory.assert_called_with( jobUuid=job_uuid, - headers={ - "X-Tapis-Tracking-ID": f"portals.{self.client.session.session_key}" - }, + headers={"X-Tapis-Tracking-ID": f"portals.{self.client.session.session_key}"}, ) data = json.loads(response.content) diff --git a/server/portal/apps/workspace/api/urls.py b/server/portal/apps/workspace/api/urls.py index 735a4f0fc4..680035ceb1 100644 --- a/server/portal/apps/workspace/api/urls.py +++ b/server/portal/apps/workspace/api/urls.py @@ -1,21 +1,20 @@ -"""Workpace API Urls -""" +"""Workpace API Urls""" + from django.urls import re_path from portal.apps.workspace.api import views -app_name = 'workspace_api' +app_name = "workspace_api" urlpatterns = [ - re_path(r'^apps/?', views.AppsView.as_view()), + re_path(r"^apps/?", views.AppsView.as_view()), re_path( - r'^jobs/(?P[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}\-[0-9a-fA-F]{3})/history/?$', - views.JobHistoryView.as_view() + r"^jobs/(?P[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}\-[0-9a-fA-F]{3})/history/?$", + views.JobHistoryView.as_view(), ), - re_path(r'^jobs/(?P\w+)/?$', views.JobsView.as_view()), - re_path(r'^jobs/?', views.JobsView.as_view()), + re_path(r"^jobs/(?P\w+)/?$", views.JobsView.as_view()), + re_path(r"^jobs/?", views.JobsView.as_view()), # TODOv3: dropV2Jobs - re_path(r'^historic/?', views.HistoricJobsView.as_view()), - re_path(r'^systems/?', views.SystemsView.as_view()), - re_path(r'^tray/?', views.AppsTrayView.as_view()) - + re_path(r"^historic/?", views.HistoricJobsView.as_view()), + re_path(r"^systems/?", views.SystemsView.as_view()), + re_path(r"^tray/?", views.AppsTrayView.as_view()), ] diff --git a/server/portal/apps/workspace/api/utils.py b/server/portal/apps/workspace/api/utils.py index 379f929655..46746b78d8 100644 --- a/server/portal/apps/workspace/api/utils.py +++ b/server/portal/apps/workspace/api/utils.py @@ -27,19 +27,11 @@ def check_job_for_timeout(job): if isinstance(job.notes, str): notes = json.loads(job.notes) else: - notes = ( - job.notes - if isinstance(job.notes, dict) - else getattr(job.notes, "__dict__", {}) - ) + notes = job.notes if isinstance(job.notes, dict) else getattr(job.notes, "__dict__", {}) is_failed = job.status == "FAILED" - is_interactive = ( - notes.get("isInteractive", False) if isinstance(notes, dict) else False - ) - has_timeout_message = job.lastMessage in get_tapis_timeout_error_messages( - job.remoteJobId - ) + is_interactive = notes.get("isInteractive", False) if isinstance(notes, dict) else False + has_timeout_message = job.lastMessage in get_tapis_timeout_error_messages(job.remoteJobId) if is_failed and is_interactive and has_timeout_message: job.status = "FINISHED" @@ -59,9 +51,7 @@ def should_push_keys(system_def: object, username) -> bool: """ If defaultAuthnMethod is not TMS_KEYS, return true. Otherwise, false. """ - return (not is_tms_system(system_def)) and ( - system_def.get("effectiveUserId") == username - ) + return (not is_tms_system(system_def)) and (system_def.get("effectiveUserId") == username) def system_credentials_ok(user: object, system_id: str, path: str = "/") -> bool: @@ -97,9 +87,7 @@ def test_system_access_ok(user: object, system_id: str, path: str = "/") -> bool raise e -def push_keys_required_if_not_credentials_ensured( - user: object, system_id: str, path: str = "/" -) -> bool: +def push_keys_required_if_not_credentials_ensured(user: object, system_id: str, path: str = "/") -> bool: """ Check if system credentials are required to be pushed by the user on the system, or attempt to create credentials if they are not present. @@ -116,9 +104,7 @@ def push_keys_required_if_not_credentials_ensured( if is_tms_system(system_def): if settings.IS_TACC_PORTAL is False: return True - create_system_credentials_with_tms( - tapis, user.username, system_id - ) + create_system_credentials_with_tms(tapis, user.username, system_id) elif should_push_keys(system_def, user.username): logger.info( diff --git a/server/portal/apps/workspace/api/utils_unit_test.py b/server/portal/apps/workspace/api/utils_unit_test.py index 2b1e1698de..fe76ec4913 100644 --- a/server/portal/apps/workspace/api/utils_unit_test.py +++ b/server/portal/apps/workspace/api/utils_unit_test.py @@ -34,9 +34,7 @@ def test_push_keys_required_if_not_credentials_ensured_successful_credential_cre "metadata": None, } - result = push_keys_required_if_not_credentials_ensured( - authenticated_user, "test_system", "/" - ) + result = push_keys_required_if_not_credentials_ensured(authenticated_user, "test_system", "/") assert result is False mock_tapis_client.systems.createUserCredential.assert_called_once_with( @@ -68,18 +66,14 @@ def test_push_keys_required_if_not_credentials_ensured_credentials_ok( "metadata": None, } - result = push_keys_required_if_not_credentials_ensured( - authenticated_user, "test_system", "/" - ) + result = push_keys_required_if_not_credentials_ensured(authenticated_user, "test_system", "/") assert result is False mock_should_push_keys.assert_not_called() mock_create_system_credentials_with_tms.assert_not_called() -def test_push_keys_required_if_not_credentials_ensured_push_keys_required( - authenticated_user, mock_tapis_client -): +def test_push_keys_required_if_not_credentials_ensured_push_keys_required(authenticated_user, mock_tapis_client): """ Test that the push_keys_required_if_not_credentials_ensured function returns True when the user does not have system credentials @@ -97,9 +91,7 @@ def test_push_keys_required_if_not_credentials_ensured_push_keys_required( mock_tapis_client.files.listFiles.side_effect = UnauthorizedError() mock_tapis_client.systems.getSystem.return_value = tapis_system - result = push_keys_required_if_not_credentials_ensured( - authenticated_user, "test_system", "/" - ) + result = push_keys_required_if_not_credentials_ensured(authenticated_user, "test_system", "/") assert result is True mock_tapis_client.systems.createUserCredential.assert_not_called() diff --git a/server/portal/apps/workspace/api/views.py b/server/portal/apps/workspace/api/views.py index b038cfd43f..8f62fbd857 100644 --- a/server/portal/apps/workspace/api/views.py +++ b/server/portal/apps/workspace/api/views.py @@ -2,6 +2,7 @@ .. :module:: apps.workspace.api.views :synopsys: Views to handle Workspace API """ + import logging import json from urllib.parse import urlparse @@ -23,16 +24,13 @@ from portal.apps.workspace.models import JobSubmission from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry from .handlers.tapis_handlers import tapis_get_handler -from portal.apps.workspace.api.utils import ( - check_job_for_timeout, - push_keys_required_if_not_credentials_ensured -) +from portal.apps.workspace.api.utils import check_job_for_timeout, push_keys_required_if_not_credentials_ensured from portal.utils import get_client_ip from portal.apps.datafiles.utils import evaluate_datafiles_storage_system logger = logging.getLogger(__name__) -METRICS = logging.getLogger('metrics.{}'.format(__name__)) +METRICS = logging.getLogger("metrics.{}".format(__name__)) def _app_license_type(app_def): @@ -55,7 +53,7 @@ def _get_exec_systems(user, systems): tapis = user.tapis_oauth.client search_string = "(canExec.eq.true)~(enabled.eq.true)" if systems != ["All"]: - system_id_search = ','.join(systems) + system_id_search = ",".join(systems) search_string = f"(id.in.{system_id_search})~{search_string}" return tapis.systems.getSystems(listType="ALL", select="allAttributes", search=search_string) @@ -67,32 +65,30 @@ def _get_app(app_id, app_version, user): else: app_def = tapis.apps.getAppLatestVersion(appId=app_id) - data = {'definition': app_def} + data = {"definition": app_def} dynamic_exec_systems = getattr(app_def.notes, "dynamicExecSystems", []) exec_systems = [system.systemId for system in dynamic_exec_systems] if len(exec_systems) > 0: - data['execSystems'] = _get_exec_systems(user, exec_systems) + data["execSystems"] = _get_exec_systems(user, exec_systems) else: # GET EXECUTION SYSTEM INFO TO PROCESS SPECIFIC SYSTEM DATA E.G. QUEUE INFORMATION - data['execSystems'] = [tapis.systems.getSystem(systemId=app_def.jobAttributes.execSystemId)] + data["execSystems"] = [tapis.systems.getSystem(systemId=app_def.jobAttributes.execSystemId)] lic_type = _app_license_type(app_def) - data['license'] = { - 'type': lic_type - } + data["license"] = {"type": lic_type} if lic_type is not None: lic = _get_user_app_license(lic_type, user) - data['license']['enabled'] = lic is not None + data["license"]["enabled"] = lic is not None return data -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class AppsView(BaseApiView): def get(self, request, *args, **kwargs): tapis = request.user.tapis_oauth.client - app_id = request.GET.get('appId') + app_id = request.GET.get("appId") if app_id: METRICS.info( "Apps", @@ -105,16 +101,16 @@ def get(self, request, *args, **kwargs): "info": {"query": request.GET.dict()}, }, ) - app_version = request.GET.get('appVersion') + app_version = request.GET.get("appVersion") data = _get_app(app_id, app_version, request.user) # Check if default storage system needs keys pushed if settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM: - system_id = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM['system'] - if push_keys_required_if_not_credentials_ensured(request.user, system_id, '/'): + system_id = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM["system"] + if push_keys_required_if_not_credentials_ensured(request.user, system_id, "/"): system_def = tapis.systems.getSystem(systemId=system_id) - data['systemNeedsKeys'] = True - data['pushKeysSystem'] = system_def + data["systemNeedsKeys"] = True + data["pushKeysSystem"] = system_def else: METRICS.info( @@ -128,36 +124,35 @@ def get(self, request, *args, **kwargs): "info": {"query": request.GET.dict()}, }, ) - data = {'appListing': tapis.apps.getApps()} + data = {"appListing": tapis.apps.getApps()} return JsonResponse( { - 'status': 200, - 'response': data, + "status": 200, + "response": data, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) # TODOv3: dropV2Jobs -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class HistoricJobsView(BaseApiView): def get(self, request, *args, **kwargs): - limit = int(request.GET.get('limit', 10)) - offset = int(request.GET.get('offset', 0)) + limit = int(request.GET.get("limit", 10)) + offset = int(request.GET.get("offset", 0)) - jobs = JobSubmission.objects.all().filter(user=request.user).exclude(data__isnull=True).order_by('-time') - data = [job.data for job in jobs[offset:offset + limit]] + jobs = JobSubmission.objects.all().filter(user=request.user).exclude(data__isnull=True).order_by("-time") + data = [job.data for job in jobs[offset : offset + limit]] return JsonResponse({"response": data}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class JobsView(BaseApiView): - def get(self, request, operation=None): - allowed_actions = ['listing', 'search', 'select'] + allowed_actions = ["listing", "search", "select"] tapis = request.user.tapis_oauth.client @@ -179,7 +174,7 @@ def get(self, request, operation=None): op = getattr(self, operation) data = op(tapis, request) - if (isinstance(data, list)): + if isinstance(data, list): for index, job in enumerate(data): data[index] = check_job_for_timeout(job) else: @@ -187,29 +182,32 @@ def get(self, request, operation=None): return JsonResponse( { - 'status': 200, - 'response': data, + "status": 200, + "response": data, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) def select(self, client, request): - job_uuid = request.GET.get('job_uuid') - data = client.jobs.getJob(jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + job_uuid = request.GET.get("job_uuid") + data = client.jobs.getJob( + jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) return data def listing(self, client, request): - limit = int(request.GET.get('limit', 10)) - offset = int(request.GET.get('offset', 0)) + limit = int(request.GET.get("limit", 10)) + offset = int(request.GET.get("offset", 0)) portal_name = settings.PORTAL_NAMESPACE data = client.jobs.getJobSearchList( limit=limit, skip=offset, - orderBy='lastUpdated(desc),name(asc)', - _tapis_query_parameters={'tags.contains': f'portalName: {portal_name}'}, - select='allAttributes', headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + orderBy="lastUpdated(desc),name(asc)", + _tapis_query_parameters={"tags.contains": f"portalName: {portal_name}"}, + select="allAttributes", + headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}, ) return data @@ -218,21 +216,21 @@ def search(self, client, request): # TODO WP-1116: declared on react frontend under client/src/components/Jobs/JobsStatus/JobsStatus.jsx # all status search features to be moved to frontend STATUS_TEXT_MAP = { - 'PENDING': 'Processing', - 'PROCESSING_INPUTS': 'Processing', - 'STAGING_INPUTS': 'Queueing', - 'STAGING_JOB': 'Queueing', - 'SUBMITTING_JOB': 'Queueing', - 'QUEUED': 'Queueing', - 'RUNNING': 'Running', - 'ARCHIVING': 'Finishing', - 'FINISHED': 'Finished', - 'STOPPED': 'Stopped', - 'FAILED': 'Failure', - 'BLOCKED': 'Blocked', - 'PAUSED': 'Paused', - 'CANCELLED': 'Cancelled', - 'ARCHIVED': 'Archived', + "PENDING": "Processing", + "PROCESSING_INPUTS": "Processing", + "STAGING_INPUTS": "Queueing", + "STAGING_JOB": "Queueing", + "SUBMITTING_JOB": "Queueing", + "QUEUED": "Queueing", + "RUNNING": "Running", + "ARCHIVING": "Finishing", + "FINISHED": "Finished", + "STOPPED": "Stopped", + "FAILED": "Failure", + "BLOCKED": "Blocked", + "PAUSED": "Paused", + "CANCELLED": "Cancelled", + "ARCHIVED": "Archived", } def get_statuses_for_label(label): @@ -248,7 +246,7 @@ def get_statuses_for_label(label): return statuses def is_interactive(job): - notes = getattr(job, 'notes', None) + notes = getattr(job, "notes", None) if not notes: return False if isinstance(notes, str): @@ -256,39 +254,39 @@ def is_interactive(job): notes = json.loads(notes) except Exception: return False - val = notes.get('isInteractive') + val = notes.get("isInteractive") if isinstance(val, str): - return val.strip().lower() == 'true' + return val.strip().lower() == "true" return bool(val) def has_timeout_message(job): - msg = getattr(job, 'lastMessage', '') or '' + msg = getattr(job, "lastMessage", "") or "" msg = msg.upper() - return ('TIME_EXPIRED' in msg) or ('TIMEOUT' in msg) + return ("TIME_EXPIRED" in msg) or ("TIMEOUT" in msg) - query_string = request.GET.get('query_string') + query_string = request.GET.get("query_string") # limiting search down to the first word if multiple words are inputted if query_string: query_string = query_string.split()[0] - limit = int(request.GET.get('limit', 10)) - offset = int(request.GET.get('offset', 0)) + limit = int(request.GET.get("limit", 10)) + offset = int(request.GET.get("offset", 0)) portal_name = settings.PORTAL_NAMESPACE - status_searches = get_statuses_for_label(query_string or '') + status_searches = get_statuses_for_label(query_string or "") # 3 most common cases for case insensitivity - qs_lower = query_string.lower() if query_string else '' - qs_upper = query_string.upper() if query_string else '' - qs_title = query_string.title() if query_string else '' + qs_lower = query_string.lower() if query_string else "" + qs_upper = query_string.upper() if query_string else "" + qs_title = query_string.title() if query_string else "" # TODO WP-1116: all status search add-ons to be removed and added to drop-down feature on frontend if status_searches: enhanced_status_conditions = [] for status in status_searches: - if status == 'FINISHED': + if status == "FINISHED": enhanced_status_conditions.append( "(status = 'FINISHED' OR (status = 'FAILED' AND (lastMessage LIKE '%TIME_EXPIRED%' OR lastMessage LIKE '%TIMEOUT%')))" ) - elif status == 'FAILED': + elif status == "FAILED": enhanced_status_conditions.append("(status = 'FAILED')") else: enhanced_status_conditions.append(f"(status = '{status}')") @@ -339,8 +337,8 @@ def has_timeout_message(job): # For "Failed" search, want to get all # (i) FAILED jobs except those that were interactive and have the timeout/expired message (excluded because they are shown as FINISHED on UI) - is_finished = 'FINISHED' in status_searches - is_failed = 'FAILED' in status_searches + is_finished = "FINISHED" in status_searches + is_failed = "FAILED" in status_searches if is_finished or is_failed: target_offset = offset @@ -352,10 +350,10 @@ def has_timeout_message(job): page = client.jobs.getJobSearchListByPostSqlStr( limit=upstream_page_size, skip=upstream_offset, - orderBy='lastUpdated(desc),name(asc)', + orderBy="lastUpdated(desc),name(asc)", request_body={"search": sql_queries}, select="allAttributes", - headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}, ) if not page: break @@ -363,19 +361,19 @@ def has_timeout_message(job): upstream_offset += len(page) for job in page: - status = getattr(job, 'status', None) + status = getattr(job, "status", None) if is_finished: - if status == 'FINISHED': + if status == "FINISHED": pass - elif status == 'FAILED': + elif status == "FAILED": if not (has_timeout_message(job) and is_interactive(job)): continue else: continue elif is_failed: - if status != 'FAILED': + if status != "FAILED": continue if has_timeout_message(job) and is_interactive(job): continue @@ -396,12 +394,10 @@ def has_timeout_message(job): data = client.jobs.getJobSearchListByPostSqlStr( limit=limit, skip=offset, - orderBy='lastUpdated(desc),name(asc)', - request_body={ - "search": sql_queries - }, + orderBy="lastUpdated(desc),name(asc)", + request_body={"search": sql_queries}, select="allAttributes", - headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}, ) return data @@ -419,28 +415,32 @@ def delete(self, request, *args, **kwargs): }, ) tapis = request.user.tapis_oauth.client - job_uuid = request.GET.get('job_uuid') - data = tapis.jobs.hideJob(jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + job_uuid = request.GET.get("job_uuid") + data = tapis.jobs.hideJob( + jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) return JsonResponse( { - 'status': 200, - 'response': data, + "status": 200, + "response": data, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) def post(self, request, *args, **kwargs): tapis = request.user.tapis_oauth.client username = request.user.username body = json.loads(request.body) - job_uuid = body.get('job_uuid') - job_action = body.get('action') - job_post = body.get('job') + job_uuid = body.get("job_uuid") + job_action = body.get("action") + job_post = body.get("job") if job_uuid and job_action: - if job_action == 'resubmit': + if job_action == "resubmit": logger.info("user:{} is resubmitting job uuid:{}".format(username, job_uuid)) - data = tapis.jobs.resubmitJob(jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + data = tapis.jobs.resubmitJob( + jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) if isinstance(data, TapisResult): metrics_info = { "body": body, @@ -460,9 +460,11 @@ def post(self, request, *args, **kwargs): }, ) - elif job_action == 'cancel': + elif job_action == "cancel": logger.info("user:{} is canceling/stopping job uuid:{}".format(username, job_uuid)) - data = tapis.jobs.cancelJob(jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + data = tapis.jobs.cancelJob( + jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) if isinstance(data, TapisResult): metrics_info = { "body": body, @@ -482,24 +484,28 @@ def post(self, request, *args, **kwargs): }, ) else: - raise ApiException("user:{} is trying to run an unsupported job action: {} for job uuid: {}".format( - username, - job_action, - job_uuid - ), status=400) + raise ApiException( + "user:{} is trying to run an unsupported job action: {} for job uuid: {}".format( + username, job_action, job_uuid + ), + status=400, + ) return JsonResponse( { - 'status': 200, - 'response': data, + "status": 200, + "response": data, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) elif not job_post: - raise ApiException("user:{} is submitting a request with no job body.".format( - username, - ), status=400) + raise ApiException( + "user:{} is submitting a request with no job body.".format( + username, + ), + status=400, + ) # submit job else: @@ -507,12 +513,16 @@ def post(self, request, *args, **kwargs): # Provide default job archive configuration if none is provided and portal has default system if settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM: - default_system = evaluate_datafiles_storage_system(request.user.tapis_oauth, settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM) - if not job_post.get('archiveSystemId'): - job_post['archiveSystemId'] = default_system['system'] - if not job_post.get('archiveSystemDir'): - homeDir = default_system['homeDir'] - job_post['archiveSystemDir'] = f'{homeDir}/tapis-jobs-archive/${{JobCreateDate}}/${{JobName}}-${{JobUUID}}' + default_system = evaluate_datafiles_storage_system( + request.user.tapis_oauth, settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM + ) + if not job_post.get("archiveSystemId"): + job_post["archiveSystemId"] = default_system["system"] + if not job_post.get("archiveSystemDir"): + homeDir = default_system["homeDir"] + job_post["archiveSystemDir"] = ( + f"{homeDir}/tapis-jobs-archive/${{JobCreateDate}}/${{JobName}}-${{JobUUID}}" + ) execSystemId = job_post.get("execSystemId") if not execSystemId: @@ -524,7 +534,7 @@ def post(self, request, *args, **kwargs): job_post["appVersion"] = app["definition"].version # Check for and set license environment variable if app requires one - lic_type = body.get('licenseType') + lic_type = body.get("licenseType") if lic_type: lic = _get_user_app_license(lic_type, request.user) if lic is None: @@ -539,7 +549,7 @@ def post(self, request, *args, **kwargs): # job_post['parameterSet']['envVariables'] = job_post['parameterSet'].get('envVariables', []) + [license_var] # Test file listing on relevant systems to determine whether keys need to be pushed manually - for system_id in list(filter(None, [job_post.get('archiveSystemId'), execSystemId])): + for system_id in list(filter(None, [job_post.get("archiveSystemId"), execSystemId])): if push_keys_required_if_not_credentials_ensured(request.user, system_id): system_def = tapis.systems.getSystem(systemId=system_id) return JsonResponse( @@ -565,34 +575,31 @@ def post(self, request, *args, **kwargs): jobs_wh_url = request.build_absolute_uri(reverse("webhooks:jobs_wh_handler")) # Add additional data for interactive apps - if body.get('isInteractive'): + if body.get("isInteractive"): # Add webhook URL environment variable for interactive apps - job_post["parameterSet"]["envVariables"] = job_post["parameterSet"].get( - "envVariables", [] - ) + [{"key": "_INTERACTIVE_WEBHOOK_URL", "value": interactive_wh_url}] + job_post["parameterSet"]["envVariables"] = job_post["parameterSet"].get("envVariables", []) + [ + {"key": "_INTERACTIVE_WEBHOOK_URL", "value": interactive_wh_url} + ] # Add portalName tag to job in order to filter jobs by portal portal_name = settings.PORTAL_NAMESPACE - job_post['tags'] = job_post.get('tags', []) + [f'portalName: {portal_name}'] + job_post["tags"] = job_post.get("tags", []) + [f"portalName: {portal_name}"] # Add webhook subscription for job status updates - job_post["subscriptions"] = job_post.get('subscriptions', []) + [ - { + job_post["subscriptions"] = job_post.get("subscriptions", []) + [ + { "description": "Portal job status notification", "enabled": True, "eventCategoryFilter": "JOB_NEW_STATUS", "ttlMinutes": 0, # ttlMinutes of 0 corresponds to max default (1 week) - "deliveryTargets": [ - { - "deliveryMethod": "WEBHOOK", - "deliveryAddress": jobs_wh_url - } - ] + "deliveryTargets": [{"deliveryMethod": "WEBHOOK", "deliveryAddress": jobs_wh_url}], } ] logger.info("user:{} is submitting job:{}".format(username, job_post)) - response = tapis.jobs.submitJob(**job_post, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + response = tapis.jobs.submitJob( + **job_post, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) if isinstance(response, TapisResult): metrics_info = { @@ -615,20 +622,19 @@ def post(self, request, *args, **kwargs): return JsonResponse( { - 'status': 200, - 'response': response, + "status": 200, + "response": response, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class SystemsView(BaseApiView): - def get(self, request, *args, **kwargs): - roles = request.GET.get('roles') - user_role = request.GET.get('user_role') - system_id = request.GET.get('system_id') + roles = request.GET.get("roles") + user_role = request.GET.get("user_role") + system_id = request.GET.get("system_id") if roles: logger.info("user:{} tapis.systems.listRoles system_id:{}".format(request.user.username, system_id)) agc = service_account() @@ -641,29 +647,28 @@ def get(self, request, *args, **kwargs): def post(self, request, *args, **kwargs): body = json.loads(request.body) - role = body['role'] - system_id = body['system_id'] + role = body["role"] + system_id = body["system_id"] logger.info("user:{} tapis.systems.updateRole system_id:{}".format(request.user.username, system_id)) - role_body = { - 'username': request.user.username, - 'role': role - } + role_body = {"username": request.user.username, "role": role} agc = service_account() data = agc.systems.updateRole(systemId=system_id, body=role_body) return JsonResponse({"response": data}) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class JobHistoryView(BaseApiView): def get(self, request, job_uuid): tapis = request.user.tapis_oauth.client - data = tapis.jobs.getJobHistory(jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}) + data = tapis.jobs.getJobHistory( + jobUuid=job_uuid, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"} + ) return JsonResponse( { - 'status': 200, - 'response': data, + "status": 200, + "response": data, }, - encoder=BaseTapisResultSerializer + encoder=BaseTapisResultSerializer, ) @@ -745,36 +750,54 @@ def getPublicApps(self, user): # Traverse category records in descending priority for category in portal_app_categories.order_by("-priority"): - # Retrieve all apps known to the portal in that category - portal_apps = list(AppTrayEntry.objects.all().filter(available=True, category=category, appType='tapis') - .order_by(Coalesce('label', 'appId')).values('appId', 'appType', 'html', 'icon', 'label', 'version')) + portal_apps = list( + AppTrayEntry.objects.all() + .filter(available=True, category=category, appType="tapis") + .order_by(Coalesce("label", "appId")) + .values("appId", "appType", "html", "icon", "label", "version") + ) # Only return Tapis apps that are known to exist and are enabled tapis_apps = [] for portal_app in portal_apps: - portal_app_id = (portal_app['appId'], portal_app['version']) if portal_app['version'] else portal_app['appId'] + portal_app_id = ( + (portal_app["appId"], portal_app["version"]) if portal_app["version"] else portal_app["appId"] + ) # Look for matching app in tapis apps list, and append tapis app label if portal app has no label - matching_app = next((x for x in sorted(apps_listing, key=lambda y: y.version) if portal_app_id in [x.id, (x.id, x.version)]), None) + matching_app = next( + ( + x + for x in sorted(apps_listing, key=lambda y: y.version) + if portal_app_id in [x.id, (x.id, x.version)] + ), + None, + ) if matching_app: - tapis_apps.append({**portal_app, 'label': portal_app['label'] or matching_app.notes.label}) + tapis_apps.append({**portal_app, "label": portal_app["label"] or matching_app.notes.label}) - html_apps = list(AppTrayEntry.objects.all().filter(available=True, category=category, appType='html') - .order_by(Coalesce('label', 'appId')).values('appId', 'appType', 'html', 'icon', 'label', 'version')) + html_apps = list( + AppTrayEntry.objects.all() + .filter(available=True, category=category, appType="html") + .order_by(Coalesce("label", "appId")) + .values("appId", "appType", "html", "icon", "label", "version") + ) categoryResult = { "title": category.category, - "apps": [{k: v for k, v in tapis_app.items() if v != ''} for tapis_app in tapis_apps] # Remove empty strings from response + "apps": [ + {k: v for k, v in tapis_app.items() if v != ""} for tapis_app in tapis_apps + ], # Remove empty strings from response } # Add html apps to html_definitions for app in html_apps: - html_definitions[app['appId']] = app + html_definitions[app["appId"]] = app categoryResult["apps"].append(app) - categoryResult["apps"] = sorted(categoryResult["apps"], key=lambda app: app['label'] or app['appId']) + categoryResult["apps"] = sorted(categoryResult["apps"], key=lambda app: app["label"] or app["appId"]) categories.append(categoryResult) return categories, html_definitions @@ -806,31 +829,21 @@ def get(self, request): shared_apps = self.getSharedApps(request.user) # Only return tabs that are non-empty - tabs = list( - filter(lambda tab: len(tab["apps"]) > 0, [my_apps] + [shared_apps] + tabs) - ) + tabs = list(filter(lambda tab: len(tab["apps"]) > 0, [my_apps] + [shared_apps] + tabs)) - return JsonResponse( - { - "tabs": tabs, - "htmlDefinitions": html_definitions - }, - encoder=BaseTapisResultSerializer - ) + return JsonResponse({"tabs": tabs, "htmlDefinitions": html_definitions}, encoder=BaseTapisResultSerializer) -@method_decorator(login_required, name='dispatch') +@method_decorator(login_required, name="dispatch") class TapisAppsView(BaseApiView): def get(self, request, operation=None): try: client = request.user.tapis_oauth.client except AttributeError: - return JsonResponse( - {'message': 'This view requires authentication.'}, - status=403) + return JsonResponse({"message": "This view requires authentication."}, status=403) get_params = request.GET.dict() - logger.info('user:%s op:%s query_params:%s' % (request.user.username, operation, get_params)) + logger.info("user:%s op:%s query_params:%s" % (request.user.username, operation, get_params)) response = tapis_get_handler(client, operation, **get_params) - return JsonResponse({'data': response}) + return JsonResponse({"data": response}) diff --git a/server/portal/apps/workspace/api/views_unit_test.py b/server/portal/apps/workspace/api/views_unit_test.py index 0e927aed6f..832dd3aadc 100644 --- a/server/portal/apps/workspace/api/views_unit_test.py +++ b/server/portal/apps/workspace/api/views_unit_test.py @@ -39,10 +39,7 @@ def tapis_apps_list(): app_tray_data = json.load(f) for entry in app_tray_data: - if ( - entry.get("model") == "workspace.apptrayentry" - and entry["fields"]["appType"] == "tapis" - ): + if entry.get("model") == "workspace.apptrayentry" and entry["fields"]["appType"] == "tapis": app = TapisResult( **{ "id": entry["fields"]["appId"], @@ -57,9 +54,7 @@ def tapis_apps_list(): @pytest.fixture def tapis_get_systems_list(): system_list = [] - with open( - os.path.join(settings.BASE_DIR, "fixtures/tapis/systems/listing.json") - ) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tapis/systems/listing.json")) as f: systems = json.load(f) for entry in systems: system_list.append(TapisResult(**entry)) @@ -192,9 +187,7 @@ def test_job_post_is_logged_for_metrics( # Ensure metric-related logging is being performed logging_metric_mock.assert_called_with( - "user:{} is submitting job:{}".format( - authenticated_user.username, tapis_job_submission - ) + "user:{} is submitting job:{}".format(authenticated_user.username, tapis_job_submission) ) @@ -227,9 +220,7 @@ def test_get_jobs_bad_offset(client, authenticated_user, mock_tapis_client): def test_tray_get_private_apps(authenticated_user, mock_tapis_client, mocker): view = AppsTrayView() - app = TapisResult( - **{"id": "myapp-0.1", "version": "0.1", "notes": {"label": "Matlab"}} - ) + app = TapisResult(**{"id": "myapp-0.1", "version": "0.1", "notes": {"label": "Matlab"}}) mock_tapis_client.apps.getApps.return_value = [app] expected_list = [ { @@ -246,9 +237,7 @@ def test_tray_get_private_apps(authenticated_user, mock_tapis_client, mocker): @pytest.mark.django_db(transaction=True) -def test_tray_get_public_apps( - django_db_blocker, mock_tapis_client, authenticated_user, tapis_apps_list -): +def test_tray_get_public_apps(django_db_blocker, mock_tapis_client, authenticated_user, tapis_apps_list): # Load fixtures with django_db_blocker.unblock(): call_command("loaddata", "app-tray.json") @@ -288,9 +277,7 @@ def test_get_app_dynamic_exec_sys( with django_db_blocker.unblock(): call_command("loaddata", "app-tray.json") - with open( - os.path.join(settings.BASE_DIR, "fixtures/tapis/apps/hello-world-app-def.json") - ) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures/tapis/apps/hello-world-app-def.json")) as f: app = json.load(f) if dynamic_exec_system: app["notes"]["dynamicExecSystems"] = [ diff --git a/server/portal/apps/workspace/apps.py b/server/portal/apps/workspace/apps.py index bb97d35631..552cf6d8ec 100644 --- a/server/portal/apps/workspace/apps.py +++ b/server/portal/apps/workspace/apps.py @@ -2,4 +2,4 @@ class WorkspaceConfig(AppConfig): - name = 'portal.apps.workspace' + name = "portal.apps.workspace" diff --git a/server/portal/apps/workspace/management/commands/import-apps.py b/server/portal/apps/workspace/management/commands/import-apps.py index 33fba74c8d..e9b86da5be 100644 --- a/server/portal/apps/workspace/management/commands/import-apps.py +++ b/server/portal/apps/workspace/management/commands/import-apps.py @@ -3,10 +3,7 @@ from django.conf import settings from tapipy.errors import NotFoundError from portal.libs.agave.utils import service_account -from portal.apps.workspace.models import ( - AppTrayCategory, - AppTrayEntry -) +from portal.apps.workspace.models import AppTrayCategory, AppTrayEntry logger = logging.getLogger(__name__) @@ -22,14 +19,14 @@ class Command(BaseCommand): help = "Import all app metadata from the tenant into AppTrayCategory and AppTrayEntry models" def add_arguments(self, parser): - parser.add_argument('-n', '--names', type=str, help="Portal app names to import") - parser.add_argument('-c', '--clean', action='store_true', help="Remove nonexistant apps") - parser.add_argument('-s', '--skip', action='store_true', help="Skip import") + parser.add_argument("-n", "--names", type=str, help="Portal app names to import") + parser.add_argument("-c", "--clean", action="store_true", help="Remove nonexistant apps") + parser.add_argument("-s", "--skip", action="store_true", help="Skip import") def clean(self): client = service_account() - portal_apps = AppTrayEntry.objects.filter(appType='tapis') + portal_apps = AppTrayEntry.objects.filter(appType="tapis") if portal_apps: logger.info("Deleting app entries with no corresponding app in tenant") @@ -40,7 +37,9 @@ def clean(self): else: client.apps.getAppLatestVersion(appId=app.appId) except NotFoundError: - logger.info("App not found. id: {} and version: {}:. Deleting...".format(app.appId, app.version or 'None')) + logger.info( + "App not found. id: {} and version: {}:. Deleting...".format(app.appId, app.version or "None") + ) app.delete() def import_apps(self, portal_names): @@ -54,16 +53,14 @@ def import_apps(self, portal_names): data = client.apps.searchAppsRequestBody(search=query, select="id,notes,version") for app in data: try: - category = app.notes.get('category') or "Uncategorized" - category_entry, _ = AppTrayCategory.objects.get_or_create( - category=category - ) + category = app.notes.get("category") or "Uncategorized" + category_entry, _ = AppTrayCategory.objects.get_or_create(category=category) app_entry = AppTrayEntry.objects.get_or_create( category=category_entry, - icon=app.notes.get('icon') or "", - version=app.get('version') or "", - appId=app.get('id') + icon=app.notes.get("icon") or "", + version=app.get("version") or "", + appId=app.get("id"), ) logger.info("Imported {}".format(app_entry)) @@ -72,12 +69,12 @@ def import_apps(self, portal_names): logger.info("Following app could not be imported: {}".format(app)) def handle(self, *args, **options): - if options['clean']: + if options["clean"]: self.clean() - if not options['skip']: - if options['names']: - portal_names = options['names'].split(',') + if not options["skip"]: + if options["names"]: + portal_names = options["names"].split(",") else: portal_names = settings.PORTAL_APPS_NAMES_SEARCH diff --git a/server/portal/apps/workspace/management/commands/import-jobs.py b/server/portal/apps/workspace/management/commands/import-jobs.py index 7925817cd8..3cc45d29a3 100644 --- a/server/portal/apps/workspace/management/commands/import-jobs.py +++ b/server/portal/apps/workspace/management/commands/import-jobs.py @@ -29,9 +29,7 @@ def handle(self, *args, **options): for job in jobs: if not any(existing.jobId == job["id"] for existing in userjobs): job = JobSubmission.objects.create( - user=user, - jobId=job["id"], - time=dateutil.parser.parse(job["created"]) + user=user, jobId=job["id"], time=dateutil.parser.parse(job["created"]) ) offset += 100 done = len(jobs) < 100 diff --git a/server/portal/apps/workspace/management/commands/unit_test.py b/server/portal/apps/workspace/management/commands/unit_test.py index 6798da9c5a..a1466309ab 100644 --- a/server/portal/apps/workspace/management/commands/unit_test.py +++ b/server/portal/apps/workspace/management/commands/unit_test.py @@ -8,35 +8,26 @@ @pytest.mark.django_db(transaction=True) class TestImportJobs(TransactionTestCase): - fixtures = ['users'] + fixtures = ["users"] def setUp(self): - self.mock_client_patcher = patch('portal.apps.workspace.management.commands.import-jobs.service_account') + self.mock_client_patcher = patch("portal.apps.workspace.management.commands.import-jobs.service_account") self.mock_client = self.mock_client_patcher.start() self.user = get_user_model().objects.get(username="username") def tearDown(self): self.mock_client_patcher.stop() - @patch('portal.apps.workspace.management.commands.import-jobs.get_user_model') + @patch("portal.apps.workspace.management.commands.import-jobs.get_user_model") def test_import(self, mock_user_model): mock_user_model.return_value.objects.all.return_value = [self.user] - JobSubmission.objects.create( - jobId="1234", - user=self.user - ) + JobSubmission.objects.create(jobId="1234", user=self.user) self.mock_client.return_value.jobs.list.return_value = [ - { - "id": "1234", - "created": "2019-10-29T18:30:13Z" - }, - { - "id": "5678", - "created": "2019-10-29T19:30:13Z" - } + {"id": "1234", "created": "2019-10-29T18:30:13Z"}, + {"id": "5678", "created": "2019-10-29T19:30:13Z"}, ] - call_command('import-jobs') + call_command("import-jobs") result = JobSubmission.objects.all().filter(user=self.user) self.assertEqual(len(result), 2) diff --git a/server/portal/apps/workspace/migrations/0001_initial.py b/server/portal/apps/workspace/migrations/0001_initial.py index 68eec7e3ad..c4ef3f9a6c 100644 --- a/server/portal/apps/workspace/migrations/0001_initial.py +++ b/server/portal/apps/workspace/migrations/0001_initial.py @@ -7,7 +7,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -16,12 +15,17 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='JobSubmission', + name="JobSubmission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('time', models.DateTimeField(default=datetime.datetime.now)), - ('jobId', models.CharField(max_length=300)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("time", models.DateTimeField(default=datetime.datetime.now)), + ("jobId", models.CharField(max_length=300)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), ], ), ] diff --git a/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py b/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py index b278b3659d..0b38d84553 100644 --- a/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py +++ b/server/portal/apps/workspace/migrations/0001_squashed_0006_alter_jobsubmission_data.py @@ -7,8 +7,15 @@ class Migration(migrations.Migration): - - replaces = [('workspace', '0001_initial'), ('workspace', '0002_auto_20200218_2115'), ('workspace', '0003_apptraycategory_apptrayentry'), ('workspace', '0004_jobsubmission_data'), ('workspace', '0004_auto_20221013_2240'), ('workspace', '0005_merge_20230119_1627'), ('workspace', '0006_alter_jobsubmission_data')] + replaces = [ + ("workspace", "0001_initial"), + ("workspace", "0002_auto_20200218_2115"), + ("workspace", "0003_apptraycategory_apptrayentry"), + ("workspace", "0004_jobsubmission_data"), + ("workspace", "0004_auto_20221013_2240"), + ("workspace", "0005_merge_20230119_1627"), + ("workspace", "0006_alter_jobsubmission_data"), + ] initial = True @@ -18,35 +25,79 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='AppTrayCategory', + name="AppTrayCategory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('category', models.CharField(help_text='A category for the app tray', max_length=64)), - ('priority', models.IntegerField(default=0, help_text='Category priority, where higher priority tabs appear before lower ones')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("category", models.CharField(help_text="A category for the app tray", max_length=64)), + ( + "priority", + models.IntegerField( + default=0, help_text="Category priority, where higher priority tabs appear before lower ones" + ), + ), ], ), migrations.CreateModel( - name='AppTrayEntry', + name="AppTrayEntry", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('label', models.CharField(blank=True, help_text='The display name of this app in the App Tray', max_length=64)), - ('icon', models.CharField(blank=True, help_text='The icon to apply to this application', max_length=64)), - ('version', models.CharField(blank=True, help_text='The version number of the app', max_length=64)), - ('appId', models.CharField(help_text='The id of this app. The app id + version denotes a unique app', max_length=64)), - ('appType', models.CharField(choices=[('tapis', 'Tapis'), ('html', 'HTML')], default='tapis', help_text='Application type', max_length=10)), - ('html', models.TextField(blank=True, default='', help_text='HTML definition to display when Application is loaded')), - ('available', models.BooleanField(default=True, help_text='App visibility in app tray')), - ('category', models.ForeignKey(help_text='The App Category for this app entry', on_delete=django.db.models.deletion.CASCADE, related_name='+', to='workspace.apptraycategory')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "label", + models.CharField( + blank=True, help_text="The display name of this app in the App Tray", max_length=64 + ), + ), + ( + "icon", + models.CharField(blank=True, help_text="The icon to apply to this application", max_length=64), + ), + ("version", models.CharField(blank=True, help_text="The version number of the app", max_length=64)), + ( + "appId", + models.CharField( + help_text="The id of this app. The app id + version denotes a unique app", max_length=64 + ), + ), + ( + "appType", + models.CharField( + choices=[("tapis", "Tapis"), ("html", "HTML")], + default="tapis", + help_text="Application type", + max_length=10, + ), + ), + ( + "html", + models.TextField( + blank=True, default="", help_text="HTML definition to display when Application is loaded" + ), + ), + ("available", models.BooleanField(default=True, help_text="App visibility in app tray")), + ( + "category", + models.ForeignKey( + help_text="The App Category for this app entry", + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="workspace.apptraycategory", + ), + ), ], ), migrations.CreateModel( - name='JobSubmission', + name="JobSubmission", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('time', models.DateTimeField(default=django.utils.timezone.now)), - ('jobId', models.CharField(max_length=300)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), - ('data', models.JSONField(null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("time", models.DateTimeField(default=django.utils.timezone.now)), + ("jobId", models.CharField(max_length=300)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="+", to=settings.AUTH_USER_MODEL + ), + ), + ("data", models.JSONField(null=True)), ], ), ] diff --git a/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py b/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py index 9b0bf91f15..df8bfbfb31 100644 --- a/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py +++ b/server/portal/apps/workspace/migrations/0002_auto_20200218_2115.py @@ -5,15 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0001_initial'), + ("workspace", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='jobsubmission', - name='time', + model_name="jobsubmission", + name="time", field=models.DateTimeField(default=django.utils.timezone.now), ), ] diff --git a/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py b/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py index 1504ce08c4..25f43b388b 100644 --- a/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py +++ b/server/portal/apps/workspace/migrations/0003_apptraycategory_apptrayentry.py @@ -5,37 +5,80 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0002_auto_20200218_2115'), + ("workspace", "0002_auto_20200218_2115"), ] operations = [ migrations.CreateModel( - name='AppTrayCategory', + name="AppTrayCategory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('category', models.CharField(help_text='A category for the app tray', max_length=64)), - ('priority', models.IntegerField(default=0, help_text='Category priority, where higher priority tabs appear before lower ones')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("category", models.CharField(help_text="A category for the app tray", max_length=64)), + ( + "priority", + models.IntegerField( + default=0, help_text="Category priority, where higher priority tabs appear before lower ones" + ), + ), ], ), migrations.CreateModel( - name='AppTrayEntry', + name="AppTrayEntry", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(blank=True, help_text='The short name of the Agave app', max_length=64)), - ('label', models.CharField(help_text='The display name of this app in the App Tray', max_length=64)), - ('icon', models.CharField(blank=True, help_text='The icon to apply to this application', max_length=64)), - ('version', models.CharField(blank=True, help_text='The version number of the app', max_length=64)), - ('revision', models.CharField(blank=True, help_text='The revision of the app', max_length=3)), - ('appId', models.CharField(blank=True, help_text='Specifying an app by id will override all other app specifications', max_length=64)), - ('lastRetrieved', models.CharField(help_text='The latest retrieved version of this app', max_length=64)), - ('appType', models.CharField(choices=[('agave', 'agave'), ('html', 'html')], default='agave', help_text='Application type', max_length=10)), - ('html', models.TextField(blank=True, default='', help_text='HTML definition to display when Application is loaded')), - ('htmlId', models.CharField(blank=True, help_text='A non-agave portal specific ID for an HTML app', max_length=64)), - ('available', models.BooleanField(default=True, help_text='App visibility in app tray')), - ('shortDescription', models.TextField(blank=True, default='', help_text='App short description')), - ('category', models.ForeignKey(help_text='The App Category for this app entry', on_delete=django.db.models.deletion.CASCADE, related_name='+', to='workspace.AppTrayCategory')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(blank=True, help_text="The short name of the Agave app", max_length=64)), + ("label", models.CharField(help_text="The display name of this app in the App Tray", max_length=64)), + ( + "icon", + models.CharField(blank=True, help_text="The icon to apply to this application", max_length=64), + ), + ("version", models.CharField(blank=True, help_text="The version number of the app", max_length=64)), + ("revision", models.CharField(blank=True, help_text="The revision of the app", max_length=3)), + ( + "appId", + models.CharField( + blank=True, + help_text="Specifying an app by id will override all other app specifications", + max_length=64, + ), + ), + ( + "lastRetrieved", + models.CharField(help_text="The latest retrieved version of this app", max_length=64), + ), + ( + "appType", + models.CharField( + choices=[("agave", "agave"), ("html", "html")], + default="agave", + help_text="Application type", + max_length=10, + ), + ), + ( + "html", + models.TextField( + blank=True, default="", help_text="HTML definition to display when Application is loaded" + ), + ), + ( + "htmlId", + models.CharField( + blank=True, help_text="A non-agave portal specific ID for an HTML app", max_length=64 + ), + ), + ("available", models.BooleanField(default=True, help_text="App visibility in app tray")), + ("shortDescription", models.TextField(blank=True, default="", help_text="App short description")), + ( + "category", + models.ForeignKey( + help_text="The App Category for this app entry", + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="workspace.AppTrayCategory", + ), + ), ], ), ] diff --git a/server/portal/apps/workspace/migrations/0004_auto_20221013_2240.py b/server/portal/apps/workspace/migrations/0004_auto_20221013_2240.py index 6aaeb5837a..bafdbdfd88 100644 --- a/server/portal/apps/workspace/migrations/0004_auto_20221013_2240.py +++ b/server/portal/apps/workspace/migrations/0004_auto_20221013_2240.py @@ -4,45 +4,51 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0003_apptraycategory_apptrayentry'), + ("workspace", "0003_apptraycategory_apptrayentry"), ] operations = [ migrations.RemoveField( - model_name='apptrayentry', - name='htmlId', + model_name="apptrayentry", + name="htmlId", ), migrations.RemoveField( - model_name='apptrayentry', - name='lastRetrieved', + model_name="apptrayentry", + name="lastRetrieved", ), migrations.RemoveField( - model_name='apptrayentry', - name='name', + model_name="apptrayentry", + name="name", ), migrations.RemoveField( - model_name='apptrayentry', - name='revision', + model_name="apptrayentry", + name="revision", ), migrations.RemoveField( - model_name='apptrayentry', - name='shortDescription', + model_name="apptrayentry", + name="shortDescription", ), migrations.AlterField( - model_name='apptrayentry', - name='appId', - field=models.CharField(help_text='The id of this app. The app id + version denotes a unique app', max_length=64), + model_name="apptrayentry", + name="appId", + field=models.CharField( + help_text="The id of this app. The app id + version denotes a unique app", max_length=64 + ), ), migrations.AlterField( - model_name='apptrayentry', - name='appType', - field=models.CharField(choices=[('tapis', 'Tapis'), ('html', 'HTML')], default='tapis', help_text='Application type', max_length=10), + model_name="apptrayentry", + name="appType", + field=models.CharField( + choices=[("tapis", "Tapis"), ("html", "HTML")], + default="tapis", + help_text="Application type", + max_length=10, + ), ), migrations.AlterField( - model_name='apptrayentry', - name='label', - field=models.CharField(blank=True, help_text='The display name of this app in the App Tray', max_length=64), + model_name="apptrayentry", + name="label", + field=models.CharField(blank=True, help_text="The display name of this app in the App Tray", max_length=64), ), ] diff --git a/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py b/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py index d5b6fb9d8c..f42683aefd 100644 --- a/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py +++ b/server/portal/apps/workspace/migrations/0004_jobsubmission_data.py @@ -5,15 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0003_apptraycategory_apptrayentry'), + ("workspace", "0003_apptraycategory_apptrayentry"), ] operations = [ migrations.AddField( - model_name='jobsubmission', - name='data', + model_name="jobsubmission", + name="data", field=portal.utils.fields.JSONField(null=True), ), ] diff --git a/server/portal/apps/workspace/migrations/0005_merge_20230119_1627.py b/server/portal/apps/workspace/migrations/0005_merge_20230119_1627.py index 35d26e1ecd..3432c1897c 100644 --- a/server/portal/apps/workspace/migrations/0005_merge_20230119_1627.py +++ b/server/portal/apps/workspace/migrations/0005_merge_20230119_1627.py @@ -4,11 +4,9 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0004_jobsubmission_data'), - ('workspace', '0004_auto_20221013_2240'), + ("workspace", "0004_jobsubmission_data"), + ("workspace", "0004_auto_20221013_2240"), ] - operations = [ - ] + operations = [] diff --git a/server/portal/apps/workspace/migrations/0006_alter_jobsubmission_data.py b/server/portal/apps/workspace/migrations/0006_alter_jobsubmission_data.py index ce934e1371..09adea2276 100644 --- a/server/portal/apps/workspace/migrations/0006_alter_jobsubmission_data.py +++ b/server/portal/apps/workspace/migrations/0006_alter_jobsubmission_data.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('workspace', '0005_merge_20230119_1627'), + ("workspace", "0005_merge_20230119_1627"), ] operations = [ migrations.AlterField( - model_name='jobsubmission', - name='data', + model_name="jobsubmission", + name="data", field=models.JSONField(null=True), ), ] diff --git a/server/portal/apps/workspace/models.py b/server/portal/apps/workspace/models.py index af7dde3576..2b61566370 100644 --- a/server/portal/apps/workspace/models.py +++ b/server/portal/apps/workspace/models.py @@ -9,11 +9,8 @@ class JobSubmission(models.Model): Used for tracking jobs that originate from this portal for filtering purposes """ - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - related_name="+", - on_delete=models.CASCADE - ) + + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="+", on_delete=models.CASCADE) # Timestamp for event time = models.DateTimeField(default=timezone.now) @@ -26,29 +23,27 @@ class JobSubmission(models.Model): class AppTrayCategory(models.Model): - category = models.CharField(help_text='A category for the app tray', max_length=64) - priority = models.IntegerField(help_text='Category priority, where higher priority tabs appear before lower ones', default=0) + category = models.CharField(help_text="A category for the app tray", max_length=64) + priority = models.IntegerField( + help_text="Category priority, where higher priority tabs appear before lower ones", default=0 + ) def __str__(self): return "%s" % (self.category) class AppTrayEntry(models.Model): - APP_TYPES = [('tapis', 'Tapis'), ('html', 'HTML')] - label = models.CharField(help_text='The display name of this app in the App Tray', max_length=64, blank=True) - icon = models.CharField(help_text='The icon to apply to this application', max_length=64, blank=True) - version = models.CharField(help_text='The version number of the app', max_length=64, blank=True) - appId = models.CharField(help_text='The id of this app. The app id + version denotes a unique app', max_length=64) - appType = models.CharField(help_text='Application type', max_length=10, choices=APP_TYPES, default='tapis') - html = models.TextField(help_text='HTML definition to display when Application is loaded', - default="", blank=True) - available = models.BooleanField(help_text='App visibility in app tray', default=True) + APP_TYPES = [("tapis", "Tapis"), ("html", "HTML")] + label = models.CharField(help_text="The display name of this app in the App Tray", max_length=64, blank=True) + icon = models.CharField(help_text="The icon to apply to this application", max_length=64, blank=True) + version = models.CharField(help_text="The version number of the app", max_length=64, blank=True) + appId = models.CharField(help_text="The id of this app. The app id + version denotes a unique app", max_length=64) + appType = models.CharField(help_text="Application type", max_length=10, choices=APP_TYPES, default="tapis") + html = models.TextField(help_text="HTML definition to display when Application is loaded", default="", blank=True) + available = models.BooleanField(help_text="App visibility in app tray", default=True) category = models.ForeignKey( - AppTrayCategory, - related_name="+", - help_text="The App Category for this app entry", - on_delete=models.CASCADE + AppTrayCategory, related_name="+", help_text="The App Category for this app entry", on_delete=models.CASCADE ) def __str__(self): @@ -57,5 +52,5 @@ def __str__(self): return "%s%s%s" % ( f"{self.label}: " if self.label else "", self.appId, - f"-{self.version}" if self.version else "" + f"-{self.version}" if self.version else "", ) diff --git a/server/portal/apps/workspace/models_unit_test.py b/server/portal/apps/workspace/models_unit_test.py index 555ed719c0..6a1fc6cd96 100644 --- a/server/portal/apps/workspace/models_unit_test.py +++ b/server/portal/apps/workspace/models_unit_test.py @@ -1,25 +1,15 @@ - -from portal.apps.workspace.models import ( - JobSubmission, - AppTrayCategory, - AppTrayEntry -) +from portal.apps.workspace.models import JobSubmission, AppTrayCategory, AppTrayEntry def test_job_submission_model(django_db_reset_sequences, regular_user): - event = JobSubmission.objects.create( - user=regular_user, - jobId="1234" - ) + event = JobSubmission.objects.create(user=regular_user, jobId="1234") event = JobSubmission.objects.all()[0] assert event.user == regular_user assert event.jobId == "1234" def test_app_tray_models(django_db_reset_sequences): - category = AppTrayCategory.objects.create( - category="test_category" - ) + category = AppTrayCategory.objects.create(category="test_category") assert str(AppTrayCategory.objects.all()[0]) == "test_category" AppTrayEntry.objects.create( category=category, @@ -30,10 +20,5 @@ def test_app_tray_models(django_db_reset_sequences): appType="tapis", ) assert str(AppTrayEntry.objects.all()[0]) == "Matlab Latest: matlab-0.0.1" - htmlApp = AppTrayEntry.objects.create( - category=category, - appType="html", - label="Jupyter", - appId="jupyterhub" - ) + htmlApp = AppTrayEntry.objects.create(category=category, appType="html", label="Jupyter", appId="jupyterhub") assert str(htmlApp) == "Jupyter: jupyterhub (HTML)" diff --git a/server/portal/apps/workspace/unit_test.py b/server/portal/apps/workspace/unit_test.py index ba21e4e18b..e22af05f19 100644 --- a/server/portal/apps/workspace/unit_test.py +++ b/server/portal/apps/workspace/unit_test.py @@ -9,15 +9,15 @@ @pytest.mark.django_db(transaction=True) class TestAppsApiViews(TestCase): - fixtures = ['users', 'auth'] + fixtures = ["users", "auth"] @classmethod def setUpClass(cls): super(TestAppsApiViews, cls).setUpClass() - cls.mock_client_patcher = patch('portal.apps.auth.models.TapisOAuthToken.client') + cls.mock_client_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client") cls.mock_client = cls.mock_client_patcher.start() - cls.mock_get_user_data_patcher = patch('portal.apps.users.utils.get_user_data') - with open(os.path.join(settings.BASE_DIR, 'fixtures/tas/tas_user.json')) as f: + cls.mock_get_user_data_patcher = patch("portal.apps.users.utils.get_user_data") + with open(os.path.join(settings.BASE_DIR, "fixtures/tas/tas_user.json")) as f: tas_user = json.load(f) cls.mock_get_user_data = cls.mock_get_user_data_patcher.start() cls.mock_get_user_data.return_value = tas_user @@ -29,64 +29,49 @@ def tearDownClass(cls): cls.mock_client_patcher.stop() def setUp(self): - agave_path = os.path.join(settings.BASE_DIR, 'fixtures/agave') - with open( - os.path.join( - agave_path, - 'systems', - 'execution.json' - ) - ) as _file: + agave_path = os.path.join(settings.BASE_DIR, "fixtures/agave") + with open(os.path.join(agave_path, "systems", "execution.json")) as _file: self.agave_execution = json.load(_file) - with open(os.path.join(settings.BASE_DIR, 'fixtures', 'job-submission.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures", "job-submission.json")) as f: self.job_data = json.load(f) - with open(os.path.join(agave_path, 'apps', 'app-def.json')) as f: + with open(os.path.join(agave_path, "apps", "app-def.json")) as f: self.app_def = json.load(f) - with open(os.path.join(settings.BASE_DIR, 'fixtures', 'tas', 'tas_user.json')) as f: + with open(os.path.join(settings.BASE_DIR, "fixtures", "tas", "tas_user.json")) as f: self.tas_user = json.load(f) def test_apps_list(self): user = get_user_model().objects.get(username="username") self.client.force_login(user) - apps = [ - { - "id": "app-one", - "executionSystem": "stampede2" - }, - { - "id": "app-two", - "executionSystem": "stampede2" - } - ] + apps = [{"id": "app-one", "executionSystem": "stampede2"}, {"id": "app-two", "executionSystem": "stampede2"}] # need to do a return_value on the mock_client because # the calling signature is something like client = Agave(**kwargs).apps.list() self.mock_client.apps.getApps.return_value = apps - response = self.client.get('/api/workspace/apps/') + response = self.client.get("/api/workspace/apps/") data = response.json() # If the request is sent successfully, then I expect a response to be returned. self.assertEqual(response.status_code, 200) self.assertTrue("response" in data) - self.assertEqual(len(data["response"]['appListing']), 2) - self.assertTrue(data["response"]['appListing'] == apps) + self.assertEqual(len(data["response"]["appListing"]), 2) + self.assertTrue(data["response"]["appListing"] == apps) @pytest.mark.skip(reason="job post/notifications not implemented yet") - @patch('portal.apps.users.utils.get_user_data') + @patch("portal.apps.users.utils.get_user_data") def test_job_submit_notifications(self, tas_mock): tas_mock.return_value = self.tas_user user = get_user_model().objects.get(username="username") app_def = self.app_def - app_def['owner'] = user.username + app_def["owner"] = user.username self.mock_client.apps.get.return_value = app_def self.mock_client.jobs.submit.return_value = {"status": "ok"} self.client.force_login(user) - response = self.client.post('/api/workspace/jobs/', json.dumps(self.job_data), content_type="application/json") + response = self.client.post("/api/workspace/jobs/", json.dumps(self.job_data), content_type="application/json") data = response.json() self.assertTrue("response" in data) self.assertTrue(self.mock_client.jobs.submit.called) @@ -96,19 +81,19 @@ def test_job_submit_notifications(self, tas_mock): body = kwargs["body"] self.assertTrue("notifications" in body) notifications = body["notifications"] - pending = {'url': 'http://testserver/webhooks/jobs/', 'event': 'PENDING'} - finished = {'url': 'http://testserver/webhooks/jobs/', 'event': 'FINISHED'} + pending = {"url": "http://testserver/webhooks/jobs/", "event": "PENDING"} + finished = {"url": "http://testserver/webhooks/jobs/", "event": "FINISHED"} self.assertTrue(pending in notifications) self.assertTrue(finished in notifications) @pytest.mark.skip(reason="job post not implemented yet") - @patch('portal.apps.users.utils.get_user_data') + @patch("portal.apps.users.utils.get_user_data") def test_job_submit_parse_urls(self, tas_mock): tas_mock.return_value = self.tas_user user = get_user_model().objects.get(username="username") app_def = self.app_def - app_def['owner'] = user.username + app_def["owner"] = user.username self.mock_client.apps.get.return_value = app_def # the spaces should get quoted out @@ -117,7 +102,7 @@ def test_job_submit_parse_urls(self, tas_mock): self.mock_client.jobs.submit.return_value = {"status": "ok"} self.client.force_login(user) - response = self.client.post('/api/workspace/jobs/', json.dumps(job_data), content_type="application/json") + response = self.client.post("/api/workspace/jobs/", json.dumps(job_data), content_type="application/json") self.assertEqual(response.status_code, 200) args, kwargs = self.mock_client.jobs.submit.call_args body = kwargs["body"] diff --git a/server/portal/apps/workspace/urls.py b/server/portal/apps/workspace/urls.py index dd6fd2eac2..0220e5165d 100644 --- a/server/portal/apps/workspace/urls.py +++ b/server/portal/apps/workspace/urls.py @@ -1,8 +1,8 @@ -"""Workspace URLs -""" +"""Workspace URLs""" + from django.urls import re_path from portal.apps.workspace import views urlpatterns = [ - re_path(r'^', views.WorkspaceView.as_view(), name="workspace"), + re_path(r"^", views.WorkspaceView.as_view(), name="workspace"), ] diff --git a/server/portal/apps/workspace/views.py b/server/portal/apps/workspace/views.py index dc934a7199..ee83bfe391 100644 --- a/server/portal/apps/workspace/views.py +++ b/server/portal/apps/workspace/views.py @@ -2,6 +2,7 @@ .. :module: apps.workspace.views :synopsis: Views to handle Workspace """ + from django.views.generic.base import TemplateView from django.utils.decorators import method_decorator from django.views.decorators.csrf import ensure_csrf_cookie @@ -11,7 +12,8 @@ @method_decorator(login_required, name="dispatch") class WorkspaceView(TemplateView): """Workspace View""" - template_name = 'portal/apps/workspace/workspace.html' + + template_name = "portal/apps/workspace/workspace.html" @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): diff --git a/server/portal/asgi.py b/server/portal/asgi.py index 1b243c0422..620514d02a 100644 --- a/server/portal/asgi.py +++ b/server/portal/asgi.py @@ -11,13 +11,11 @@ import portal.apps.notifications.routing -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings") django.setup() -application = ProtocolTypeRouter({ - "http": get_asgi_application(), - "websocket": AuthMiddlewareStack( - URLRouter( - portal.apps.notifications.routing.websocket_urlpatterns - ) - ) -}) +application = ProtocolTypeRouter( + { + "http": get_asgi_application(), + "websocket": AuthMiddlewareStack(URLRouter(portal.apps.notifications.routing.websocket_urlpatterns)), + } +) diff --git a/server/portal/celery.py b/server/portal/celery.py index b93853356e..5ac2f1f9f9 100644 --- a/server/portal/celery.py +++ b/server/portal/celery.py @@ -1,19 +1,18 @@ - import os from celery import Celery from celery.schedules import crontab from django.conf import settings # set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings") -app = Celery('portal') +app = Celery("portal") # Using a string here means the worker don't have to serialize # the configuration object to child processes. # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. -app.config_from_object('django.conf:settings', namespace='CELERY') +app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django app configs. app.autodiscover_tasks() @@ -21,12 +20,12 @@ app.conf.beat_schedule = {} if settings.COMMUNITY_INDEX_SCHEDULE: - app.conf.beat_schedule['index_community'] = { - 'task': 'portal.apps.search.tasks.index_community_data', - 'schedule': crontab(**settings.COMMUNITY_INDEX_SCHEDULE) + app.conf.beat_schedule["index_community"] = { + "task": "portal.apps.search.tasks.index_community_data", + "schedule": crontab(**settings.COMMUNITY_INDEX_SCHEDULE), } @app.task(bind=True) def debug_task(self): - print(('Request: {0!r}'.format(self.request))) + print(("Request: {0!r}".format(self.request))) diff --git a/server/portal/exceptions/api.py b/server/portal/exceptions/api.py index 3f40f6a7a2..df8564ac4d 100644 --- a/server/portal/exceptions/api.py +++ b/server/portal/exceptions/api.py @@ -2,6 +2,7 @@ .. module: portal.exceptions.api :synopsis: Exceptions used within the API """ + from requests.exceptions import RequestException from requests.models import Response @@ -25,14 +26,7 @@ class ApiException(RequestException): """ - def __init__( - self, - message=None, - status=None, - extra=None, - *args, - **kwargs - ): + def __init__(self, message=None, status=None, extra=None, *args, **kwargs): """Custom exception based on :class:`~requests.exceptions.RequestException` diff --git a/server/portal/libs/agave/exceptions.py b/server/portal/libs/agave/exceptions.py index e745aa8ae3..afaba55e04 100644 --- a/server/portal/libs/agave/exceptions.py +++ b/server/portal/libs/agave/exceptions.py @@ -2,30 +2,35 @@ .. module: portal.libs.agave.exceptions :synopsis: Exceptions for the agave """ + import logging from portal.libs.exceptions import PortalLibException # pylint: disable=invalid-name logger = logging.getLogger(__name__) -METRICS = logging.getLogger('metrics.{}'.format(__name__)) +METRICS = logging.getLogger("metrics.{}".format(__name__)) # pylint: enable=invalid-name class ValidationError(PortalLibException): """Validation error""" + pass class CreationError(PortalLibException): """Creation Error""" + pass class DeletionError(PortalLibException): """Deletion Error""" + pass class APIError(PortalLibException): """API Error""" + pass diff --git a/server/portal/libs/agave/filter_mapping.py b/server/portal/libs/agave/filter_mapping.py index 3e81ecd65a..29ba546208 100644 --- a/server/portal/libs/agave/filter_mapping.py +++ b/server/portal/libs/agave/filter_mapping.py @@ -1,18 +1,5 @@ filter_mapping = { - "Audio": [ - "aac", - "aifc", - "aiff", - "amr", - "au", - "flac", - "m4a", - "mp3", - "ogg", - "ra", - "wav", - "wma" - ], + "Audio": ["aac", "aifc", "aiff", "amr", "au", "flac", "m4a", "mp3", "ogg", "ra", "wav", "wma"], "Code": [ "c", "css", @@ -59,17 +46,9 @@ "xsd", "xsl", "yaml", - "yml" - ], - "Documents": [ - "doc", - "dot", - "docx", - "docm", - "dotx", - "dotm", - "docb" + "yml", ], + "Documents": ["doc", "dot", "docx", "docm", "dotx", "dotm", "docb"], "Folders": [], "Images": [ "ai", @@ -88,35 +67,12 @@ "dicm", "dicom", "svs", - "tga" - ], - "Jupyter Notebook": [ - "ipynb" - ], - "PDF": [ - "pdf" - ], - "Presentation": [ - "ppt", - "pot", - "pps", - "pptx", - "pptm", - "potx", - "ppsx", - "ppsm", - "sldx", - "sldm" - ], - "Spreadsheet": [ - "xls", - "xlt", - "xlm", - "xlsx", - "xlsm", - "xltx", - "xltm" + "tga", ], + "Jupyter Notebook": ["ipynb"], + "PDF": ["pdf"], + "Presentation": ["ppt", "pot", "pps", "pptx", "pptm", "potx", "ppsx", "ppsm", "sldx", "sldm"], + "Spreadsheet": ["xls", "xlt", "xlm", "xlsx", "xlsm", "xltx", "xltm"], "Shape File": [ "shp", "shx", @@ -132,20 +88,10 @@ "mxs", "prj", "xml", - "cpg" - ], - "Text": [ - "err", - "log", - "out", - "txt" - ], - "ZIP": [ - "zip", - "tar", - "gz", - "tgz" + "cpg", ], + "Text": ["err", "log", "out", "txt"], + "ZIP": ["zip", "tar", "gz", "tgz"], "3D Visualization": [ "pov", "vrml", @@ -169,6 +115,6 @@ "collada", "3ds", "iges", - "step" - ] + "step", + ], } diff --git a/server/portal/libs/agave/models/applications.py b/server/portal/libs/agave/models/applications.py index 6262f40041..79a71ecdd8 100644 --- a/server/portal/libs/agave/models/applications.py +++ b/server/portal/libs/agave/models/applications.py @@ -2,15 +2,11 @@ .. :module:: portal.libs.agave.models.applications :synopsis: Classes to represent Agave Applications """ + from collections import namedtuple import logging from cached_property import cached_property_with_ttl -from portal.libs.agave.exceptions import ( - ValidationError, - CreationError, - DeletionError, - APIError -) +from portal.libs.agave.exceptions import ValidationError, CreationError, DeletionError, APIError from portal.libs.agave.models.base import BaseAgaveResource from portal.libs.agave.models.permissions import ApplicationPermissions @@ -24,72 +20,56 @@ class Application(BaseAgaveResource): """Agave application definition representation.""" _body_fields = [ - 'id', - 'name', - 'icon', - 'uuid', - 'parallelism', - 'default_processors_per_node', - 'default_memory_per_node', - 'default_node_count', - 'default_max_run_time', - 'default_queue', - 'version', - 'revision', - 'is_public', - 'help_uri', - 'label', - 'owner', - 'short_description', - 'long_description', - 'tags', - 'ontology', - 'execution_type', - 'execution_system', - 'deployment_path', - 'deployment_system', - 'template_path', - 'test_path', - 'checkpointable', - 'last_modified', - 'modules', - 'available', - 'inputs', - 'parameters', - 'outputs', - '_links' + "id", + "name", + "icon", + "uuid", + "parallelism", + "default_processors_per_node", + "default_memory_per_node", + "default_node_count", + "default_max_run_time", + "default_queue", + "version", + "revision", + "is_public", + "help_uri", + "label", + "owner", + "short_description", + "long_description", + "tags", + "ontology", + "execution_type", + "execution_system", + "deployment_path", + "deployment_system", + "template_path", + "test_path", + "checkpointable", + "last_modified", + "modules", + "available", + "inputs", + "parameters", + "outputs", + "_links", ] - _PARALLELISM = ['SERIAL', 'PARALLEL', 'PTHREAD'] - PARALLELISM = namedtuple( - 'Parallelism', - _PARALLELISM - )( - SERIAL='SERIAL', - PARALLEL='PARALLEL', - PTHREAD='PTHREAD' - ) + _PARALLELISM = ["SERIAL", "PARALLEL", "PTHREAD"] + PARALLELISM = namedtuple("Parallelism", _PARALLELISM)(SERIAL="SERIAL", PARALLEL="PARALLEL", PTHREAD="PTHREAD") - _EXECUTION_TYPE = ['ATMOSPHERE', 'HPC', 'CONDOR', 'CLI'] - EXECUTION_TYPE = namedtuple( - 'ExecutionType', - _EXECUTION_TYPE - )( - ATMOSPHERE='ATMOSPHERE', - HPC='HPC', - CONDOR='CONDOR', - CLI='CLI' + _EXECUTION_TYPE = ["ATMOSPHERE", "HPC", "CONDOR", "CLI"] + EXECUTION_TYPE = namedtuple("ExecutionType", _EXECUTION_TYPE)( + ATMOSPHERE="ATMOSPHERE", HPC="HPC", CONDOR="CONDOR", CLI="CLI" ) def __init__(self, client, id=None, load=True, ignore_error=404, **kwargs): - """Agave application definition representation. - """ + """Agave application definition representation.""" wrapped = {} if id is not None and load: # try: - wrapped = client.apps.get( - appId=id - ) + wrapped = client.apps.get(appId=id) # except HTTPError as exc: # if exc.response.status_code != ignore_error: # raise @@ -98,40 +78,40 @@ def __init__(self, client, id=None, load=True, ignore_error=404, **kwargs): super(Application, self).__init__(client, **wrapped) - self.id = getattr(self, 'id', None) - self.name = getattr(self, 'name', None) - self.icon = getattr(self, 'icon', None) - self.uuid = getattr(self, 'uuid', None) - self.parallelism = getattr(self, 'parallelism', None) - self.default_processors_per_node = getattr(self, 'default_processors_per_node', None) - self.default_memory_per_node = getattr(self, 'default_memory_per_node', None) - self.default_node_count = getattr(self, 'default_node_count', None) - self.default_max_run_time = getattr(self, 'default_max_run_time', None) - self.default_queue = getattr(self, 'default_queue', None) - self.version = getattr(self, 'version', None) - self.revision = getattr(self, 'revision', None) - self.is_public = getattr(self, 'is_public', False) - self.help_uri = getattr(self, 'help_uri', None) - self.label = getattr(self, 'label', '') - self.owner = getattr(self, 'owner', None) - self.short_description = getattr(self, 'short_description', '') - self.long_description = getattr(self, 'long_description', None) - self.tags = getattr(self, 'tags', []) - self.ontology = getattr(self, 'ontology', []) - self.execution_type = getattr(self, 'execution_type', None) - self.execution_system = getattr(self, 'execution_system', None) - self.deployment_path = getattr(self, 'deployment_path', None) - self.deployment_system = getattr(self, 'deployment_system', None) - self.template_path = getattr(self, 'template_path', None) - self.test_path = getattr(self, 'test_path', None) - self.checkpointable = getattr(self, 'checkpointable', False) - self.last_modified = getattr(self, 'last_modified', None) - self.modules = getattr(self, 'modules', []) - self.available = getattr(self, 'available', True) - self.inputs = getattr(self, 'inputs', []) - self.parameters = getattr(self, 'parameters', []) - self.outputs = getattr(self, 'outputs', []) - self._links = getattr(self, '_links', {}) + self.id = getattr(self, "id", None) + self.name = getattr(self, "name", None) + self.icon = getattr(self, "icon", None) + self.uuid = getattr(self, "uuid", None) + self.parallelism = getattr(self, "parallelism", None) + self.default_processors_per_node = getattr(self, "default_processors_per_node", None) + self.default_memory_per_node = getattr(self, "default_memory_per_node", None) + self.default_node_count = getattr(self, "default_node_count", None) + self.default_max_run_time = getattr(self, "default_max_run_time", None) + self.default_queue = getattr(self, "default_queue", None) + self.version = getattr(self, "version", None) + self.revision = getattr(self, "revision", None) + self.is_public = getattr(self, "is_public", False) + self.help_uri = getattr(self, "help_uri", None) + self.label = getattr(self, "label", "") + self.owner = getattr(self, "owner", None) + self.short_description = getattr(self, "short_description", "") + self.long_description = getattr(self, "long_description", None) + self.tags = getattr(self, "tags", []) + self.ontology = getattr(self, "ontology", []) + self.execution_type = getattr(self, "execution_type", None) + self.execution_system = getattr(self, "execution_system", None) + self.deployment_path = getattr(self, "deployment_path", None) + self.deployment_system = getattr(self, "deployment_system", None) + self.template_path = getattr(self, "template_path", None) + self.test_path = getattr(self, "test_path", None) + self.checkpointable = getattr(self, "checkpointable", False) + self.last_modified = getattr(self, "last_modified", None) + self.modules = getattr(self, "modules", []) + self.available = getattr(self, "available", True) + self.inputs = getattr(self, "inputs", []) + self.parameters = getattr(self, "parameters", []) + self.outputs = getattr(self, "outputs", []) + self._links = getattr(self, "_links", {}) self.exec_sys = None @@ -140,16 +120,13 @@ def permissions(self): """Permissions""" if self.is_public: - raise APIError( - "Cannot list permissions on public apps." - "\"is_public\" must be \"False\"." - ) + raise APIError('Cannot list permissions on public apps."is_public" must be "False".') pems = self._ac.apps.listPermissions(appId=self.id) return ApplicationPermissions(self._ac, pems, self) def __str__(self): - return '{id}'.format(id=self.id) + return "{id}".format(id=self.id) # def __repr__(self): # return '{class_name}(id={id}, label={label})'.format( @@ -168,113 +145,79 @@ def _populate_obj(self): def validate_available(self): """Validate self.available""" if not isinstance(self.available, bool): - raise ValidationError( - "'available' should be of type 'bool'" - ) + raise ValidationError("'available' should be of type 'bool'") def validate_inputs(self): """Validate self.inputs""" if self.inputs is None: - raise ValidationError( - "'inputs' should not be None" - ) + raise ValidationError("'inputs' should not be None") def validate_execution_system(self): """Validate self.execution_system""" if not self.execution_system: - raise ValidationError( - "'execution_system' should not be None" - ) + raise ValidationError("'execution_system' should not be None") def validate_test_path(self): """Validate self.test_path""" if not self.test_path: - raise ValidationError( - "'test_path' should not be empty" - ) + raise ValidationError("'test_path' should not be empty") def validate_deployment_path(self): """Validate self.deployment_path""" if not self.deployment_path: - raise ValidationError( - "'deployment_path' should not be empty" - ) + raise ValidationError("'deployment_path' should not be empty") def validate_template_path(self): """Validate self.version""" if not self.template_path: - raise ValidationError( - "'template_path' should not be empty" - ) + raise ValidationError("'template_path' should not be empty") def validate_deployment_system(self): """Validate self.deployment_system""" if not self.deployment_system: - raise ValidationError( - "'deployment_system' should not be empty" - ) + raise ValidationError("'deployment_system' should not be empty") def validate_name(self): """Validate self.name""" if not self.name: - raise ValidationError( - "'name' should not be empty" - ) + raise ValidationError("'name' should not be empty") def validate_parameters(self): """Validate self.parameters""" if self.parameters is None: - raise ValidationError( - "'parameters' should not be None" - ) + raise ValidationError("'parameters' should not be None") def validate_execution_type(self): """Validate self.execution_type""" types = self._EXECUTION_TYPE if self.execution_type not in types: - raise ValidationError( - "'execution_type' should be one of: {types}".format( - types=types - ) - ) + raise ValidationError("'execution_type' should be one of: {types}".format(types=types)) def validate_version(self): """Validate self.version""" if not self.version: - raise ValidationError( - "'version' should not be empty" - ) + raise ValidationError("'version' should not be empty") def validate_checkpointable(self): """Validate self.checkpointable""" if not isinstance(self.checkpointable, bool): - raise ValidationError( - "'checkpointable' should be of type 'bool'" - ) + raise ValidationError("'checkpointable' should be of type 'bool'") def validate_label(self): """Validate self.label""" if self.label is None: - raise ValidationError( - "'label' should not be None" - ) + raise ValidationError("'label' should not be None") def validate_parallelism(self): """Validate self.parallelism""" types = self._PARALLELISM if self.parallelism not in types: - raise ValidationError( - "'parallelism' should be one of: {types}".format( - types=types - ) - ) + raise ValidationError("'parallelism' should be one of: {types}".format(types=types)) def validate_short_description(self): """Validate self.short_description""" if self.short_description is None: - raise ValidationError( - "'short_description' should not be None" - ) + raise ValidationError("'short_description' should not be None") @staticmethod def remove(client, id): @@ -292,11 +235,8 @@ def create(cls, client, app_def): :param client: Agave client. :param dict app_def: `dict` representing application definition. """ - if app_def.get('id') is not None: - raise CreationError( - "Cannot specify \"id\" if creating application." - "\"id\" must be \"None\"." - ) + if app_def.get("id") is not None: + raise CreationError('Cannot specify "id" if creating application."id" must be "None".') resp = client.apps.add(body=app_def) return cls(client, **resp) @@ -319,21 +259,14 @@ def delete(self): the application id easily. """ if self.id is None: - raise DeletionError( - "Must specify \"id\" to delete application record" - ) - res = self._ac.apps.delete( - appId=self.id - ) + raise DeletionError('Must specify "id" to delete application record') + res = self._ac.apps.delete(appId=self.id) return res def update(self): """Update an application record.""" self.validate() - self._ac.apps.update( - appId=self.id, - body=self.to_dict() - ) + self._ac.apps.update(appId=self.id, body=self.to_dict()) def save(self): """Save this app record. @@ -342,28 +275,22 @@ def save(self): and the app does not already exist. """ self.validate() - self._ac.apps.add( - body=self.to_dict() - ) + self._ac.apps.add(body=self.to_dict()) def clone(self, client, depl_path=None, exec_sys=None, depl_sys=None, name=None, ver=None): - """Clone this application record - """ + """Clone this application record""" self.validate() if self.last_modified is None: - raise CreationError( - "Host app must already exist to be cloned." - "\"last_modified\" must not be \"None\"." - ) + raise CreationError('Host app must already exist to be cloned."last_modified" must not be "None".') body = { - 'action': 'clone', - 'deploymentPath': depl_path, - 'executionSystem': exec_sys, - 'deploymentSystem': depl_sys, - 'name': name, - 'version': ver + "action": "clone", + "deploymentPath": depl_path, + "executionSystem": exec_sys, + "deploymentSystem": depl_sys, + "name": name, + "version": ver, } resp = client.apps.manage(appId=self.id, body=body) @@ -371,19 +298,13 @@ def clone(self, client, depl_path=None, exec_sys=None, depl_sys=None, name=None, return Application(client, **resp) def publish(self, client): - """Publish this app - """ + """Publish this app""" self.validate() if self.is_public: - raise CreationError( - "Cannot publish public apps." - "\"is_public\" must be \"False\"." - ) + raise CreationError('Cannot publish public apps."is_public" must be "False".') - body = { - 'action': 'publish' - } + body = {"action": "publish"} resp = client.apps.manage(appId=self.id, body=body) diff --git a/server/portal/libs/agave/models/permissions.py b/server/portal/libs/agave/models/permissions.py index b18eabbe5b..7089897d34 100644 --- a/server/portal/libs/agave/models/permissions.py +++ b/server/portal/libs/agave/models/permissions.py @@ -2,6 +2,7 @@ .. :module:: portal.libs.agave.models.permissions :synopsis: Classes representing Agave permissions for different resources. """ + import logging from portal.libs.agave.exceptions import CreationError @@ -12,22 +13,23 @@ class Permission(object): """A single permission""" - READ = 'READ' - READ_WRITE = 'READ_WRITE' - READ_EXECUTE = 'READ_EXECUTE' - WRITE = 'WRITE' - WRITE_EXECUTE = 'WRITE_EXECUTE' - EXECUTE = 'EXECUTE' - ALL = 'ALL' - NONE = 'NONE' + + READ = "READ" + READ_WRITE = "READ_WRITE" + READ_EXECUTE = "READ_EXECUTE" + WRITE = "WRITE" + WRITE_EXECUTE = "WRITE_EXECUTE" + EXECUTE = "EXECUTE" + ALL = "ALL" + NONE = "NONE" def __init__(self, permission): - self.username = permission.get('username') - self.recursive = permission.get('recursive', False) - _pem = permission.get('permission') - self.read = _pem.get('read', False) - self.write = _pem.get('write', False) - self.execute = _pem.get('execute', False) + self.username = permission.get("username") + self.recursive = permission.get("recursive", False) + _pem = permission.get("permission") + self.read = _pem.get("read", False) + self.write = _pem.get("write", False) + self.execute = _pem.get("execute", False) @property def value(self): @@ -36,63 +38,44 @@ def value(self): This is the string value which should be one of the constants in this class. e.g. READ, READ_WRITE, etc... """ - pem = '' + pem = "" if self.read: - pem += 'READ' + pem += "READ" if self.write: - pem += '_WRITE' + pem += "_WRITE" if self.execute: - pem += '_EXECUTE' - pem = pem.strip('_') - if pem == 'READ_WRITE_EXECUTE': - pem = 'ALL' + pem += "_EXECUTE" + pem = pem.strip("_") + if pem == "READ_WRITE_EXECUTE": + pem = "ALL" elif not pem: - pem = 'NONE' + pem = "NONE" return pem def to_dict(self): """Dict representation""" - return { - 'username': self.username, - 'recursive': self.recursive, - 'permission': self.value - } + return {"username": self.username, "recursive": self.recursive, "permission": self.value} def __str__(self): """String -> self.username [R,W,E]""" - return '{username} {recursive}[{read}, {write}, {execute}]'.format( - username=self.username, - recursive=self.recursive, - read=self.read, - write=self.write, - execute=self.execute + return "{username} {recursive}[{read}, {write}, {execute}]".format( + username=self.username, recursive=self.recursive, read=self.read, write=self.write, execute=self.execute ) def __repr__(self): """Repr -> Permissions(username, R, W, E)""" - return ( - 'Permissions(' - '{username},' - 'recursive={recursive},' - 'read={read},' - 'write={write},' - 'execute={execute})' - ).format( - username=self.username, - recursive=self.recursive, - read=self.read, - write=self.write, - execute=self.execute + return ("Permissions({username},recursive={recursive},read={read},write={write},execute={execute})").format( + username=self.username, recursive=self.recursive, read=self.read, write=self.write, execute=self.execute ) def __eq__(self, other): """Equality""" return ( - self.username == other.username and - self.recursive == other.recursive and - self.read == other.read and - self.write == other.write and - self.execute == other.execute + self.username == other.username + and self.recursive == other.recursive + and self.read == other.read + and self.write == other.write + and self.execute == other.execute ) @@ -107,9 +90,7 @@ def __init__(self, client, permissions): agave's pems endpoint. """ self._ac = client - self.permissions = [ - Permission(permission) for permission in permissions - ] + self.permissions = [Permission(permission) for permission in permissions] self._updated_pems = [] @property @@ -122,8 +103,7 @@ def _mark_as_updated(self, pem): :param pem: :class:`Permission` object """ - pems = [pem_o for pem_o in self._updated_pems - if pem_o.username != pem.username] + pems = [pem_o for pem_o in self._updated_pems if pem_o.username != pem.username] pems.append(pem) self._updated_pems = pems @@ -132,21 +112,14 @@ def for_user(self, username): :param str username: Username. """ - res = [pem for pem in self.permissions - if pem.username == username] + res = [pem for pem in self.permissions if pem.username == username] if res: return res[0] # If the user doesn't have a permission in the list, it means the user # has no permission at all. - return Permission({ - 'username': username, - 'recursive': False, - 'permission': { - 'read': False, - 'write': False, - 'execute': False - } - }) + return Permission( + {"username": username, "recursive": False, "permission": {"read": False, "write": False, "execute": False}} + ) def can_user(self, username, pem): """Check if user has permission. @@ -158,14 +131,7 @@ def can_user(self, username, pem): val = getattr(permission, pem.lower(), False) return val - def add( - self, - username, - recursive=True, - read=False, - write=False, - execute=False - ): # pylint: disable=too-many-arguments + def add(self, username, recursive=True, read=False, write=False, execute=False): # pylint: disable=too-many-arguments """Add permission for user. :param str username: Username. @@ -175,8 +141,7 @@ def add( """ if not read and not write and not execute: raise CreationError("User must set at least one permission.") - pems = [pem for pem in self.permissions - if pem.username == username] + pems = [pem for pem in self.permissions if pem.username == username] if pems: pem = pems[0] pem.recursive = recursive @@ -185,15 +150,13 @@ def add( pem.execute = execute self._mark_as_updated(pem) else: - pem = Permission({ - 'username': username, - 'recursive': recursive, - 'permission': { - 'read': read, - 'write': write, - 'execute': execute + pem = Permission( + { + "username": username, + "recursive": recursive, + "permission": {"read": read, "write": write, "execute": execute}, } - }) + ) self.permissions.append(pem) self._mark_as_updated(pem) return self @@ -217,11 +180,9 @@ def save(self): """Save.""" for pem in self.to_update: res = self._ac.files.updatePermissions( - filePath=self.parent.path, - systemId=self.parent.system, - body=pem.to_dict() + filePath=self.parent.path, systemId=self.parent.system, body=pem.to_dict() ) - logger.debug('Saving file permissions response: %s', res) + logger.debug("Saving file permissions response: %s", res) return self @@ -243,14 +204,11 @@ def __init__(self, client, permissions, parent): def save(self): """Save.""" for pem in self.to_update: - self._ac.meta.updateMetadataPermissions( - uuid=self.parent.uuid, - body=pem.to_dict() - ) + self._ac.meta.updateMetadataPermissions(uuid=self.parent.uuid, body=pem.to_dict()) # We are using cached_property and this is the way to # invalidate the cache. - del self.parent.__dict__['permissions'] + del self.parent.__dict__["permissions"] return self @@ -271,10 +229,7 @@ def __init__(self, client, permissions, parent): def save(self): """Save.""" for pem in self.to_update: - res = self._ac.apps.updateApplicationPermissions( - appId=self.parent.id, - body=pem.to_dict() - ) - logger.debug('Saving applications permissions response: %s', res) + res = self._ac.apps.updateApplicationPermissions(appId=self.parent.id, body=pem.to_dict()) + logger.debug("Saving applications permissions response: %s", res) return self diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 31ef484048..48e3ffb753 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -15,11 +15,22 @@ from pathlib import Path from tapipy.errors import BaseTapyException from portal.apps.projects.workspace_operations.project_meta_operations import ( - add_file_associations, create_file_obj, get_entity, get_file_obj, - get_ordered_value, get_value, patch_entity_and_node, - patch_file_association, remove_file_obj_by_path) + add_file_associations, + create_file_obj, + get_entity, + get_file_obj, + get_ordered_value, + get_value, + patch_entity_and_node, + patch_file_association, + remove_file_obj_by_path, +) from portal.apps.projects.schema_models import constants -from portal.apps.projects.workspace_operations.graph_operations import get_or_create_trash_entity, get_root_node, get_node_from_path +from portal.apps.projects.workspace_operations.graph_operations import ( + get_or_create_trash_entity, + get_root_node, + get_node_from_path, +) logger = logging.getLogger(__name__) @@ -48,18 +59,20 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): List of dicts containing file metadata from Elasticsearch """ - pattern = 'regex:^(?!.Trash)' if 'hideTrash' in kwargs and kwargs['hideTrash'] else '' + pattern = "regex:^(?!.Trash)" if "hideTrash" in kwargs and kwargs["hideTrash"] else "" - raw_listing = client.files.listFiles(systemId=system, - path=quote(path, safe='/'), - pattern=pattern, - offset=int(offset), - limit=int(limit), - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + raw_listing = client.files.listFiles( + systemId=system, + path=quote(path, safe="/"), + pattern=pattern, + offset=int(offset), + limit=int(limit), + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) # Only add project metadata for projects scheme listings, never touch # the graph for My Data / Community / Public systems. - metadata_enabled = settings.PORTAL_PROJECTS_ENABLE_METADATA and kwargs.get('scheme') == 'projects' + metadata_enabled = settings.PORTAL_PROJECTS_ENABLE_METADATA and kwargs.get("scheme") == "projects" folder_entity_value = get_value(system, path) if metadata_enabled else None @@ -71,30 +84,30 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): if not metadata_enabled: value = None uuid = None - elif f.type == 'dir': + elif f.type == "dir": value = get_value(system, f.path) entity = get_entity(system, f.path) - uuid = entity.to_dict().get('uuid') if entity else None + uuid = entity.to_dict().get("uuid") if entity else None else: file_obj = get_file_obj(system, f.path) - value = get_ordered_value(constants.FILE, file_obj.get('value')) if file_obj else None - uuid = file_obj.get('uuid') if file_obj else None - - listing.append({ - 'uuid': uuid, - 'system': system, - 'type': 'dir' if f.type == 'dir' else 'file', - 'format': 'folder' if f.type == 'dir' else 'raw', - 'mimeType': f.mimeType, - 'path': f.path, - 'name': f.name, - 'length': f.size, - 'lastModified': f.lastModified, - '_links': { - 'self': {'href': f.url} - }, - 'metadata': value if value else None - }) + value = get_ordered_value(constants.FILE, file_obj.get("value")) if file_obj else None + uuid = file_obj.get("uuid") if file_obj else None + + listing.append( + { + "uuid": uuid, + "system": system, + "type": "dir" if f.type == "dir" else "file", + "format": "folder" if f.type == "dir" else "raw", + "mimeType": f.mimeType, + "path": f.path, + "name": f.name, + "length": f.size, + "lastModified": f.lastModified, + "_links": {"self": {"href": f.url}}, + "metadata": value if value else None, + } + ) except IndexError: logger.exception(f"Error parsing listing response from Tapis for system {system} and path {path}") # Return [] if the listing is empty. @@ -102,46 +115,50 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): # Update Elasticsearch after each listing. tapis_listing_indexer.delay(listing) - return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_entity_value} + return {"listing": listing, "reachedEnd": len(listing) < int(limit), "folder_metadata": folder_entity_value} def detail(client, system, path, *args, **kwargs): """ Retrieve the uuid for a file by parsing the query string in _links.metadata.href """ - _listing = client.files.listFiles(systemId=system, path=urllib.parse.quote(path), offset=0, limit=1, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + _listing = client.files.listFiles( + systemId=system, + path=urllib.parse.quote(path), + offset=0, + limit=1, + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) f = _listing[0] listing_res = { - 'system': system, - 'type': 'dir' if f.type == 'dir' else 'file', - 'format': 'folder' if f.type == 'dir' else 'raw', - 'mimeType': f.mimeType, - 'path': f"/{f.path}", - 'name': f.name, - 'length': f.size, - 'lastModified': f.lastModified, - '_links': { - 'self': {'href': f.url} - }} + "system": system, + "type": "dir" if f.type == "dir" else "file", + "format": "folder" if f.type == "dir" else "raw", + "mimeType": f.mimeType, + "path": f"/{f.path}", + "name": f.name, + "length": f.size, + "lastModified": f.lastModified, + "_links": {"self": {"href": f.url}}, + } return listing_res def iterate_listing(client, system, path, limit=100): """Iterate over a filesystem level yielding an attrdict for each file/folder - on the level. - :param str client: an Agave client - :param str system: system - :param str path: path to walk - :param int limit: Number of docs to retrieve per API call + on the level. + :param str client: an Agave client + :param str system: system + :param str path: path to walk + :param int limit: Number of docs to retrieve per API call - :rtype agavepy.agave.AttrDict + :rtype agavepy.agave.AttrDict """ offset = 0 while True: - page = listing(client, system, path, offset, limit)['listing'] + page = listing(client, system, path, offset, limit)["listing"] yield from page offset += limit if len(page) != limit: @@ -149,7 +166,7 @@ def iterate_listing(client, system, path, limit=100): break -def search(client, system, path='', offset=0, limit=100, query_string='', filter=None, **kwargs): +def search(client, system, path="", offset=0, limit=100, query_string="", filter=None, **kwargs): """ Perform a search for files using a query string. @@ -177,42 +194,37 @@ def search(client, system, path='', offset=0, limit=100, query_string='', filter # Perform a listing to ensure the user has access to the directory they're searching listing(client, system, path) - if filter == 'Folders': - filter_query = Q('term', **{'format': 'folder'}) + if filter == "Folders": + filter_query = Q("term", **{"format": "folder"}) else: filter_extensions = filter_mapping.get(filter, []) - filter_query = Q('terms', **{'name._pattern': filter_extensions}) + filter_query = Q("terms", **{"name._pattern": filter_extensions}) - ngram_query = Q("query_string", query=query_string, - fields=["name"], - minimum_should_match='100%', - default_operator='or') - match_query = Q("query_string", query=query_string, - fields=[ - "name._exact, name._pattern"], - default_operator='and') + ngram_query = Q( + "query_string", query=query_string, fields=["name"], minimum_should_match="100%", default_operator="or" + ) + match_query = Q("query_string", query=query_string, fields=["name._exact, name._pattern"], default_operator="and") search = IndexedFile.search() if query_string: search = search.query(ngram_query | match_query) else: # search without a query should just filter current path - search = search.sort('name._exact') - search = search.filter('term', **{'basePath._exact': path.strip('/')}) + search = search.sort("name._exact") + search = search.filter("term", **{"basePath._exact": path.strip("/")}) if filter: search = search.filter(filter_query) - if 'hideTrash' in kwargs and kwargs['hideTrash']: - hide_trash_query = ~Q("query_string", query='\\/.Trash\\/', fields=["path"]) + if "hideTrash" in kwargs and kwargs["hideTrash"]: + hide_trash_query = ~Q("query_string", query="\\/.Trash\\/", fields=["path"]) search = search.filter(hide_trash_query) - search = search.filter('prefix', **{'path._exact': path.strip('/')}) - search = search.filter('term', **{'system._exact': system}) + search = search.filter("prefix", **{"path._exact": path.strip("/")}) + search = search.filter("term", **{"system._exact": system}) search = search.extra(from_=int(offset), size=int(limit)) res = search.execute() hits = [hit.to_dict() for hit in res] - return {'listing': hits, 'count': res.hits.total.value, - 'reachedEnd': len(hits) < int(limit)} + return {"listing": hits, "count": res.hits.total.value, "reachedEnd": len(hits) < int(limit)} def download(client, system, path, max_uses=3, lifetime=600, **kwargs): @@ -235,9 +247,11 @@ def download(client, system, path, max_uses=3, lifetime=600, **kwargs): Post it link. """ - create_postit_result = client.files.createPostIt(systemId=system, path=path, allowedUses=max_uses, validSeconds=lifetime) + create_postit_result = client.files.createPostIt( + systemId=system, path=path, allowedUses=max_uses, validSeconds=lifetime + ) - redeemUrl = f'{create_postit_result.redeemUrl}?download=true' + redeemUrl = f"{create_postit_result.redeemUrl}?download=true" return redeemUrl @@ -266,11 +280,14 @@ def mkdir(client, system, path, dir_name, metadata=None, **kwargs): client.files.mkdir(systemId=system, path=path_input) - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': system, - 'filePath': path, - 'recurse': False}, - ) + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": system, + "filePath": path, + "recurse": False, + }, + ) return {"result": "OK"} @@ -303,64 +320,74 @@ def move(client, src_system, src_path, dest_system, dest_path, file_name=None, m raise ApiException("Cross-system file moves are not supported") if file_name is None: - file_name = src_path.strip('/').split('/')[-1] + file_name = src_path.strip("/").split("/")[-1] - dest_path_full = os.path.join(dest_path.strip('/'), file_name) + dest_path_full = os.path.join(dest_path.strip("/"), file_name) # Handle attempt to move a file into its current path. if src_system == dest_system and src_path == dest_path_full: - return {'system': src_system, 'path': src_path, 'name': file_name} + return {"system": src_system, "path": src_path, "name": file_name} # list the directory and check if file_name exists file_listing = client.files.listFiles(systemId=dest_system, path=dest_path) file_name = increment_file_name(listing=file_listing, file_name=file_name) - dest_path_full = os.path.join(dest_path.strip('/'), file_name) + dest_path_full = os.path.join(dest_path.strip("/"), file_name) if metadata is not None: - if (metadata.get('data_type') == 'file'): - patch_file_association(src_system, metadata, src_path.strip('/'), dest_path_full, file_name, 'move') + if metadata.get("data_type") == "file": + patch_file_association(src_system, metadata, src_path.strip("/"), dest_path_full, file_name, "move") else: - patch_entity_and_node(src_system, metadata, src_path.strip('/'), dest_path, file_name) + patch_entity_and_node(src_system, metadata, src_path.strip("/"), dest_path, file_name) if src_system == dest_system: - move_result = client.files.moveCopy(systemId=src_system, - path=src_path, - operation="MOVE", - newPath=dest_path_full, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + move_result = client.files.moveCopy( + systemId=src_system, + path=src_path, + operation="MOVE", + newPath=dest_path_full, + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) if os.path.dirname(src_path) != dest_path or src_path != dest_path: - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': src_system, - 'filePath': os.path.dirname(src_path), - 'recurse': False}, - routing_key='indexing' - ) - - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': dest_system, - 'filePath': os.path.dirname(dest_path_full), - 'recurse': False}, - routing_key='indexing' - ) + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": src_system, + "filePath": os.path.dirname(src_path), + "recurse": False, + }, + routing_key="indexing", + ) + + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": dest_system, + "filePath": os.path.dirname(dest_path_full), + "recurse": False, + }, + routing_key="indexing", + ) # get information about file to check if it is a dir or not file_info = client.files.getStatInfo(systemId=dest_system, path=dest_path_full) - if (file_info.dir): - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': dest_system, - 'filePath': dest_path_full, - 'recurse': True}, - routing_key='indexing' - ) + if file_info.dir: + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": dest_system, + "filePath": dest_path_full, + "recurse": True, + }, + routing_key="indexing", + ) return move_result @transaction.atomic -def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, metadata=None, - *args, **kwargs): +def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, metadata=None, *args, **kwargs): """Copies the current file to the provided destination path. Params @@ -383,67 +410,69 @@ def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, m dict """ if file_name is None: - file_name = src_path.strip('/').split('/')[-1] + file_name = src_path.strip("/").split("/")[-1] # list the directory and check if file_name exists file_listing = client.files.listFiles(systemId=dest_system, path=dest_path) file_name = increment_file_name(listing=file_listing, file_name=file_name) - dest_path_full = os.path.join(dest_path.strip('/'), file_name) + dest_path_full = os.path.join(dest_path.strip("/"), file_name) if metadata is not None: - if (metadata.get('data_type') == 'file'): - patch_file_association(src_system, metadata, src_path, dest_path_full, file_name, 'copy') + if metadata.get("data_type") == "file": + patch_file_association(src_system, metadata, src_path, dest_path_full, file_name, "copy") if src_system == dest_system: - copy_result = client.files.moveCopy(systemId=src_system, - path=src_path, - operation="COPY", - newPath=dest_path_full, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + copy_result = client.files.moveCopy( + systemId=src_system, + path=src_path, + operation="COPY", + newPath=dest_path_full, + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) else: + src_url = f"tapis://{src_system}/{src_path}" + dest_url = f"tapis://{dest_system}/{dest_path_full}" - src_url = f'tapis://{src_system}/{src_path}' - dest_url = f'tapis://{dest_system}/{dest_path_full}' - - copy_response = client.files.createTransferTask(elements=[{ - 'sourceURI': src_url, - 'destinationURI': dest_url - }], headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + copy_response = client.files.createTransferTask( + elements=[{"sourceURI": src_url, "destinationURI": dest_url}], + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) copy_result = { - 'uuid': copy_response.uuid, - 'status': copy_response.status, + "uuid": copy_response.uuid, + "status": copy_response.status, } - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': dest_system, - 'filePath': os.path.dirname(dest_path_full), - 'recurse': False}, - routing_key='indexing' - ) - - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': dest_system, - 'filePath': dest_path_full, - 'recurse': True}, - routing_key='indexing' - ) + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": dest_system, + "filePath": os.path.dirname(dest_path_full), + "recurse": False, + }, + routing_key="indexing", + ) + + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": dest_system, + "filePath": dest_path_full, + "recurse": True, + }, + routing_key="indexing", + ) return copy_result -def makepublic(client, src_system, src_path, dest_path='/', *args, **kwargs): - dest_system = next((sys['system'] - for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS - if sys['scheme'] == 'public')) +def makepublic(client, src_system, src_path, dest_path="/", *args, **kwargs): + dest_system = next( + (sys["system"] for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["scheme"] == "public") + ) - return copy(client, - src_system, - src_path, - dest_system, - dest_path, - *args, **kwargs) + return copy(client, src_system, src_path, dest_system, dest_path, *args, **kwargs) @transaction.atomic @@ -453,9 +482,9 @@ def delete(client, system, path, *args, **kwargs): if settings.PORTAL_PROJECTS_ENABLE_METADATA and get_file_obj(system, path): remove_file_obj_by_path(system, path) - return client.files.delete(systemId=system, - path=path, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + return client.files.delete( + systemId=system, path=path, headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")} + ) def rename(client, system, path, new_name, metadata=None, *args, **kwargs): @@ -478,8 +507,16 @@ def rename(client, system, path, new_name, metadata=None, *args, **kwargs): dict """ new_path = os.path.dirname(path) - return move(client, src_system=system, src_path=path, - dest_system=system, dest_path=new_path, file_name=new_name, metadata=metadata, **kwargs) + return move( + client, + src_system=system, + src_path=path, + dest_system=system, + dest_path=new_path, + file_name=new_name, + metadata=metadata, + **kwargs, + ) def trash(client, system, path, homeDir, metadata=None, *args, **kwargs): @@ -499,23 +536,23 @@ def trash(client, system, path, homeDir, metadata=None, *args, **kwargs): dict """ - file_name = path.strip('/').split('/')[-1] + file_name = path.strip("/").split("/")[-1] # Create a .Trash path if none exists try: - client.files.listFiles(systemId=system, - path=f'{homeDir}/{settings.TAPIS_DEFAULT_TRASH_NAME}') + client.files.listFiles(systemId=system, path=f"{homeDir}/{settings.TAPIS_DEFAULT_TRASH_NAME}") except BaseTapyException as err: if err.response.status_code != 404: - logger.error(f'Unexpected exception listing .trash path in {system}') + logger.error(f"Unexpected exception listing .trash path in {system}") raise mkdir(client, system, homeDir, settings.TAPIS_DEFAULT_TRASH_NAME) if metadata is not None: get_or_create_trash_entity(system) - resp = move(client, system, path, system, - f'{homeDir}/{settings.TAPIS_DEFAULT_TRASH_NAME}', file_name, metadata, **kwargs) + resp = move( + client, system, path, system, f"{homeDir}/{settings.TAPIS_DEFAULT_TRASH_NAME}", file_name, metadata, **kwargs + ) return resp @@ -541,41 +578,42 @@ def upload(client, system, path, uploaded_file, metadata=None, *args, **kwargs): file_listing = client.files.listFiles(systemId=system, path=path) uploaded_file.name = increment_file_name(listing=file_listing, file_name=uploaded_file.name) - dest_path = os.path.join(path.strip('/'), uploaded_file.name) - - if settings.PORTAL_PROJECTS_ENABLE_METADATA and metadata is not None and getattr(constants, metadata.get('data_type').upper(), None): + dest_path = os.path.join(path.strip("/"), uploaded_file.name) + if ( + settings.PORTAL_PROJECTS_ENABLE_METADATA + and metadata is not None + and getattr(constants, metadata.get("data_type").upper(), None) + ): parent_node = get_node_from_path(system, path) file_obj = create_file_obj(system, uploaded_file.name, uploaded_file.size, dest_path, metadata) - if parent_node and parent_node['id'] != 'NODE_ROOT': - add_file_associations(parent_node['uuid'], [file_obj]) + if parent_node and parent_node["id"] != "NODE_ROOT": + add_file_associations(parent_node["uuid"], [file_obj]) else: # Add file association to root node if no parent node/entity exists root_node = get_root_node(system) - add_file_associations(root_node['uuid'], [file_obj]) + add_file_associations(root_node["uuid"], [file_obj]) upload_url = f"{settings.TAPIS_TENANT_BASEURL}/v3/files/ops/{system}/{dest_path.lstrip('/')}" - headers = {'x-tapis-token': client.get_access_jwt(), - "X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "") - } - - res = httpx.post(upload_url, - headers=headers, - files={'file': uploaded_file}, - timeout=600) + headers = {"x-tapis-token": client.get_access_jwt(), "X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")} + + res = httpx.post(upload_url, headers=headers, files={"file": uploaded_file}, timeout=600) res.raise_for_status() # NOTE: tapipy causing issues currently by reading file into memory, so not using it for uploads at the moment # response_json = client.files.insert(systemId=system, # path=dest_path, # file=uploaded_file, # headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) - tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, - 'systemId': system, - 'filePath': path, - 'recurse': False}, - ) + tapis_indexer.apply_async( + kwargs={ + "access_token": client.access_token.access_token, + "systemId": system, + "filePath": path, + "recurse": False, + }, + ) return res.json() @@ -600,40 +638,42 @@ def preview(client, system, path, max_uses=3, lifetime=600, **kwargs): dict """ - file_name = path.strip('/').split('/')[-1] + file_name = path.strip("/").split("/")[-1] file_ext = os.path.splitext(file_name)[1].lower() - postit = client.files.createPostIt(systemId=system, - path=path, allowedUses=max_uses, - validSeconds=lifetime, - headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}) + postit = client.files.createPostIt( + systemId=system, + path=path, + allowedUses=max_uses, + validSeconds=lifetime, + headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")}, + ) url = postit.redeemUrl txt = None error = None file_type = None if file_ext in settings.SUPPORTED_TEXT_PREVIEW_EXTS: - file_type = 'text' + file_type = "text" elif file_ext in settings.SUPPORTED_IMAGE_PREVIEW_EXTS: - file_type = 'image' + file_type = "image" elif any([ext for ext in settings.SUPPORTED_BRAINMAP_PREVIEW_EXTS if file_name.endswith(ext)]): - file_type = 'brainmap' + file_type = "brainmap" elif file_ext in settings.SUPPORTED_OBJECT_PREVIEW_EXTS: - file_type = 'object' + file_type = "object" elif file_ext in settings.SUPPORTED_MS_OFFICE: - file_type = 'ms-office' - url = 'https://view.officeapps.live.com/op/view.aspx?src={}'.\ - format(url) + file_type = "ms-office" + url = "https://view.officeapps.live.com/op/view.aspx?src={}".format(url) elif file_ext in settings.SUPPORTED_IPYNB_PREVIEW_EXTS: - file_type = 'ipynb' - tmp = url.replace('https://', '') - url = 'https://nbviewer.jupyter.org/urls/{tmp}'.format(tmp=tmp) + file_type = "ipynb" + tmp = url.replace("https://", "") + url = "https://nbviewer.jupyter.org/urls/{tmp}".format(tmp=tmp) elif file_ext in settings.SUPPORTED_NEW_WINDOW_PREVIEW_EXTS: error = "This file type must be previewed in a new window." else: - file_type = 'other' + file_type = "other" - if file_type in ['other', 'text']: + if file_type in ["other", "text"]: if get_file_size(client, system, path) < 5000000: try: txt = text_preview(url) @@ -642,7 +682,7 @@ def preview(client, system, path, max_uses=3, lifetime=600, **kwargs): error = "Unable to show preview." else: error = "File too large to preview in this window." - return {'href': url, 'fileType': file_type, 'content': txt, 'error': error} + return {"href": url, "fileType": file_type, "content": txt, "error": error} def download_bytes(client, system, path, *args, **kwargs): @@ -669,17 +709,20 @@ def download_bytes(client, system, path, *args, **kwargs): @transaction.atomic def upload_file_metadata(client, system, path, file_name, file_size, metadata, **kwargs): - dest_path = os.path.join(path.strip('/'), file_name) - - if settings.PORTAL_PROJECTS_ENABLE_METADATA and metadata is not None and getattr(constants, metadata.get('data_type').upper(), None): + dest_path = os.path.join(path.strip("/"), file_name) + if ( + settings.PORTAL_PROJECTS_ENABLE_METADATA + and metadata is not None + and getattr(constants, metadata.get("data_type").upper(), None) + ): parent_node = get_node_from_path(system, path) file_obj = create_file_obj(system, file_name, file_size, dest_path, metadata) - if parent_node and parent_node['id'] != 'NODE_ROOT': - add_file_associations(parent_node['uuid'], [file_obj]) + if parent_node and parent_node["id"] != "NODE_ROOT": + add_file_associations(parent_node["uuid"], [file_obj]) else: # Add file association to root node if no parent node/entity exists root_node = get_root_node(system) - add_file_associations(root_node['uuid'], [file_obj]) + add_file_associations(root_node["uuid"], [file_obj]) diff --git a/server/portal/libs/agave/operations_unit_test.py b/server/portal/libs/agave/operations_unit_test.py index 1becc3ea24..6e48dd3767 100644 --- a/server/portal/libs/agave/operations_unit_test.py +++ b/server/portal/libs/agave/operations_unit_test.py @@ -8,151 +8,154 @@ class TestOperations(TestCase): - - @patch('portal.libs.agave.operations.tapis_listing_indexer') + @patch("portal.libs.agave.operations.tapis_listing_indexer") def test_listing(self, mock_indexer): client = MagicMock() - mock_tapis_listing = [TapisResult(**{ - "mimeType": None, - "type": "file", - "url": "tapis://cloud.data/path/to/file", - "lastModified": "2020-04-23T06:25:56Z", - "name": "file", - "path": '/path/to/file', - "size": 1 - })] + mock_tapis_listing = [ + TapisResult( + **{ + "mimeType": None, + "type": "file", + "url": "tapis://cloud.data/path/to/file", + "lastModified": "2020-04-23T06:25:56Z", + "name": "file", + "path": "/path/to/file", + "size": 1, + } + ) + ] client.files.listFiles.return_value = mock_tapis_listing - ls = listing(client, 'test.system', '/path/to/file', 1) - - client.files.listFiles.assert_called_with(systemId='test.system', - path='/path/to/file', - pattern='', - offset=1, - limit=100, - headers={'X-Tapis-Tracking-ID': ''}) - - mock_response_listing = [{'uuid': None, - 'system': 'test.system', - 'type': 'file', - 'format': 'raw', - 'mimeType': None, - 'path': '/path/to/file', - 'name': 'file', - 'length': 1, - 'lastModified': '2020-04-23T06:25:56Z', - '_links': { - 'self': { - 'href': 'tapis://cloud.data/path/to/file' - } - }, - 'metadata': None - }] + ls = listing(client, "test.system", "/path/to/file", 1) + + client.files.listFiles.assert_called_with( + systemId="test.system", + path="/path/to/file", + pattern="", + offset=1, + limit=100, + headers={"X-Tapis-Tracking-ID": ""}, + ) + + mock_response_listing = [ + { + "uuid": None, + "system": "test.system", + "type": "file", + "format": "raw", + "mimeType": None, + "path": "/path/to/file", + "name": "file", + "length": 1, + "lastModified": "2020-04-23T06:25:56Z", + "_links": {"self": {"href": "tapis://cloud.data/path/to/file"}}, + "metadata": None, + } + ] mock_indexer.delay.assert_called_with(mock_response_listing) - self.assertEqual(ls, {'listing': mock_response_listing, - 'reachedEnd': True, - 'folder_metadata': None}) + self.assertEqual(ls, {"listing": mock_response_listing, "reachedEnd": True, "folder_metadata": None}) - @patch('portal.libs.agave.operations.listing') - @patch('portal.libs.agave.operations.IndexedFile.search') + @patch("portal.libs.agave.operations.listing") + @patch("portal.libs.agave.operations.IndexedFile.search") def test_search(self, mock_search, mock_listing): mock_hit = Hit({}) - mock_hit.system = 'test.system' - mock_hit.path = '/path/to/file' + mock_hit.system = "test.system" + mock_hit.path = "/path/to/file" mock_result = MagicMock() mock_result.__iter__.return_value = [mock_hit] mock_result.hits.total.value = 1 - mock_search().query().filter().filter().filter().extra().execute\ - .return_value = mock_result - - search_res = search(None, 'test.system', '/path', query_string='query', hideTrash='True') - - mock_search().query.assert_called_with(Q("query_string", query='query', - fields=["name"], - minimum_should_match='100%', - default_operator='or') | - Q("query_string", query='query', - fields=[ - "name._exact, name._pattern"], - default_operator='and')) - - mock_search().query().filter.assert_called_with(~Q("query_string", query='\\/.Trash\\/', fields=["path"])) - mock_search().query().filter().filter.assert_called_with('prefix', **{'path._exact': 'path'}) - mock_search().query().filter().filter().filter.assert_called_with('term', **{'system._exact': 'test.system'}) + mock_search().query().filter().filter().filter().extra().execute.return_value = mock_result + + search_res = search(None, "test.system", "/path", query_string="query", hideTrash="True") + + mock_search().query.assert_called_with( + Q("query_string", query="query", fields=["name"], minimum_should_match="100%", default_operator="or") + | Q("query_string", query="query", fields=["name._exact, name._pattern"], default_operator="and") + ) + + mock_search().query().filter.assert_called_with(~Q("query_string", query="\\/.Trash\\/", fields=["path"])) + mock_search().query().filter().filter.assert_called_with("prefix", **{"path._exact": "path"}) + mock_search().query().filter().filter().filter.assert_called_with("term", **{"system._exact": "test.system"}) mock_search().query().filter().filter().filter().extra.assert_called_with(from_=int(0), size=int(100)) - self.assertEqual(search_res, {'listing': - [{'system': 'test.system', - 'path': '/path/to/file'}], - 'reachedEnd': True, 'count': 1}) + self.assertEqual( + search_res, + {"listing": [{"system": "test.system", "path": "/path/to/file"}], "reachedEnd": True, "count": 1}, + ) - @patch('portal.libs.agave.operations.tapis_indexer') + @patch("portal.libs.agave.operations.tapis_indexer") def test_mkdir(self, mock_indexer): client = MagicMock() - client.access_token.access_token = 'my_access_token' + client.access_token.access_token = "my_access_token" - mkdir(client, 'test.system', '/root', 'testfolder') + mkdir(client, "test.system", "/root", "testfolder") - client.files.mkdir.assert_called_with(systemId='test.system', path='/root/testfolder') + client.files.mkdir.assert_called_with(systemId="test.system", path="/root/testfolder") - mock_indexer.apply_async.assert_called_with(kwargs={'access_token': 'my_access_token', 'systemId': 'test.system', - 'filePath': '/root', 'recurse': False}) + mock_indexer.apply_async.assert_called_with( + kwargs={"access_token": "my_access_token", "systemId": "test.system", "filePath": "/root", "recurse": False} + ) - @patch('portal.libs.agave.operations.move') + @patch("portal.libs.agave.operations.move") def test_rename(self, mock_move): client = MagicMock() - rename(client, 'test.system', '/path/to/file', 'newname') - mock_move.assert_called_with(client, - src_system='test.system', - src_path='/path/to/file', - dest_system='test.system', dest_path='/path/to', - file_name='newname', metadata=None) - - @patch('portal.libs.agave.operations.tapis_indexer') + rename(client, "test.system", "/path/to/file", "newname") + mock_move.assert_called_with( + client, + src_system="test.system", + src_path="/path/to/file", + dest_system="test.system", + dest_path="/path/to", + file_name="newname", + metadata=None, + ) + + @patch("portal.libs.agave.operations.tapis_indexer") def test_move(self, mock_indexer): client = MagicMock() - client.files.moveCopy.return_value = {'status': 'success'} - client.files.getStatInfo.return_value = TapisResult(**{'dir': True}) + client.files.moveCopy.return_value = {"status": "success"} + client.files.getStatInfo.return_value = TapisResult(**{"dir": True}) - move(client, 'test.system', '/path/to/src', 'test.system', '/path/to/dest') + move(client, "test.system", "/path/to/src", "test.system", "/path/to/dest") - client.files.moveCopy.assert_called_with(systemId='test.system', - path='/path/to/src', - operation='MOVE', - newPath='path/to/dest/src', - headers={'X-Tapis-Tracking-ID': ''}) + client.files.moveCopy.assert_called_with( + systemId="test.system", + path="/path/to/src", + operation="MOVE", + newPath="path/to/dest/src", + headers={"X-Tapis-Tracking-ID": ""}, + ) self.assertEqual(mock_indexer.apply_async.call_count, 3) def test_cross_system_move(self): client = MagicMock() with self.assertRaises(ApiException): - move(client, 'test.system', '/path/to/src', 'other.system', '/path/to/dest') + move(client, "test.system", "/path/to/src", "other.system", "/path/to/dest") - @patch('portal.libs.agave.operations.tapis_indexer') + @patch("portal.libs.agave.operations.tapis_indexer") def test_copy(self, mock_indexer): client = MagicMock() - client.files.moveCopy.return_value = {'status': 'success'} + client.files.moveCopy.return_value = {"status": "success"} - copy(client, 'test.system', '/path/to/src', 'test.system', '/path/to/dest') + copy(client, "test.system", "/path/to/src", "test.system", "/path/to/dest") - client.files.moveCopy.assert_called_with(systemId='test.system', - path='/path/to/src', - operation='COPY', - newPath='path/to/dest/src', - headers={'X-Tapis-Tracking-ID': ''}) + client.files.moveCopy.assert_called_with( + systemId="test.system", + path="/path/to/src", + operation="COPY", + newPath="path/to/dest/src", + headers={"X-Tapis-Tracking-ID": ""}, + ) self.assertEqual(mock_indexer.apply_async.call_count, 2) - @patch('portal.libs.agave.operations.copy') + @patch("portal.libs.agave.operations.copy") def test_make_public(self, mock_copy): client = MagicMock() - makepublic(client, 'test.system', '/path/to/src') + makepublic(client, "test.system", "/path/to/src") - mock_copy.assert_called_with(client, - 'test.system', - '/path/to/src', - 'cloud.data', '/') + mock_copy.assert_called_with(client, "test.system", "/path/to/src", "cloud.data", "/") diff --git a/server/portal/libs/agave/serializers.py b/server/portal/libs/agave/serializers.py index bed28957c8..ec8a2c66e4 100644 --- a/server/portal/libs/agave/serializers.py +++ b/server/portal/libs/agave/serializers.py @@ -3,6 +3,7 @@ :synopsis: Necessary classes to serialize a class which wrapps an agave object into a dict. """ + import logging import json from tapipy.tapis import TapisResult diff --git a/server/portal/libs/agave/unit_test.py b/server/portal/libs/agave/unit_test.py index 99015cc2f4..98b36f88aa 100644 --- a/server/portal/libs/agave/unit_test.py +++ b/server/portal/libs/agave/unit_test.py @@ -2,6 +2,7 @@ .. :module:: portal.libs.agave.unit_test :synopsis: Unit tests for Agave libraries. """ + import logging import os import json @@ -23,10 +24,7 @@ class TestAgaveUtils(TestCase): @classmethod def setUpClass(cls): super(TestAgaveUtils, cls).setUpClass() - cls.magave_patcher = patch( - 'portal.apps.auth.models.TapisOAuthToken.client', - autospec=True - ) + cls.magave_patcher = patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) cls.magave = cls.magave_patcher.start() @classmethod @@ -34,57 +32,32 @@ def tearDownClass(cls): cls.magave_patcher.stop() def setUp(self): - agave_path = os.path.join(settings.BASE_DIR, 'fixtures/agave') - - with open( - os.path.join( - agave_path, - 'files', - 'file.json' - ) - ) as _file: + agave_path = os.path.join(settings.BASE_DIR, "fixtures/agave") + + with open(os.path.join(agave_path, "files", "file.json")) as _file: self.agave_file = json.load(_file) - with open( - os.path.join( - agave_path, - 'files', - 'directory.json' - ) - ) as _file: + with open(os.path.join(agave_path, "files", "directory.json")) as _file: self.agave_directory = json.load(_file) - with open( - os.path.join( - agave_path, - 'files', - 'listing.json' - ) - ) as _file: + with open(os.path.join(agave_path, "files", "listing.json")) as _file: self.agave_listing = json.load(_file) - with open( - os.path.join( - agave_path, - 'files', - 'file-listing.json' - ) - ) as _file: + with open(os.path.join(agave_path, "files", "file-listing.json")) as _file: self.agave_file_listing = [TapisResult(**f) for f in json.load(_file)] def test_to_camel_case(self): """Test `to_camel_case` util.""" - attr = 'some_attribute' + attr = "some_attribute" res = AgaveUtils.to_camel_case(attr) - self.assertEqual(res, 'someAttribute') + self.assertEqual(res, "someAttribute") def test_walk_levels(self): """Test `walk_levels` util.""" self.magave.reset_mock() - agave_dir = [obj for obj in self.agave_listing - if obj['type'] == 'dir' and obj['name'] != '.'][0] + agave_dir = [obj for obj in self.agave_listing if obj["type"] == "dir" and obj["name"] != "."][0] sub_root = copy.deepcopy(agave_dir) - sub_root['name'] = '.' + sub_root["name"] = "." listings = [ [TapisResult(**f) for f in self.agave_listing], [TapisResult(**sub_root), TapisResult(**self.agave_file)], @@ -98,55 +71,34 @@ def test_walk_levels(self): levels_visited = [] for root, folders, files in AgaveUtils.walk_levels( - self.magave, - self.agave_listing[0]['system'], - self.agave_listing[0]['path'] + self.magave, self.agave_listing[0]["system"], self.agave_listing[0]["path"] ): levels_visited.append((root, folders, files)) self.assertEqual( self.magave.files.listFiles.call_args_list, - [call( - systemId=self.agave_listing[0]['system'], - path=self.agave_listing[0]['path'], - offset=0, - limit=100), - call( - systemId=agave_dir['system'], - path=agave_dir['path'], - offset=0, - limit=100) - ] + [ + call(systemId=self.agave_listing[0]["system"], path=self.agave_listing[0]["path"], offset=0, limit=100), + call(systemId=agave_dir["system"], path=agave_dir["path"], offset=0, limit=100), + ], ) for index, level in enumerate(levels_visited): listing = listings_check[index] - root = listing[0]['path'] - folders = [f['path'] for f in listing - if f['format'] == 'folder' and - f['name'] != '.'] - files = [f['path'] for f in listing - if f['format'] != 'folder'] - self.assertEqual( - root, - level[0] - ) - self.assertEqual( - folders, - [f['path'] for f in level[1]] - ) - self.assertEqual( - files, - [f['path'] for f in level[2]] - ) + root = listing[0]["path"] + folders = [f["path"] for f in listing if f["format"] == "folder" and f["name"] != "."] + files = [f["path"] for f in listing if f["format"] != "folder"] + self.assertEqual(root, level[0]) + self.assertEqual(folders, [f["path"] for f in level[1]]) + self.assertEqual(files, [f["path"] for f in level[2]]) def test_increment_file_name(self): """Test `increment_file_name` util.""" self.magave.reset_mock() - file_name = 'some_file_name.txt' + file_name = "some_file_name.txt" res = AgaveUtils.increment_file_name(self.agave_file_listing, file_name) - self.assertEqual(res, 'some_file_name.txt') + self.assertEqual(res, "some_file_name.txt") - file_name = 'file.txt' + file_name = "file.txt" res = AgaveUtils.increment_file_name(self.agave_file_listing, file_name) - self.assertEqual(res, 'file(1).txt') + self.assertEqual(res, "file(1).txt") diff --git a/server/portal/libs/agave/utils.py b/server/portal/libs/agave/utils.py index 3075b0fec9..ba77bc81bf 100644 --- a/server/portal/libs/agave/utils.py +++ b/server/portal/libs/agave/utils.py @@ -2,6 +2,7 @@ .. module:: portal.libs.agave.utils """ + import logging import os from django.conf import settings @@ -22,48 +23,44 @@ def to_camel_case(input_str): :return: lowerCamelCase string :rtype: str """ - left_cnt = len(input_str) - len(input_str.lstrip('_')) - right_cnt = len(input_str) - len(input_str.rstrip('_')) - comps = input_str[left_cnt:].split('_') - right_side = ''.join(w.title() for w in comps[1:]) - camel_case = ''.join( - ['_' * left_cnt, - comps[0], - right_side, - '_' * right_cnt] - ) + left_cnt = len(input_str) - len(input_str.lstrip("_")) + right_cnt = len(input_str) - len(input_str.rstrip("_")) + comps = input_str[left_cnt:].split("_") + right_side = "".join(w.title() for w in comps[1:]) + camel_case = "".join(["_" * left_cnt, comps[0], right_side, "_" * right_cnt]) return camel_case def iterate_level(client, system, path, limit=100): """Iterate over a filesystem level yielding an attrdict for each file/folder - on the level. - :param str client: an Agave client - :param str system: system - :param str path: path to walk - :param int limit: Number of docs to retrieve per API call + on the level. + :param str client: an Agave client + :param str system: system + :param str path: path to walk + :param int limit: Number of docs to retrieve per API call - :rtype agavepy.agave.AttrDict + :rtype agavepy.agave.AttrDict """ offset = 0 while True: - _page = client.files.listFiles(systemId=system, - path=path, - offset=int(offset), - limit=int(limit)) - page = list(map(lambda f: { - 'system': system, - 'type': 'dir' if f.type == 'dir' else 'file', - 'format': 'folder' if f.type == 'dir' else 'raw', - 'mimeType': f.mimeType, - 'path': f.path, - 'name': f.name, - 'length': f.size, - 'lastModified': f.lastModified, - '_links': { - 'self': {'href': f.url} - }}, _page)) + _page = client.files.listFiles(systemId=system, path=path, offset=int(offset), limit=int(limit)) + page = list( + map( + lambda f: { + "system": system, + "type": "dir" if f.type == "dir" else "file", + "format": "folder" if f.type == "dir" else "raw", + "mimeType": f.mimeType, + "path": f.path, + "name": f.name, + "length": f.size, + "lastModified": f.lastModified, + "_links": {"self": {"href": f.url}}, + }, + _page, + ) + ) yield from page offset += limit if len(page) != limit: @@ -108,27 +105,18 @@ def walk_levels(client, system, path, bottom_up=False, ignore_hidden=False): folders = [] files = [] for agave_file in iterate_level(client, system, path): - if agave_file['name'] == '.': + if agave_file["name"] == ".": continue - if ignore_hidden and agave_file['name'][0] == '.': + if ignore_hidden and agave_file["name"][0] == ".": continue - if agave_file['format'] == 'folder': + if agave_file["format"] == "folder": folders.append(agave_file) else: files.append(agave_file) if not bottom_up: yield (path, folders, files) for child in folders: - for ( - child_path, - child_folders, - child_files - ) in walk_levels( - client, - system, - child['path'], - bottom_up=bottom_up - ): + for child_path, child_folders, child_files in walk_levels(client, system, child["path"], bottom_up=bottom_up): yield (child_path, child_folders, child_files) if bottom_up: @@ -137,17 +125,17 @@ def walk_levels(client, system, path, bottom_up=False, ignore_hidden=False): def service_account(): """Return a Tapis instance with the admin account.""" - return Tapis( - base_url=settings.TAPIS_TENANT_BASEURL, - access_token=settings.TAPIS_ADMIN_JWT) + return Tapis(base_url=settings.TAPIS_TENANT_BASEURL, access_token=settings.TAPIS_ADMIN_JWT) def user_account(access_token): """Return a Tapis instance with the user credentials""" - return Tapis(base_url=getattr(settings, 'TAPIS_TENANT_BASEURL'), - client_id=getattr(settings, 'TAPIS_CLIENT_ID'), - client_key=getattr(settings, 'TAPIS_CLIENT_KEY'), - access_token=access_token) + return Tapis( + base_url=getattr(settings, "TAPIS_TENANT_BASEURL"), + client_id=getattr(settings, "TAPIS_CLIENT_ID"), + client_key=getattr(settings, "TAPIS_CLIENT_KEY"), + access_token=access_token, + ) def text_preview(url): @@ -163,10 +151,10 @@ def text_preview(url): """ try: resp = requests.get(url) - if (resp.content or (resp.encoding is not None and resp.encoding.lower() == 'utf-8')): + if resp.content or (resp.encoding is not None and resp.encoding.lower() == "utf-8"): content = resp.text # Raises UnicodeDecodeError for files with non-ascii characters - content.encode('ascii', 'strict') + content.encode("ascii", "strict") return content else: raise ValueError("File does not contain text") @@ -181,22 +169,21 @@ def increment_file_name(listing, file_name): _ext = os.path.splitext(file_name)[1] _name = os.path.splitext(file_name)[0] _inc = "({})".format(inc) - file_name = '{}{}{}'.format(_name, _inc, _ext) + file_name = "{}{}{}".format(_name, _inc, _ext) while any(x.name for x in listing if x.name == file_name): inc += 1 _inc = "({})".format(inc) - file_name = '{}{}{}'.format(_name, _inc, _ext) + file_name = "{}{}{}".format(_name, _inc, _ext) return file_name def get_file_size(client, system, path): - """ Get file size + """Get file size :param client: an Agave client :param system: system of file :param path: path of file :return: file size in bytes """ - file_response = client.files.listFiles(systemId=system, - path=path) + file_response = client.files.listFiles(systemId=system, path=path) return int(file_response[0].size) diff --git a/server/portal/libs/elasticsearch/analyzers.py b/server/portal/libs/elasticsearch/analyzers.py index 6f962e7b46..75e99857f5 100644 --- a/server/portal/libs/elasticsearch/analyzers.py +++ b/server/portal/libs/elasticsearch/analyzers.py @@ -2,6 +2,7 @@ .. module: portal.lis.elasticsearch.analyzers :synopsis: Elastic Search Analyzers """ + import logging from elasticsearch_dsl import analyzer, token_filter, tokenizer @@ -9,21 +10,26 @@ logger = logging.getLogger(__name__) -path_analyzer = analyzer('path_analyzer', - tokenizer=tokenizer('path_hierarchy')) +path_analyzer = analyzer("path_analyzer", tokenizer=tokenizer("path_hierarchy")) -file_analyzer = analyzer('file_analyzer', - tokenizer=tokenizer('trigram', 'ngram', min_gram=3, max_gram=3, - token_chars=["letter", "digit", "punctuation", "symbol", "whitespace"]), - filter='lowercase') +file_analyzer = analyzer( + "file_analyzer", + tokenizer=tokenizer( + "trigram", + "ngram", + min_gram=3, + max_gram=3, + token_chars=["letter", "digit", "punctuation", "symbol", "whitespace"], + ), + filter="lowercase", +) -file_query_analyzer = analyzer('file_query_analyzer', - tokenizer='whitespace', filter=['lowercase', token_filter('trunc20', 'truncate', length=20)]) +file_query_analyzer = analyzer( + "file_query_analyzer", tokenizer="whitespace", filter=["lowercase", token_filter("trunc20", "truncate", length=20)] +) -file_pattern_analyzer = analyzer('file_ext_analyzer', - tokenizer=tokenizer('file_pattern', 'pattern', pattern='\\.'), - filter='lowercase') +file_pattern_analyzer = analyzer( + "file_ext_analyzer", tokenizer=tokenizer("file_pattern", "pattern", pattern="\\."), filter="lowercase" +) -reverse_file_analyzer = analyzer('file_reverse', - tokenizer=tokenizer('keyword'), - filter=['lowercase', 'reverse']) +reverse_file_analyzer = analyzer("file_reverse", tokenizer=tokenizer("keyword"), filter=["lowercase", "reverse"]) diff --git a/server/portal/libs/elasticsearch/docs/base.py b/server/portal/libs/elasticsearch/docs/base.py index 94df9ef846..2020e4228f 100644 --- a/server/portal/libs/elasticsearch/docs/base.py +++ b/server/portal/libs/elasticsearch/docs/base.py @@ -2,13 +2,18 @@ .. module: portal.libs.elasticsearch.docs.base :synopsis: Wrapper classes for ES different doc types. """ + import logging import datetime from django.conf import settings from elasticsearch import Elasticsearch -from elasticsearch_dsl import (Document, Date, Object, Text, Long, Boolean, - Keyword) -from portal.libs.elasticsearch.analyzers import path_analyzer, file_analyzer, file_pattern_analyzer, reverse_file_analyzer +from elasticsearch_dsl import Document, Date, Object, Text, Long, Boolean, Keyword +from portal.libs.elasticsearch.analyzers import ( + path_analyzer, + file_analyzer, + file_pattern_analyzer, + reverse_file_analyzer, +) from portal.libs.elasticsearch.utils import file_uuid_sha256, get_sha256_hash # pylint: disable=invalid-name @@ -17,20 +22,13 @@ class IndexedProject(Document): - id = Keyword(fields={'_exact': Keyword()}) - title = Text(fields={'_exact': Keyword()}) + id = Keyword(fields={"_exact": Keyword()}) + title = Text(fields={"_exact": Keyword()}) description = Text() path = Text() name = Text() host = Text() - owner = Object( - properties={ - 'username': Keyword(), - 'firstName': Text(), - 'lastName': Text(), - 'email': Text() - } - ) + owner = Object(properties={"username": Keyword(), "firstName": Text(), "lastName": Text(), "email": Text()}) updated = Date() @classmethod @@ -38,7 +36,7 @@ def from_id(cls, projectId): return cls.get(projectId) class Index: - name = settings.ES_INDEX_PREFIX.format('projects') + name = settings.ES_INDEX_PREFIX.format("projects") class IndexedFile(Document): @@ -46,35 +44,37 @@ class IndexedFile(Document): Elasticsearch document representing an indexed file. Thin wrapper around `elasticsearch_dsl.Document`. """ - name = Text(analyzer=file_analyzer, fields={ - '_exact': Keyword(), - '_pattern': Text(analyzer=file_pattern_analyzer), - '_reverse': Text(analyzer=reverse_file_analyzer)}) - path = Text(fields={ - '_comps': Text(analyzer=path_analyzer), - '_exact': Keyword(), - '_reverse': Text(analyzer=reverse_file_analyzer)}, + + name = Text( + analyzer=file_analyzer, + fields={ + "_exact": Keyword(), + "_pattern": Text(analyzer=file_pattern_analyzer), + "_reverse": Text(analyzer=reverse_file_analyzer), + }, + ) + path = Text( + fields={ + "_comps": Text(analyzer=path_analyzer), + "_exact": Keyword(), + "_reverse": Text(analyzer=reverse_file_analyzer), + }, ) lastModified = Date() length = Long() format = Text() mimeType = Keyword() type = Text() - system = Text(fields={'_exact': Keyword()}) - basePath = Text( - fields={ - '_comps': Text(analyzer=path_analyzer), - '_exact': Keyword()}) + system = Text(fields={"_exact": Keyword()}) + basePath = Text(fields={"_comps": Text(analyzer=path_analyzer), "_exact": Keyword()}) lastUpdated = Date() - pems = Object(properties={ - 'username': Keyword(), - 'recursive': Boolean(), - 'permission': Object(properties={ - 'read': Boolean(), - 'write': Boolean(), - 'execute': Boolean() - }) - }) + pems = Object( + properties={ + "username": Keyword(), + "recursive": Boolean(), + "permission": Object(properties={"read": Boolean(), "write": Boolean(), "execute": Boolean()}), + } + ) def save(self, *args, **kwargs): """ @@ -121,8 +121,8 @@ def children(self): IndexedFile """ search = self.search() - search = search.filter('term', **{'basePath._exact': self.path}) - search = search.filter('term', **{'system._exact': self.system}) + search = search.filter("term", **{"basePath._exact": self.path}) + search = search.filter("term", **{"system._exact": self.system}) for hit in search.scan(): yield self.get(hit.meta.id) @@ -140,15 +140,16 @@ def delete_recursive(self): self.delete() class Index: - name = settings.ES_INDEX_PREFIX.format('files') + name = settings.ES_INDEX_PREFIX.format("files") class ReindexedFile(IndexedFile): """Identical to IndexedFile, but using a separate index for zero-downtime reindexing applications. """ + class Index: - name = settings.ES_INDEX_PREFIX.format('files-reindex') + name = settings.ES_INDEX_PREFIX.format("files-reindex") class IndexedAllocation(Document): @@ -157,7 +158,7 @@ class IndexedAllocation(Document): `elasticsearch_dsl.Document`. """ - username = Text(fields={'_exact': Keyword()}) + username = Text(fields={"_exact": Keyword()}) value = Object() @classmethod @@ -177,14 +178,12 @@ def from_username(cls, username): ------ elasticsearch.exceptions.NotFoundError """ - es_client = Elasticsearch(hosts=settings.ES_HOSTS, - http_auth=settings.ES_AUTH, - timeout=10) + es_client = Elasticsearch(hosts=settings.ES_HOSTS, http_auth=settings.ES_AUTH, timeout=10) uuid = get_sha256_hash(username) return cls.get(uuid, using=es_client) class Index: - name = settings.ES_INDEX_PREFIX.format('allocations') + name = settings.ES_INDEX_PREFIX.format("allocations") class IndexedPublication(Document): @@ -194,4 +193,4 @@ class IndexedPublication(Document): class Index: """Index meta settings""" - name = settings.ES_INDEX_PREFIX.format('publications') + name = settings.ES_INDEX_PREFIX.format("publications") diff --git a/server/portal/libs/elasticsearch/docs/unit_test.py b/server/portal/libs/elasticsearch/docs/unit_test.py index 3f61ff6aef..faf3d74f13 100644 --- a/server/portal/libs/elasticsearch/docs/unit_test.py +++ b/server/portal/libs/elasticsearch/docs/unit_test.py @@ -4,45 +4,44 @@ class TestIndexedFile(TestCase): - def setUp(self): self.depth = 1 - @patch('portal.libs.elasticsearch.docs.base.Document.save') + @patch("portal.libs.elasticsearch.docs.base.Document.save") def test_save(self, mock_save): doc = IndexedFile() doc.save() mock_save.assert_called_once() - @patch('portal.libs.elasticsearch.docs.base.Document.update') + @patch("portal.libs.elasticsearch.docs.base.Document.update") def test_update(self, mock_update): doc = IndexedFile() doc.update() mock_update.assert_called_once() - @patch('portal.libs.elasticsearch.docs.base.Document.get') + @patch("portal.libs.elasticsearch.docs.base.Document.get") def test_from_path(self, mock_get): - IndexedFile.from_path('test.system', '/path/to/file') - mock_get.assert_called_once_with('c7765edebe9d7b715865b83a8319703975680be5a3f5f77503bdc47e7978429c') + IndexedFile.from_path("test.system", "/path/to/file") + mock_get.assert_called_once_with("c7765edebe9d7b715865b83a8319703975680be5a3f5f77503bdc47e7978429c") - @patch('portal.libs.elasticsearch.docs.base.Document.search') - @patch('portal.libs.elasticsearch.docs.base.Document.get') + @patch("portal.libs.elasticsearch.docs.base.Document.search") + @patch("portal.libs.elasticsearch.docs.base.Document.get") def test_children(self, mock_get, mock_search): res1 = MagicMock() - res1.meta.id = 'id1' + res1.meta.id = "id1" def scan_side_effect(): yield res1 mock_search().filter().filter().scan.side_effect = scan_side_effect - doc = IndexedFile(system='test.system', path='/test/path') + doc = IndexedFile(system="test.system", path="/test/path") children = doc.children() next(children) - mock_get.assert_called_once_with('id1') + mock_get.assert_called_once_with("id1") - @patch('portal.libs.elasticsearch.docs.base.IndexedFile.children') - @patch('portal.libs.elasticsearch.docs.base.Document.delete') + @patch("portal.libs.elasticsearch.docs.base.IndexedFile.children") + @patch("portal.libs.elasticsearch.docs.base.Document.delete") def test_delete(self, mock_delete, mock_children): child = IndexedFile() @@ -63,18 +62,19 @@ def children_side_effect(): class TestIndexedAllocation(TestCase): - @patch('portal.libs.elasticsearch.docs.base.IndexedAllocation.get') - @patch('portal.libs.elasticsearch.docs.base.Elasticsearch') + @patch("portal.libs.elasticsearch.docs.base.IndexedAllocation.get") + @patch("portal.libs.elasticsearch.docs.base.Elasticsearch") def test_from_username(self, mock_es, mock_get): mock_es_client = MagicMock() mock_es.return_value = mock_es_client - IndexedAllocation.from_username('testuser') - mock_get.assert_called_once_with('ae5deb822e0d71992900471a7199d0d95b8e7c9d05c40a8245a281fd2c1d6684', using=mock_es_client) + IndexedAllocation.from_username("testuser") + mock_get.assert_called_once_with( + "ae5deb822e0d71992900471a7199d0d95b8e7c9d05c40a8245a281fd2c1d6684", using=mock_es_client + ) class TestIndexedProject(TestCase): - - @patch('portal.libs.elasticsearch.docs.base.IndexedProject.get') + @patch("portal.libs.elasticsearch.docs.base.IndexedProject.get") def test_from_id(self, mock_get): - IndexedProject.from_id('cep.test-2') - mock_get.assert_called_once_with('cep.test-2') + IndexedProject.from_id("cep.test-2") + mock_get.assert_called_once_with("cep.test-2") diff --git a/server/portal/libs/elasticsearch/exceptions.py b/server/portal/libs/elasticsearch/exceptions.py index 0f6018753c..c8eb4aa766 100644 --- a/server/portal/libs/elasticsearch/exceptions.py +++ b/server/portal/libs/elasticsearch/exceptions.py @@ -13,6 +13,7 @@ class ESException(Exception): Some times we need to be a bit more specific to know how to handle the exception. """ + pass @@ -25,4 +26,5 @@ class DocumentNotFound(ESException): if we create it or fail. """ + pass diff --git a/server/portal/libs/elasticsearch/indexes.py b/server/portal/libs/elasticsearch/indexes.py index 5416a5c8a0..34795ef411 100644 --- a/server/portal/libs/elasticsearch/indexes.py +++ b/server/portal/libs/elasticsearch/indexes.py @@ -2,13 +2,12 @@ .. module: portal.libs.elasticsearch.indexes :synopsis: ElasticSearch Index setup """ + from datetime import datetime import logging from django.conf import settings from elasticsearch_dsl import Index -from portal.libs.elasticsearch.docs.base import (IndexedFile, - IndexedAllocation, - IndexedProject, IndexedPublication) +from portal.libs.elasticsearch.docs.base import IndexedFile, IndexedAllocation, IndexedProject, IndexedPublication from portal.libs.elasticsearch.analyzers import file_query_analyzer @@ -28,10 +27,10 @@ def setup_indexes(doc_type, reindex=False, force=False): index with that alias and the provided name. """ baseName = settings.ES_INDEX_PREFIX.format(doc_type) - indexName = '{}-{}'.format(baseName, index_time_string()) + indexName = "{}-{}".format(baseName, index_time_string()) alias = baseName if reindex: - alias += '-reindex' + alias += "-reindex" index = Index(alias) if force or not index.exists(): @@ -55,7 +54,7 @@ def index_time_string(): def setup_files_index(reindex=False, force=False): - index = setup_indexes('files', reindex, force) + index = setup_indexes("files", reindex, force) if not index.exists(): index.document(IndexedFile) index.analyzer(file_query_analyzer) @@ -64,21 +63,21 @@ def setup_files_index(reindex=False, force=False): def setup_allocations_index(reindex=False, force=False): - index = setup_indexes('allocations', reindex, force) + index = setup_indexes("allocations", reindex, force) if not index.exists(): index.document(IndexedAllocation) index.create() def setup_projects_index(reindex=False, force=False): - index = setup_indexes('projects', reindex, force) + index = setup_indexes("projects", reindex, force) if not index.exists(): index.document(IndexedProject) index.create() def setup_publications_index(reindex=False, force=False): - index = setup_indexes('publications', reindex, force) + index = setup_indexes("publications", reindex, force) if not index.exists(): index.document(IndexedPublication) index.create() diff --git a/server/portal/libs/elasticsearch/unit_test.py b/server/portal/libs/elasticsearch/unit_test.py index 2580d7dab6..3427a62a3c 100644 --- a/server/portal/libs/elasticsearch/unit_test.py +++ b/server/portal/libs/elasticsearch/unit_test.py @@ -4,151 +4,167 @@ from elasticsearch_dsl.response.hit import Hit from portal.libs.elasticsearch.indexes import setup_files_index, setup_projects_index, setup_indexes -from portal.libs.elasticsearch.utils import index_listing, index_level, file_uuid_sha256, walk_children, grouper, delete_recursive +from portal.libs.elasticsearch.utils import ( + index_listing, + index_level, + file_uuid_sha256, + walk_children, + grouper, + delete_recursive, +) class TestESSetupMethods(TestCase): def setUp(self): return - @patch('portal.libs.elasticsearch.indexes.Index') + @patch("portal.libs.elasticsearch.indexes.Index") def test_generic_setup_if_index_exists(self, mock_index): # mock_index.return_value='INDEX' mock_index.return_value.exists.return_value = True - setup_indexes('type', False, False) - mock_index.assert_called_with('test-staging-type') + setup_indexes("type", False, False) + mock_index.assert_called_with("test-staging-type") - @patch('portal.libs.elasticsearch.indexes.Index') - @patch('portal.libs.elasticsearch.indexes.index_time_string') + @patch("portal.libs.elasticsearch.indexes.Index") + @patch("portal.libs.elasticsearch.indexes.index_time_string") def test_generic_setup_if_no_index_exists(self, mock_time_string, mock_index): # mock_index.return_value='INDEX' - mock_time_string.return_value = 'TIME_NOW' + mock_time_string.return_value = "TIME_NOW" mock_index.return_value.exists.return_value = False - setup_indexes('type', False, False) - mock_index.assert_has_calls([ - call('test-staging-type'), - call().exists(), - call().exists(), # from while loop - call('test-staging-type-TIME_NOW'), - call().aliases(**{'test-staging-type': {}}) - ]) - - @patch('portal.libs.elasticsearch.indexes.setup_indexes') - @patch('portal.libs.elasticsearch.indexes.index_time_string') + setup_indexes("type", False, False) + mock_index.assert_has_calls( + [ + call("test-staging-type"), + call().exists(), + call().exists(), # from while loop + call("test-staging-type-TIME_NOW"), + call().aliases(**{"test-staging-type": {}}), + ] + ) + + @patch("portal.libs.elasticsearch.indexes.setup_indexes") + @patch("portal.libs.elasticsearch.indexes.index_time_string") def test_files_setup(self, mock_timestring, mock_setup): setup_files_index() - mock_setup.assert_called_with('files', False, False) + mock_setup.assert_called_with("files", False, False) - @patch('portal.libs.elasticsearch.indexes.setup_indexes') - @patch('portal.libs.elasticsearch.indexes.index_time_string') + @patch("portal.libs.elasticsearch.indexes.setup_indexes") + @patch("portal.libs.elasticsearch.indexes.index_time_string") def test_projects_setup(self, mock_timestring, mock_setup): setup_projects_index() - mock_setup.assert_called_with('projects', False, False) + mock_setup.assert_called_with("projects", False, False) class TestESUtils(TestCase): - def test_uuid(self): - uuid = file_uuid_sha256('test.system', '/path/to/file') - self.assertEqual(uuid, 'c7765edebe9d7b715865b83a8319703975680be5a3f5f77503bdc47e7978429c') + uuid = file_uuid_sha256("test.system", "/path/to/file") + self.assertEqual(uuid, "c7765edebe9d7b715865b83a8319703975680be5a3f5f77503bdc47e7978429c") def test_grouper(self): - g = grouper('ABCDEFG', 3, 'x') - self.assertEqual(next(g), ('A', 'B', 'C')) - self.assertEqual(next(g), ('D', 'E', 'F')) - self.assertEqual(next(g), ('G', 'x', 'x')) + g = grouper("ABCDEFG", 3, "x") + self.assertEqual(next(g), ("A", "B", "C")) + self.assertEqual(next(g), ("D", "E", "F")) + self.assertEqual(next(g), ("G", "x", "x")) with self.assertRaises(StopIteration): next(g) - @patch('portal.libs.elasticsearch.docs.base.IndexedFile.search') + @patch("portal.libs.elasticsearch.docs.base.IndexedFile.search") def test_walk_children(self, mock_search): mock_search().filter().filter().scan.return_value = [Hit({})] - children = walk_children('test.system', '/file/path', include_parent=True, recurse=True) + children = walk_children("test.system", "/file/path", include_parent=True, recurse=True) next(children) - mock_search().filter().filter.assert_called_with(Q({'prefix': {'basePath._exact': '/file/path'}}) | Q({'term': {'path._exact': '/file/path'}})) + mock_search().filter().filter.assert_called_with( + Q({"prefix": {"basePath._exact": "/file/path"}}) | Q({"term": {"path._exact": "/file/path"}}) + ) - children = walk_children('test.system', '/file/path', include_parent=True, recurse=False) + children = walk_children("test.system", "/file/path", include_parent=True, recurse=False) next(children) - mock_search().filter().filter.assert_called_with(Q({'term': {'basePath._exact': '/file/path'}}) | Q({'term': {'path._exact': '/file/path'}})) + mock_search().filter().filter.assert_called_with( + Q({"term": {"basePath._exact": "/file/path"}}) | Q({"term": {"path._exact": "/file/path"}}) + ) - children = walk_children('test.system', '/file/path', include_parent=False, recurse=True) + children = walk_children("test.system", "/file/path", include_parent=False, recurse=True) next(children) - mock_search().filter().filter.assert_called_with(Q({'prefix': {'basePath._exact': '/file/path'}})) + mock_search().filter().filter.assert_called_with(Q({"prefix": {"basePath._exact": "/file/path"}})) - children = walk_children('test.system', '/file/path', include_parent=False, recurse=False) + children = walk_children("test.system", "/file/path", include_parent=False, recurse=False) next(children) - mock_search().filter().filter.assert_called_with(Q({'term': {'basePath._exact': '/file/path'}})) + mock_search().filter().filter.assert_called_with(Q({"term": {"basePath._exact": "/file/path"}})) - @patch('portal.libs.elasticsearch.utils.walk_children') - @patch('portal.libs.elasticsearch.utils.bulk') - @patch('portal.libs.elasticsearch.utils.get_connection') + @patch("portal.libs.elasticsearch.utils.walk_children") + @patch("portal.libs.elasticsearch.utils.bulk") + @patch("portal.libs.elasticsearch.utils.get_connection") def test_delete_recursive(self, mock_conn, mock_bulk, mock_children): - mock_conn.return_value = 'default' + mock_conn.return_value = "default" def children_side_effect(*args, **kwargs): dummy_hit = Hit({}) - dummy_hit.system = 'test.system' - dummy_hit.path = '/test/file' - dummy_hit.meta.id = 'ABCDEF' + dummy_hit.system = "test.system" + dummy_hit.path = "/test/file" + dummy_hit.meta.id = "ABCDEF" yield dummy_hit + mock_children.side_effect = children_side_effect - test_op = {'_index': 'test-staging-files', - '_id': 'ABCDEF', - '_op_type': 'delete'} + test_op = {"_index": "test-staging-files", "_id": "ABCDEF", "_op_type": "delete"} - delete_recursive('test.system', '/test/file') - mock_children.assert_called_once_with('test.system', - '/test/file', - include_parent=True, - recurse=True) + delete_recursive("test.system", "/test/file") + mock_children.assert_called_once_with("test.system", "/test/file", include_parent=True, recurse=True) mock_map = mock_bulk.call_args.args[1] self.assertEqual(next(mock_map), test_op) - @patch('portal.libs.elasticsearch.utils.bulk') - @patch('portal.libs.elasticsearch.utils.current_time') - @patch('portal.libs.elasticsearch.utils.get_connection') + @patch("portal.libs.elasticsearch.utils.bulk") + @patch("portal.libs.elasticsearch.utils.current_time") + @patch("portal.libs.elasticsearch.utils.get_connection") def test_index_listing(self, mock_conn, mock_time, mock_bulk): files = [ - {'name': 'file1', 'system': 'test.system', 'path': '/test/file1'}, + {"name": "file1", "system": "test.system", "path": "/test/file1"}, ] - mock_conn.return_value = 'default' - mock_time.return_value = 'TIME_NOW' + mock_conn.return_value = "default" + mock_time.return_value = "TIME_NOW" index_listing(files) mock_bulk.assert_called_once_with( - 'default', [{'_index': 'test-staging-files', - '_id': 'd9c58e96e64076fa1205c5ba23b1f5cbd609efc9d30683db00988ca95c47cfd0', - 'doc': {'system': 'test.system', - 'name': 'file1', - 'path': '/test/file1', - 'lastUpdated': 'TIME_NOW', - 'basePath': '/test'}, - '_op_type': 'update', - 'doc_as_upsert': True}]) - - @patch('portal.libs.elasticsearch.utils.index_listing') - @patch('portal.libs.elasticsearch.utils.walk_children') - @patch('portal.libs.elasticsearch.utils.delete_recursive') + "default", + [ + { + "_index": "test-staging-files", + "_id": "d9c58e96e64076fa1205c5ba23b1f5cbd609efc9d30683db00988ca95c47cfd0", + "doc": { + "system": "test.system", + "name": "file1", + "path": "/test/file1", + "lastUpdated": "TIME_NOW", + "basePath": "/test", + }, + "_op_type": "update", + "doc_as_upsert": True, + } + ], + ) + + @patch("portal.libs.elasticsearch.utils.index_listing") + @patch("portal.libs.elasticsearch.utils.walk_children") + @patch("portal.libs.elasticsearch.utils.delete_recursive") def test_index_level(self, mock_delete, mock_children, mock_index): def children_side_effect(*args, **kwargs): dummy_hit = Hit({}) - dummy_hit.system = 'test.system' - dummy_hit.path = '/deleted/file' + dummy_hit.system = "test.system" + dummy_hit.path = "/deleted/file" yield dummy_hit mock_children.side_effect = children_side_effect - testfile = {'system': 'test.system', 'path': '/test/file', 'name': 'file'} - testfolder = {'system': 'test.system', 'path': '/test/folder', 'name': 'folder'} + testfile = {"system": "test.system", "path": "/test/file", "name": "file"} + testfolder = {"system": "test.system", "path": "/test/folder", "name": "folder"} - index_level('/test', [testfolder], [testfile], 'test.system') + index_level("/test", [testfolder], [testfile], "test.system") mock_index.assert_called_once_with([testfolder, testfile]) - mock_delete.assert_called_once_with('test.system', '/deleted/file') + mock_delete.assert_called_once_with("test.system", "/deleted/file") diff --git a/server/portal/libs/elasticsearch/utils.py b/server/portal/libs/elasticsearch/utils.py index 90666209fd..5db76a63f9 100644 --- a/server/portal/libs/elasticsearch/utils.py +++ b/server/portal/libs/elasticsearch/utils.py @@ -59,8 +59,8 @@ def file_uuid_sha256(system, path): str """ - if not path.startswith('/'): - path = '/{}'.format(path) + if not path.startswith("/"): + path = "/{}".format(path) # str representation of the hash of e.g. "cep.home.user/path/to/file" return sha256((system + path).encode()).hexdigest() @@ -97,15 +97,16 @@ def walk_children(system, path, include_parent=False, recurse=False): """ from portal.libs.elasticsearch.docs.base import IndexedFile + search = IndexedFile.search() - search = search.filter(Q({'term': {'system._exact': system}})) + search = search.filter(Q({"term": {"system._exact": system}})) if recurse: - basepath_query = Q({'prefix': {'basePath._exact': path}}) + basepath_query = Q({"prefix": {"basePath._exact": path}}) else: - basepath_query = Q({'term': {'basePath._exact': path}}) + basepath_query = Q({"term": {"basePath._exact": path}}) if include_parent: - path_query = Q({'term': {'path._exact': path}}) + path_query = Q({"term": {"path._exact": path}}) search = search.filter(basepath_query | path_query) else: search = search.filter(basepath_query) @@ -131,17 +132,15 @@ def delete_recursive(system, path): Void """ from portal.libs.elasticsearch.docs.base import IndexedFile + hits = walk_children(system, path, include_parent=True, recurse=True) idx = IndexedFile.Index.name - client = get_connection('default') + client = get_connection("default") # Group children in batches of 100 for bulk deletion. for group in grouper(hits, 100): filtered_group = filter(lambda hit: hit is not None, group) - ops = map(lambda hit: {'_index': idx, - '_id': hit.meta.id, - '_op_type': 'delete'}, - filtered_group) + ops = map(lambda hit: {"_index": idx, "_id": hit.meta.id, "_op_type": "delete"}, filtered_group) bulk(client, ops) @@ -168,9 +167,9 @@ def index_level(path, folders, files, systemId, reindex=False): index_listing(folders + files) - children_paths = [_file['path'] for _file in folders + files] + children_paths = [_file["path"] for _file in folders + files] for hit in walk_children(systemId, path, recurse=False): - if hit['path'] not in children_paths: + if hit["path"] not in children_paths: delete_recursive(hit.system, hit.path) @@ -200,23 +199,18 @@ def index_listing(files): Void """ from portal.libs.elasticsearch.docs.base import IndexedFile + idx = IndexedFile.Index.name - client = get_connection('default') + client = get_connection("default") ops = [] for _file in files: file_dict = dict(_file) - if file_dict['name'][0] == '.': + if file_dict["name"][0] == ".": continue - file_dict['lastUpdated'] = current_time() - file_dict['basePath'] = os.path.dirname(file_dict['path']) - file_uuid = file_uuid_sha256(file_dict['system'], file_dict['path']) - ops.append({ - '_index': idx, - '_id': file_uuid, - 'doc': file_dict, - '_op_type': 'update', - 'doc_as_upsert': True - }) + file_dict["lastUpdated"] = current_time() + file_dict["basePath"] = os.path.dirname(file_dict["path"]) + file_uuid = file_uuid_sha256(file_dict["system"], file_dict["path"]) + ops.append({"_index": idx, "_id": file_uuid, "doc": file_dict, "_op_type": "update", "doc_as_upsert": True}) bulk(client, ops) @@ -225,19 +219,15 @@ def index_project_listing(projects): from portal.libs.elasticsearch.docs.base import IndexedProject idx = IndexedProject.Index.name - client = get_connection('default') + client = get_connection("default") ops = [] for _project in projects: project_dict = dict(_project) - project_dict['updated'] = current_time() - project_uuid = get_sha256_hash(project_dict['id']) - ops.append({ - '_index': idx, - '_id': project_uuid, - 'doc': project_dict, - '_op_type': 'update', - 'doc_as_upsert': True - }) + project_dict["updated"] = current_time() + project_uuid = get_sha256_hash(project_dict["id"]) + ops.append( + {"_index": idx, "_id": project_uuid, "doc": project_dict, "_op_type": "update", "doc_as_upsert": True} + ) bulk(client, ops) diff --git a/server/portal/libs/exceptions.py b/server/portal/libs/exceptions.py index c79bc36d35..5df5895c80 100644 --- a/server/portal/libs/exceptions.py +++ b/server/portal/libs/exceptions.py @@ -2,14 +2,16 @@ .. :module:: portal.libs.exceptions :synopsis: Exceptions defined for custom libs. """ + import logging # pylint: disable=invalid-name logger = logging.getLogger(__name__) -METRICS = logging.getLogger('metrics.{}'.format(__name__)) +METRICS = logging.getLogger("metrics.{}".format(__name__)) # pylint: enable=invalid-name class PortalLibException(Exception): """Portal Lib Exception""" + pass diff --git a/server/portal/libs/files/file_processing.py b/server/portal/libs/files/file_processing.py index 490a146642..d2635f30a8 100644 --- a/server/portal/libs/files/file_processing.py +++ b/server/portal/libs/files/file_processing.py @@ -18,25 +18,28 @@ def conf_raw(img, file): # slices, width, height should be converted to ints # NOTE: If an 8-bit raw comes through, we need to set the datatype for that to unsigned. - logger.info(f'img: {img}') + logger.info(f"img: {img}") - prefix_map = {'little_endian': '<', 'big_endian': '>'} + prefix_map = {"little_endian": "<", "big_endian": ">"} suffix_map = { - '8_bit': 'u1', '16_bit_unsigned': 'u2', '32_bit_unsigned': 'u4', '64_bit_unsigned': 'u8', - '8_bit_signed': 'i1', '16_bit_signed': 'i2', '32_bit_signed': 'i4', '64_bit_signed': 'i8', - '32_bit_real': 'f4', '64_bit_real': 'f8' + "8_bit": "u1", + "16_bit_unsigned": "u2", + "32_bit_unsigned": "u4", + "64_bit_unsigned": "u8", + "8_bit_signed": "i1", + "16_bit_signed": "i2", + "32_bit_signed": "i4", + "64_bit_signed": "i8", + "32_bit_real": "f4", + "64_bit_real": "f8", } - prefix = prefix_map.get(img['byte_order'], '|') # Default to native byte order if unknown - suffix = suffix_map.get(img['image_type']) + prefix = prefix_map.get(img["byte_order"], "|") # Default to native byte order if unknown + suffix = suffix_map.get(img["image_type"]) datatype = prefix + suffix file_data = np.frombuffer(file, dtype=datatype) - return file_data.reshape([ - int(img['number_of_images']), - int(img['height']), - int(img['width']) - ]) + return file_data.reshape([int(img["number_of_images"]), int(img["height"]), int(img["width"])]) def conf_tiff(file): @@ -47,15 +50,15 @@ def conf_tiff(file): def binary_correction(img): - logger.debug('Correcting for Binary values...') + logger.debug("Correcting for Binary values...") min_value = np.min(img) max_value = np.max(img) - k = 255/(max_value-min_value) - offset = -k*min_value + k = 255 / (max_value - min_value) + offset = -k * min_value image1 = np.floor(img * k + offset) del img - return image1.astype('uint8') + return image1.astype("uint8") def create_thumbnail(img): @@ -67,33 +70,33 @@ def create_thumbnail(img): if len(img.shape) == 3 and 3 not in img.shape: width = img.shape[2] height = img.shape[1] - depth_slice = int(np.ceil(img.shape[0]/2)) + depth_slice = int(np.ceil(img.shape[0] / 2)) elif len(img.shape) == 3 and 3 in img.shape: # TODO: Handle RGB files shape ==> (h, w, 3) - logger.debug('handle RGB') + logger.debug("handle RGB") # preserve aspect ratio and resize to fit. - modifier = dim_max/width if width > height else dim_max/height - resized_width = width*modifier - resized_height = height*modifier + modifier = dim_max / width if width > height else dim_max / height + resized_width = width * modifier + resized_height = height * modifier fig = plt.figure() fig.set_size_inches(resized_width, resized_height, dpi) - ax = plt.Axes(fig, [0., 0., 1., 1.]) + ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() fig.add_axes(ax) # TODO: Swap color mapping if image is bitmap/8bit # plt.set_cmap('gray') if img.invert_colors else plt.set_cmap('Greys') - plt.set_cmap('Greys') + plt.set_cmap("Greys") if depth_slice is not None: - logger.debug('Creating Thumbnail from 3D tif') - ax.imshow(img[depth_slice, :, :], aspect='equal', vmin=0, vmax=255) + logger.debug("Creating Thumbnail from 3D tif") + ax.imshow(img[depth_slice, :, :], aspect="equal", vmin=0, vmax=255) else: - logger.debug('Creating Thumbnail from FLAT tif') - ax.imshow(img, aspect='equal') + logger.debug("Creating Thumbnail from FLAT tif") + ax.imshow(img, aspect="equal") buffer = io.BytesIO() - plt.savefig(buffer, format='jpeg', dpi=dpi) + plt.savefig(buffer, format="jpeg", dpi=dpi) buffer.seek(0) plt.close(fig) @@ -102,27 +105,35 @@ def create_thumbnail(img): def create_histogram(img): - logger.debug('Creating Histogram') + logger.debug("Creating Histogram") nbins = 256 fig_hist = plt.figure(figsize=(4, 2.4)) - freq, bins, patches = plt.hist(img.reshape([np.size(img),]), nbins, density=True) - plt.xlabel('Gray value') - plt.ylabel('Probability') + freq, bins, patches = plt.hist( + img.reshape( + [ + np.size(img), + ] + ), + nbins, + density=True, + ) + plt.xlabel("Gray value") + plt.ylabel("Probability") plt.tight_layout() image_buffer = io.BytesIO() - fig_hist.savefig(image_buffer, format='jpeg', dpi=200) + fig_hist.savefig(image_buffer, format="jpeg", dpi=200) image_buffer.seek(0) plt.close(fig_hist) csv_buffer = io.StringIO() - histwriter = csv.writer(csv_buffer, delimiter=',') - histwriter.writerow(('Value', 'Probability')) + histwriter = csv.writer(csv_buffer, delimiter=",") + histwriter.writerow(("Value", "Probability")) for i in range(np.size(freq)): histwriter.writerow((bins[i], freq[i])) csv_buffer.seek(0) - logger.debug('Histogram Created') + logger.debug("Histogram Created") return image_buffer.getvalue(), csv_buffer.getvalue() @@ -138,10 +149,10 @@ def create_animation(img): bytes: Binary data of the animated GIF. """ if len(img.shape) < 3 or (len(img.shape) == 3 and 3 in img.shape): - logger.debug('Image is not a 3D array') + logger.debug("Image is not a 3D array") return # Exit if the image is not a 3D array - logger.debug('Creating Animated Gif') + logger.debug("Creating Animated Gif") class AnimatedGif: def __init__(self): @@ -150,18 +161,18 @@ def __init__(self): def add(self, image, h, w, dpi=100): self.fig.set_size_inches(w, h, dpi) - ax1 = plt.Axes(self.fig, [0., 0., 1., 1.]) + ax1 = plt.Axes(self.fig, [0.0, 0.0, 1.0, 1.0]) ax1.set_axis_off() self.fig.add_axes(ax1) - plt.set_cmap('Greys') - plt_im = ax1.imshow(image, aspect='equal', vmin=0, vmax=255) + plt.set_cmap("Greys") + plt_im = ax1.imshow(image, aspect="equal", vmin=0, vmax=255) self.images.append([plt_im]) def save_to_tempfile(self): """Saves the animation to a temporary file and returns its binary content.""" with tempfile.NamedTemporaryFile(suffix=".gif", delete=True) as temp_file: animation = anim.ArtistAnimation(self.fig, self.images) - animation.save(temp_file.name, writer='imagemagick', fps=6) + animation.save(temp_file.name, writer="imagemagick", fps=6) temp_file.seek(0) # Reset pointer to the beginning return temp_file.read() # Read binary content @@ -188,7 +199,7 @@ def save_to_tempfile(self): # Save the animation to a temporary file and return its binary data gif_binary_data = animated_gif.save_to_tempfile() - logger.debug('Animated Gif Created') + logger.debug("Animated Gif Created") return gif_binary_data @@ -202,10 +213,10 @@ def resize_cover_image(img): ext = ext.lower() format_map = { - '.jpg': 'JPEG', - '.jpeg': 'JPEG', - '.png': 'PNG', - '.gif': 'GIF', + ".jpg": "JPEG", + ".jpeg": "JPEG", + ".png": "PNG", + ".gif": "GIF", } image_format = format_map.get(ext) diff --git a/server/portal/libs/googledrive/files.py b/server/portal/libs/googledrive/files.py index 38a619946a..e8d324b3c8 100644 --- a/server/portal/libs/googledrive/files.py +++ b/server/portal/libs/googledrive/files.py @@ -7,31 +7,124 @@ class GoogleDriveFile(object): """Represents a google drive file""" SUPPORTED_IMAGE_PREVIEW_EXTS = [ - '.ai', '.bmp', '.gif', '.eps', '.jpeg', '.jpg', '.png', '.ps', '.psd', '.svg', '.tif', '.tiff', - '.dcm', '.dicm', '.dicom', '.svs', '.tga', + ".ai", + ".bmp", + ".gif", + ".eps", + ".jpeg", + ".jpg", + ".png", + ".ps", + ".psd", + ".svg", + ".tif", + ".tiff", + ".dcm", + ".dicm", + ".dicom", + ".svs", + ".tga", ] SUPPORTED_TEXT_PREVIEWS = [ - '.as', '.as3', '.asm', '.bat', '.c', '.cc', '.cmake', '.cpp', '.cs', '.css', '.csv', '.cxx', - '.diff', '.doc', '.docx', '.erb', '.gdoc', '.groovy', '.gsheet', '.h', '.haml', '.hh', '.htm', - '.html', '.java', '.js', '.less', '.m', '.make', '.ml', '.mm', '.msg', '.ods', '.odt', '.odp', - '.php', '.pl', '.ppt', '.pptx', '.properties', '.py', '.rb', '.rtf', '.sass', '.scala', - '.scm', '.script', '.sh', '.sml', '.sql', '.txt', '.vi', '.vim', '.wpd', '.xls', '.xlsm', - '.xlsx', '.xml', '.xsd', '.xsl', '.yaml', + ".as", + ".as3", + ".asm", + ".bat", + ".c", + ".cc", + ".cmake", + ".cpp", + ".cs", + ".css", + ".csv", + ".cxx", + ".diff", + ".doc", + ".docx", + ".erb", + ".gdoc", + ".groovy", + ".gsheet", + ".h", + ".haml", + ".hh", + ".htm", + ".html", + ".java", + ".js", + ".less", + ".m", + ".make", + ".ml", + ".mm", + ".msg", + ".ods", + ".odt", + ".odp", + ".php", + ".pl", + ".ppt", + ".pptx", + ".properties", + ".py", + ".rb", + ".rtf", + ".sass", + ".scala", + ".scm", + ".script", + ".sh", + ".sml", + ".sql", + ".txt", + ".vi", + ".vim", + ".wpd", + ".xls", + ".xlsm", + ".xlsx", + ".xml", + ".xsd", + ".xsl", + ".yaml", ] SUPPORTED_OBJECT_PREVIEW_EXTS = [ - '.pdf', - '.aac', '.aifc', '.aiff', '.amr', '.au', '.flac', '.m4a', '.mp3', '.ogg', '.ra', '.wav', '.wma', - + ".pdf", + ".aac", + ".aifc", + ".aiff", + ".amr", + ".au", + ".flac", + ".m4a", + ".mp3", + ".ogg", + ".ra", + ".wav", + ".wma", # VIDEO - '.3g2', '.3gp', '.avi', '.m2v', '.m2ts', '.m4v', '.mkv', '.mov', '.mp4', '.mpeg', '.mpg', - '.ogg', '.mts', '.qt', '.wmv', + ".3g2", + ".3gp", + ".avi", + ".m2v", + ".m2ts", + ".m4v", + ".mkv", + ".mov", + ".mp4", + ".mpeg", + ".mpg", + ".ogg", + ".mts", + ".qt", + ".wmv", ] - SUPPORTED_PREVIEW_EXTENSIONS = (SUPPORTED_IMAGE_PREVIEW_EXTS + - SUPPORTED_TEXT_PREVIEWS + - SUPPORTED_OBJECT_PREVIEW_EXTS) + SUPPORTED_PREVIEW_EXTENSIONS = ( + SUPPORTED_IMAGE_PREVIEW_EXTS + SUPPORTED_TEXT_PREVIEWS + SUPPORTED_OBJECT_PREVIEW_EXTS + ) def __init__(self, googledrive_item, parent=None, drive=None): self._item = googledrive_item @@ -44,72 +137,80 @@ def __init__(self, googledrive_item, parent=None, drive=None): @property def id(self): - return '{}/{}'.format(self.type, self._item['id']) + return "{}/{}".format(self.type, self._item["id"]) @property def name(self): - return self._item['name'].encode('utf-8') + return self._item["name"].encode("utf-8") @property def path(self): if self._parent: - if self._parent.name == 'My Drive': - path = '/{}'.format(self.name) + if self._parent.name == "My Drive": + path = "/{}".format(self.name) else: - path = '/'.join([self._parent.path, self.name]) - elif 'parents' in self._item: + path = "/".join([self._parent.path, self.name]) + elif "parents" in self._item: parent = self - path = '{}'.format(self.name) + path = "{}".format(self.name) while True: try: self._path_collection.insert( - 0, {'id': parent.id, 'name': '' if parent.name == 'My Drive' else parent.name}) - parent = GoogleDriveFile(self._driveapi.files().get( - fileId=parent._item['parents'][0], fields="parents, name, id, mimeType").execute()) - parent_name = '' if parent.name == 'My Drive' else parent.name + 0, {"id": parent.id, "name": "" if parent.name == "My Drive" else parent.name} + ) + parent = GoogleDriveFile( + self._driveapi.files() + .get(fileId=parent._item["parents"][0], fields="parents, name, id, mimeType") + .execute() + ) + parent_name = "" if parent.name == "My Drive" else parent.name path = "{}/{}".format(parent_name, path) except (AttributeError, KeyError): break else: - path = '' + path = "" return path @property def length(self): - return self._item.get('size') + return self._item.get("size") @property def last_modified(self): - return self._item.get('modifiedTime') + return self._item.get("modifiedTime") @property def type(self): - if self._item['mimeType'] == 'application/vnd.google-apps.folder': - return 'dir' + if self._item["mimeType"] == "application/vnd.google-apps.folder": + return "dir" else: - return 'file' + return "file" @property def ext(self): try: - return '.{}'.format(self._item['fileExtension']).lower() + return ".{}".format(self._item["fileExtension"]).lower() except KeyError: return None @property def trail(self): - trail = [{'name': self._path_collection[i]['name'] or '/', - 'system': None, - 'resource': 'googledrive', - 'id': self._path_collection[i]['id'], - 'path': '/'.join(j['name'] for j in self._path_collection[0:i + 1]) or '/', - } for i in range(0, len(self._path_collection))] + trail = [ + { + "name": self._path_collection[i]["name"] or "/", + "system": None, + "resource": "googledrive", + "id": self._path_collection[i]["id"], + "path": "/".join(j["name"] for j in self._path_collection[0 : i + 1]) or "/", + } + for i in range(0, len(self._path_collection)) + ] return trail @property def previewable(self): - return self.type != 'dir' and self.ext in self.SUPPORTED_PREVIEW_EXTENSIONS + return self.type != "dir" and self.ext in self.SUPPORTED_PREVIEW_EXTENSIONS @staticmethod def parse_file_id(file_id): @@ -127,31 +228,29 @@ def parse_file_id(file_id): Raises: AssertionError """ - parts = file_id.split('/') + parts = file_id.split("/") - assert len( - parts) == 2, 'The file path should be in the format {type}/{id}' - assert parts[0] in [ - 'dir', 'file'], '{type} must be one of ["folder", "file"]' + assert len(parts) == 2, "The file path should be in the format {type}/{id}" + assert parts[0] in ["dir", "file"], '{type} must be one of ["folder", "file"]' return parts[0], parts[1] def to_dict(self, trail=True, **kwargs): - pems = kwargs.get('default_pems', []) + pems = kwargs.get("default_pems", []) obj_dict = { - 'system': None, - 'id': self.id, - 'type': self.type, - 'path': self.path, - 'name': self.name, - 'ext': self.ext, - 'length': self.length, - 'lastModified': self.last_modified, - '_actions': [], - 'permissions': pems, - 'resource': 'googledrive' + "system": None, + "id": self.id, + "type": self.type, + "path": self.path, + "name": self.name, + "ext": self.ext, + "length": self.length, + "lastModified": self.last_modified, + "_actions": [], + "permissions": pems, + "resource": "googledrive", } if trail: - obj_dict['trail'] = self.trail + obj_dict["trail"] = self.trail return obj_dict diff --git a/server/portal/libs/googledrive/operations.py b/server/portal/libs/googledrive/operations.py index 722390ba4b..575b5dad85 100644 --- a/server/portal/libs/googledrive/operations.py +++ b/server/portal/libs/googledrive/operations.py @@ -8,87 +8,93 @@ logger = logging.getLogger(__name__) -def listing(client, system, path, offset=None, limit=100, nextPageToken=None, - *args, **kwargs): +def listing(client, system, path, offset=None, limit=100, nextPageToken=None, *args, **kwargs): if not path: - path = 'root' - fields = ("mimeType, name, id, modifiedTime, " - "fileExtension, size, parents, webViewLink") - listing_call = client.files()\ - .list(q="'{}' in parents and trashed=False".format(path), - fields="files({}), nextPageToken" - .format(fields), pageSize=limit, pageToken=nextPageToken)\ + path = "root" + fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents, webViewLink" + listing_call = ( + client.files() + .list( + q="'{}' in parents and trashed=False".format(path), + fields="files({}), nextPageToken".format(fields), + pageSize=limit, + pageToken=nextPageToken, + ) .execute() - listing = listing_call.get('files') - scroll_token = listing_call.get('nextPageToken') + ) + listing = listing_call.get("files") + scroll_token = listing_call.get("nextPageToken") reached_end = not bool(scroll_token) - folder_mimetype = 'application/vnd.google-apps.folder' - listing = list(map(lambda f: { - 'system': 'googledrive', - 'type': 'dir' if f['mimeType'] == folder_mimetype else 'file', - 'format': 'folder' if f['mimeType'] == folder_mimetype else 'raw', - 'mimeType': f['mimeType'], - 'path': f['id'], - 'name': f['name'], - 'length': int(f['size']) if 'size' in f.keys() else 0, - 'lastModified': f['modifiedTime'], - '_links': { - 'self': {'href': f['webViewLink']} - } - }, listing)) - - return {'listing': listing, - 'nextPageToken': scroll_token, - 'reachedEnd': reached_end} - - -def search(client, system, path, offset=None, limit=100, nextPageToken=None, - query_string='', *args, **kwargs): + folder_mimetype = "application/vnd.google-apps.folder" + listing = list( + map( + lambda f: { + "system": "googledrive", + "type": "dir" if f["mimeType"] == folder_mimetype else "file", + "format": "folder" if f["mimeType"] == folder_mimetype else "raw", + "mimeType": f["mimeType"], + "path": f["id"], + "name": f["name"], + "length": int(f["size"]) if "size" in f.keys() else 0, + "lastModified": f["modifiedTime"], + "_links": {"self": {"href": f["webViewLink"]}}, + }, + listing, + ) + ) + + return {"listing": listing, "nextPageToken": scroll_token, "reachedEnd": reached_end} + + +def search(client, system, path, offset=None, limit=100, nextPageToken=None, query_string="", *args, **kwargs): if not path: - path = 'root' - fields = ("mimeType, name, id, modifiedTime, " - "fileExtension, size, parents, webViewLink") - listing_call = client.files()\ - .list(q=("'{path}' in parents and " - "trashed=False and " - "name contains '{query_string}'") - .format(path=path, query_string=query_string), - fields="files({}), nextPageToken" - .format(fields), pageSize=limit, pageToken=nextPageToken)\ + path = "root" + fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents, webViewLink" + listing_call = ( + client.files() + .list( + q=("'{path}' in parents and trashed=False and name contains '{query_string}'").format( + path=path, query_string=query_string + ), + fields="files({}), nextPageToken".format(fields), + pageSize=limit, + pageToken=nextPageToken, + ) .execute() - listing = listing_call.get('files') - scroll_token = listing_call.get('nextPageToken') + ) + listing = listing_call.get("files") + scroll_token = listing_call.get("nextPageToken") reached_end = not bool(scroll_token) - folder_mimetype = 'application/vnd.google-apps.folder' - listing = list(map(lambda f: { - 'system': 'googledrive', - 'type': 'dir' if f['mimeType'] == folder_mimetype else 'file', - 'format': 'folder' if f['mimeType'] == folder_mimetype else 'raw', - 'mimeType': f['mimeType'], - 'path': f['id'], - 'name': f['name'], - 'length': int(f['size']) if 'size' in f.keys() else 0, - 'lastModified': f['modifiedTime'], - '_links': { - 'self': {'href': f['webViewLink']} - } - }, listing)) - - return {'listing': listing, - 'nextPageToken': scroll_token, - 'reachedEnd': reached_end} + folder_mimetype = "application/vnd.google-apps.folder" + listing = list( + map( + lambda f: { + "system": "googledrive", + "type": "dir" if f["mimeType"] == folder_mimetype else "file", + "format": "folder" if f["mimeType"] == folder_mimetype else "raw", + "mimeType": f["mimeType"], + "path": f["id"], + "name": f["name"], + "length": int(f["size"]) if "size" in f.keys() else 0, + "lastModified": f["modifiedTime"], + "_links": {"self": {"href": f["webViewLink"]}}, + }, + listing, + ) + ) + + return {"listing": listing, "nextPageToken": scroll_token, "reachedEnd": reached_end} def iterate_listing(client, system, path, limit=100): if not path: - path = 'root' + path = "root" scroll_token = None while True: - _listing = listing(client, system, path, limit=limit, - nextPageToken=scroll_token) + _listing = listing(client, system, path, limit=limit, nextPageToken=scroll_token) - scroll_token = _listing['nextPageToken'] - yield from _listing['listing'] + scroll_token = _listing["nextPageToken"] + yield from _listing["listing"] if not scroll_token: break @@ -96,47 +102,39 @@ def iterate_listing(client, system, path, limit=100): def walk_all(client, system, path, limit=100): if not path: - path = 'root' + path = "root" for f in iterate_listing(client, system, path, limit): yield f - if f['format'] == 'folder': - yield from walk_all(client, system, f['path'], limit) + if f["format"] == "folder": + yield from walk_all(client, system, f["path"], limit) def upload(client, system, path, uploaded_file, *args, **kwargs): if not path: - path = 'root' + path = "root" mimetype = magic.from_buffer(uploaded_file.getvalue(), mime=True) media = MediaIoBaseUpload(uploaded_file, mimetype=mimetype) - file_meta = { - 'name': os.path.basename(uploaded_file.name), - 'parents': [path] - } + file_meta = {"name": os.path.basename(uploaded_file.name), "parents": [path]} client.files().create(body=file_meta, media_body=media).execute() def mkdir(client, system, path, dir_name): if not path: - path = 'root' - file_metadata = { - 'name': dir_name, - 'parents': [path], - 'mimeType': 'application/vnd.google-apps.folder' - } - fields = 'mimeType, name, id, modifiedTime, fileExtension, size, parents' - newdir = client.files().create(body=file_metadata, - fields=fields).execute() + path = "root" + file_metadata = {"name": dir_name, "parents": [path], "mimeType": "application/vnd.google-apps.folder"} + fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents" + newdir = client.files().create(body=file_metadata, fields=fields).execute() - folder_mimetype = 'application/vnd.google-apps.folder' + folder_mimetype = "application/vnd.google-apps.folder" newdir_dict = { - 'system': None, - 'type': 'dir' if newdir['mimeType'] == folder_mimetype else 'file', - 'format': 'folder' if newdir['mimeType'] == folder_mimetype else 'raw', - 'mimeType': newdir['mimeType'], - 'path': newdir['id'], - 'name': newdir['name'], - 'length': 0, - 'lastModified': newdir['modifiedTime'], + "system": None, + "type": "dir" if newdir["mimeType"] == folder_mimetype else "file", + "format": "folder" if newdir["mimeType"] == folder_mimetype else "raw", + "mimeType": newdir["mimeType"], + "path": newdir["id"], + "name": newdir["name"], + "length": 0, + "lastModified": newdir["modifiedTime"], } return newdir_dict @@ -144,10 +142,9 @@ def mkdir(client, system, path, dir_name): def download(client, system, path, *args, **kwargs): if not path: - path = 'root' + path = "root" file_id = path - file_name = client.files().get(fileId=file_id, fields="name")\ - .execute()['name'] + file_name = client.files().get(fileId=file_id, fields="name").execute()["name"] request = client.files().get_media(fileId=file_id) fh = io.BytesIO() downloader = MediaIoBaseDownload(fh, request) @@ -159,22 +156,23 @@ def download(client, system, path, *args, **kwargs): return fh -def copy(client, src_system, src_path, dest_system, dest_path, file_name, - filetype='file', dest_path_name='', *args): +def copy(client, src_system, src_path, dest_system, dest_path, file_name, filetype="file", dest_path_name="", *args): from portal.libs.transfer.operations import transfer, transfer_folder + if not src_path: - src_path = 'root' + src_path = "root" # Google drive doesn't have a robust copy API, so this endpoint uses # generic transfer methods. - if filetype == 'file': - transfer(client, client, 'googledrive', 'googledrive', src_system, - dest_system, src_path, dest_path) - if filetype == 'dir': - transfer_folder(client, client, 'googledrive', 'googledrive', - src_system, dest_system, src_path, dest_path, - file_name) - - return {'nativeFormat': filetype, - 'name': dest_path_name, - 'path': os.path.join(dest_path_name, file_name), - 'systemId': dest_system} + if filetype == "file": + transfer(client, client, "googledrive", "googledrive", src_system, dest_system, src_path, dest_path) + if filetype == "dir": + transfer_folder( + client, client, "googledrive", "googledrive", src_system, dest_system, src_path, dest_path, file_name + ) + + return { + "nativeFormat": filetype, + "name": dest_path_name, + "path": os.path.join(dest_path_name, file_name), + "systemId": dest_system, + } diff --git a/server/portal/libs/googledrive/operations_unit_test.py b/server/portal/libs/googledrive/operations_unit_test.py index f3256df743..db1979ada1 100644 --- a/server/portal/libs/googledrive/operations_unit_test.py +++ b/server/portal/libs/googledrive/operations_unit_test.py @@ -6,8 +6,8 @@ @pytest.fixture def mock_uploader(mocker): from googleapiclient.http import MediaIoBaseUpload - patched = mocker.patch( - 'portal.libs.googledrive.operations.MediaIoBaseUpload') + + patched = mocker.patch("portal.libs.googledrive.operations.MediaIoBaseUpload") patched.return_value = MagicMock(spec=MediaIoBaseUpload) yield patched @@ -16,8 +16,8 @@ def mock_uploader(mocker): @pytest.fixture def mock_downloader(mocker): from googleapiclient.http import MediaIoBaseDownload - patched = mocker.patch( - 'portal.libs.googledrive.operations.MediaIoBaseDownload') + + patched = mocker.patch("portal.libs.googledrive.operations.MediaIoBaseDownload") patched.return_value = MagicMock(spec=MediaIoBaseDownload) yield patched @@ -26,13 +26,17 @@ def mock_downloader(mocker): @pytest.fixture def mock_googledrive_listing(): def side_effect(key): - if key == 'files': - return [{'mimeType': 'application/vnd.google-apps.folder', - 'id': '1234', - 'name': 'mockfile', - 'modifiedTime': 'mocktime', - 'webViewLink': 'http://webviewlink'}] - if key == 'nextPageToken': + if key == "files": + return [ + { + "mimeType": "application/vnd.google-apps.folder", + "id": "1234", + "name": "mockfile", + "modifiedTime": "mocktime", + "webViewLink": "http://webviewlink", + } + ] + if key == "nextPageToken": return None mock_listing = MagicMock() @@ -42,111 +46,111 @@ def side_effect(key): @pytest.fixture def mock_listing_operation(mocker): - res1 = {'listing': [{ - 'system': 'googledrive', - 'type': 'dir', - 'format': 'folder', - 'mimeType': 'application/vnd.google-apps.folder', - 'path': '1234', - 'name': 'mockdir', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }}], - 'nextPageToken': None, - 'reachedEnd': True + res1 = { + "listing": [ + { + "system": "googledrive", + "type": "dir", + "format": "folder", + "mimeType": "application/vnd.google-apps.folder", + "path": "1234", + "name": "mockdir", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, } + ], + "nextPageToken": None, + "reachedEnd": True, + } - res2 = {'listing': [{ - 'system': 'googledrive', - 'type': 'file', - 'format': 'file', - 'mimeType': 'text/plain', - 'path': '1234', - 'name': 'mockfile', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }}], - 'nextPageToken': None, - 'reachedEnd': True + res2 = { + "listing": [ + { + "system": "googledrive", + "type": "file", + "format": "file", + "mimeType": "text/plain", + "path": "1234", + "name": "mockfile", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, } + ], + "nextPageToken": None, + "reachedEnd": True, + } - mock_listing = mocker.patch('portal.libs.googledrive.operations.listing') + mock_listing = mocker.patch("portal.libs.googledrive.operations.listing") mock_listing.side_effect = [res1, res2] -def test_googledrive_listing(mock_googledrive_client, - mock_googledrive_listing): +def test_googledrive_listing(mock_googledrive_client, mock_googledrive_listing): from portal.libs.googledrive.operations import listing - mock_googledrive_client.files().list().execute.return_value = \ - mock_googledrive_listing + mock_googledrive_client.files().list().execute.return_value = mock_googledrive_listing - test_listing = listing(mock_googledrive_client, 'googledrive', 'abcd', - nextPageToken=None) + test_listing = listing(mock_googledrive_client, "googledrive", "abcd", nextPageToken=None) expected_result = { - 'listing': [{ - 'system': 'googledrive', - 'type': 'dir', - 'format': 'folder', - 'mimeType': 'application/vnd.google-apps.folder', - 'path': '1234', - 'name': 'mockfile', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }}], - 'nextPageToken': None, - 'reachedEnd': True + "listing": [ + { + "system": "googledrive", + "type": "dir", + "format": "folder", + "mimeType": "application/vnd.google-apps.folder", + "path": "1234", + "name": "mockfile", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } + ], + "nextPageToken": None, + "reachedEnd": True, } mock_googledrive_client.files().list.assert_called_with( q="'abcd' in parents and trashed=False", fields="files(mimeType, name, id, modifiedTime, fileExtension, size, parents, webViewLink), nextPageToken", pageSize=100, - pageToken=None + pageToken=None, ) assert test_listing == expected_result -def test_googledrive_search(mock_googledrive_client, - mock_googledrive_listing): +def test_googledrive_search(mock_googledrive_client, mock_googledrive_listing): from portal.libs.googledrive.operations import search - mock_googledrive_client.files().list().execute.return_value = \ - mock_googledrive_listing + mock_googledrive_client.files().list().execute.return_value = mock_googledrive_listing - test_listing = search(mock_googledrive_client, 'googledrive', 'abcd', - nextPageToken=None, query_string='testquery') + test_listing = search(mock_googledrive_client, "googledrive", "abcd", nextPageToken=None, query_string="testquery") expected_result = { - 'listing': [{ - 'system': 'googledrive', - 'type': 'dir', - 'format': 'folder', - 'mimeType': 'application/vnd.google-apps.folder', - 'path': '1234', - 'name': 'mockfile', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }}], - 'nextPageToken': None, - 'reachedEnd': True + "listing": [ + { + "system": "googledrive", + "type": "dir", + "format": "folder", + "mimeType": "application/vnd.google-apps.folder", + "path": "1234", + "name": "mockfile", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } + ], + "nextPageToken": None, + "reachedEnd": True, } mock_googledrive_client.files().list.assert_called_with( q="'abcd' in parents and trashed=False and name contains 'testquery'", fields="files(mimeType, name, id, modifiedTime, fileExtension, size, parents, webViewLink), nextPageToken", pageSize=100, - pageToken=None + pageToken=None, ) assert test_listing == expected_result @@ -154,104 +158,103 @@ def test_googledrive_search(mock_googledrive_client, def test_upload(mock_googledrive_client, mock_uploader): from portal.libs.googledrive.operations import upload - testfile = io.StringIO('Test File Content') + + testfile = io.StringIO("Test File Content") testfile.name = "testfile" - upload(mock_googledrive_client, 'googledrive', 'testpath', testfile) + upload(mock_googledrive_client, "googledrive", "testpath", testfile) - mock_uploader.assert_called_with(testfile, mimetype='text/plain') + mock_uploader.assert_called_with(testfile, mimetype="text/plain") mock_googledrive_client.files().create.assert_called_with( - body={'name': 'testfile', - 'parents': ['testpath']}, - media_body=mock_uploader()) + body={"name": "testfile", "parents": ["testpath"]}, media_body=mock_uploader() + ) def test_mkdir(mock_googledrive_client): from portal.libs.googledrive.operations import mkdir - mkdir(mock_googledrive_client, 'googledrive', 'testid', 'testdir') + + mkdir(mock_googledrive_client, "googledrive", "testid", "testdir") mock_googledrive_client.files().create.assert_called_with( - body={ - 'name': 'testdir', - 'parents': ['testid'], - 'mimeType': 'application/vnd.google-apps.folder' - }, - fields='mimeType, name, id, modifiedTime, fileExtension, size, parents' + body={"name": "testdir", "parents": ["testid"], "mimeType": "application/vnd.google-apps.folder"}, + fields="mimeType, name, id, modifiedTime, fileExtension, size, parents", ) def test_download(mock_googledrive_client, mock_downloader): from portal.libs.googledrive.operations import download - mock_downloader().next_chunk.return_value = ('done', True) - mock_googledrive_client.files().get().execute.return_value = {'name': - 'testfile'} - downloaded = download(mock_googledrive_client, 'googledrive', 'testid') - assert downloaded.name == 'testfile' + mock_downloader().next_chunk.return_value = ("done", True) + mock_googledrive_client.files().get().execute.return_value = {"name": "testfile"} + downloaded = download(mock_googledrive_client, "googledrive", "testid") + + assert downloaded.name == "testfile" def test_copy_file(mock_googledrive_client, mocker): from portal.libs.googledrive.operations import copy - mock_transfer = mocker.patch('portal.libs.transfer.operations.transfer') - copy(mock_googledrive_client, 'googledrive', 'src_id', 'googledrive', - 'dest_id', 'testfile', filetype='file') - mock_transfer.assert_called_with(mock_googledrive_client, - mock_googledrive_client, - 'googledrive', - 'googledrive', - 'googledrive', - 'googledrive', - 'src_id', - 'dest_id') + mock_transfer = mocker.patch("portal.libs.transfer.operations.transfer") + copy(mock_googledrive_client, "googledrive", "src_id", "googledrive", "dest_id", "testfile", filetype="file") + + mock_transfer.assert_called_with( + mock_googledrive_client, + mock_googledrive_client, + "googledrive", + "googledrive", + "googledrive", + "googledrive", + "src_id", + "dest_id", + ) def test_copy_dir(mock_googledrive_client, mocker): from portal.libs.googledrive.operations import copy - mock_transfer = mocker.patch( - 'portal.libs.transfer.operations.transfer_folder') - copy(mock_googledrive_client, 'googledrive', 'src_id', 'googledrive', - 'dest_id', 'testfile', filetype='dir') - - mock_transfer.assert_called_with(mock_googledrive_client, - mock_googledrive_client, - 'googledrive', - 'googledrive', - 'googledrive', - 'googledrive', - 'src_id', - 'dest_id', - 'testfile') + + mock_transfer = mocker.patch("portal.libs.transfer.operations.transfer_folder") + copy(mock_googledrive_client, "googledrive", "src_id", "googledrive", "dest_id", "testfile", filetype="dir") + + mock_transfer.assert_called_with( + mock_googledrive_client, + mock_googledrive_client, + "googledrive", + "googledrive", + "googledrive", + "googledrive", + "src_id", + "dest_id", + "testfile", + ) def test_walk(mock_googledrive_client, mock_listing_operation): from portal.libs.googledrive.operations import walk_all + res1 = { - 'system': 'googledrive', - 'type': 'dir', - 'format': 'folder', - 'mimeType': 'application/vnd.google-apps.folder', - 'path': '1234', - 'name': 'mockdir', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }} + "system": "googledrive", + "type": "dir", + "format": "folder", + "mimeType": "application/vnd.google-apps.folder", + "path": "1234", + "name": "mockdir", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } res2 = { - 'system': 'googledrive', - 'type': 'file', - 'format': 'file', - 'mimeType': 'text/plain', - 'path': '1234', - 'name': 'mockfile', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }} - - walk_res = walk_all(mock_googledrive_client, 'googledrive', 'root') + "system": "googledrive", + "type": "file", + "format": "file", + "mimeType": "text/plain", + "path": "1234", + "name": "mockfile", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } + + walk_res = walk_all(mock_googledrive_client, "googledrive", "root") assert next(walk_res) == res1 assert next(walk_res) == res2 diff --git a/server/portal/libs/transfer/operations.py b/server/portal/libs/transfer/operations.py index cb478d5e16..7b5d03b6c8 100644 --- a/server/portal/libs/transfer/operations.py +++ b/server/portal/libs/transfer/operations.py @@ -4,48 +4,54 @@ def api_mapping(): return { - 'googledrive': { - 'upload': googledrive_operations.upload, - 'download': googledrive_operations.download, - 'iterate_listing': googledrive_operations.iterate_listing, - 'mkdir': googledrive_operations.mkdir + "googledrive": { + "upload": googledrive_operations.upload, + "download": googledrive_operations.download, + "iterate_listing": googledrive_operations.iterate_listing, + "mkdir": googledrive_operations.mkdir, + }, + "tapis": { + "upload": tapis_operations.upload, + "download": tapis_operations.download_bytes, + "iterate_listing": tapis_operations.iterate_listing, + "mkdir": tapis_operations.mkdir, }, - 'tapis': { - 'upload': tapis_operations.upload, - 'download': tapis_operations.download_bytes, - 'iterate_listing': tapis_operations.iterate_listing, - 'mkdir': tapis_operations.mkdir - } } -def transfer(src_client, dest_client, src_api, dest_api, src_system, - dest_system, src_path, dest_path, *args, **kwargs): +def transfer(src_client, dest_client, src_api, dest_api, src_system, dest_system, src_path, dest_path, *args, **kwargs): - _download = api_mapping()[src_api]['download'] - _upload = api_mapping()[dest_api]['upload'] + _download = api_mapping()[src_api]["download"] + _upload = api_mapping()[dest_api]["upload"] file_bytes = _download(src_client, src_system, src_path) file_upload = _upload(dest_client, dest_system, dest_path, file_bytes) return file_upload -def transfer_folder(src_client, dest_client, src_api, dest_api, src_system, - dest_system, src_path, dest_path, dirname, *args, - **kwargs): - _iterate_listing = api_mapping()[src_api]['iterate_listing'] - _download = api_mapping()[src_api]['download'] - _upload = api_mapping()[dest_api]['upload'] - _mkdir = api_mapping()[dest_api]['mkdir'] +def transfer_folder( + src_client, dest_client, src_api, dest_api, src_system, dest_system, src_path, dest_path, dirname, *args, **kwargs +): + _iterate_listing = api_mapping()[src_api]["iterate_listing"] + _download = api_mapping()[src_api]["download"] + _upload = api_mapping()[dest_api]["upload"] + _mkdir = api_mapping()[dest_api]["mkdir"] newdir = _mkdir(dest_client, dest_system, dest_path, dirname) for f in _iterate_listing(src_client, src_system, src_path): - if f['format'] == 'folder': - return transfer_folder(src_client, dest_client, src_api, dest_api, - src_system, dest_system, f['path'], - newdir['path'], f['name']) + if f["format"] == "folder": + return transfer_folder( + src_client, + dest_client, + src_api, + dest_api, + src_system, + dest_system, + f["path"], + newdir["path"], + f["name"], + ) else: - file_bytes = _download(src_client, src_system, f['path']) - file_upload = _upload(dest_client, dest_system, newdir['path'], - file_bytes) + file_bytes = _download(src_client, src_system, f["path"]) + file_upload = _upload(dest_client, dest_system, newdir["path"], file_bytes) return file_upload diff --git a/server/portal/libs/transfer/operations_unit_test.py b/server/portal/libs/transfer/operations_unit_test.py index f8a8fe861f..b4cd6f36b3 100644 --- a/server/portal/libs/transfer/operations_unit_test.py +++ b/server/portal/libs/transfer/operations_unit_test.py @@ -5,86 +5,77 @@ @pytest.fixture def mock_operations(mocker): - yield mocker.patch('portal.libs.transfer.operations.tapis_operations') + yield mocker.patch("portal.libs.transfer.operations.tapis_operations") @pytest.fixture def iteration_side_effect(): res1 = { - 'system': 'googledrive', - 'type': 'dir', - 'format': 'folder', - 'mimeType': 'application/vnd.google-apps.folder', - 'path': '/path/to/res1', - 'name': 'mockdir', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }} + "system": "googledrive", + "type": "dir", + "format": "folder", + "mimeType": "application/vnd.google-apps.folder", + "path": "/path/to/res1", + "name": "mockdir", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } res2 = { - 'system': 'googledrive', - 'type': 'file', - 'format': 'file', - 'mimeType': 'text/plain', - 'path': '/path/to/res2', - 'name': 'mockfile', - 'length': 0, - 'lastModified': 'mocktime', - '_links': { - 'self': {'href': 'http://webviewlink'} - }} + "system": "googledrive", + "type": "file", + "format": "file", + "mimeType": "text/plain", + "path": "/path/to/res2", + "name": "mockfile", + "length": 0, + "lastModified": "mocktime", + "_links": {"self": {"href": "http://webviewlink"}}, + } yield [[res1], [res2]] def test_transfer(mock_operations, mock_tapis_client): from portal.libs.transfer.operations import transfer + mock_bytes = MagicMock(spec=io.BytesIO) mock_operations.download_bytes.return_value = mock_bytes - transfer(mock_tapis_client, mock_tapis_client, - 'tapis', 'tapis', - 'src.system', 'dest.system', - '/src/path', '/dest/path') - - mock_operations.download_bytes.assert_called_with(mock_tapis_client, - 'src.system', - '/src/path') - mock_operations.upload.assert_called_with(mock_tapis_client, - 'dest.system', - '/dest/path', - mock_bytes) - - -def test_transfer_folder(mock_operations, mock_tapis_client, - iteration_side_effect): + transfer( + mock_tapis_client, mock_tapis_client, "tapis", "tapis", "src.system", "dest.system", "/src/path", "/dest/path" + ) + + mock_operations.download_bytes.assert_called_with(mock_tapis_client, "src.system", "/src/path") + mock_operations.upload.assert_called_with(mock_tapis_client, "dest.system", "/dest/path", mock_bytes) + + +def test_transfer_folder(mock_operations, mock_tapis_client, iteration_side_effect): from portal.libs.transfer.operations import transfer_folder mock_bytes = MagicMock(spec=io.BytesIO) mock_operations.download_bytes.return_value = mock_bytes mock_operations.iterate_listing.side_effect = iteration_side_effect - mock_operations.mkdir.return_value = {'path': '/new/dir/path'} - - transfer_folder(mock_tapis_client, mock_tapis_client, - 'tapis', 'tapis', - 'src.system', 'dest.system', - '/src/path', '/dest/path', - 'testdir') - - mock_operations.mkdir.assert_has_calls([call(mock_tapis_client, - 'dest.system', - '/dest/path', - 'testdir'), - call(mock_tapis_client, - 'dest.system', - '/new/dir/path', - 'mockdir')]) - - mock_operations.download_bytes.assert_called_with(mock_tapis_client, - 'src.system', - '/path/to/res2') - mock_operations.upload.assert_called_with(mock_tapis_client, - 'dest.system', - '/new/dir/path', - mock_bytes) + mock_operations.mkdir.return_value = {"path": "/new/dir/path"} + + transfer_folder( + mock_tapis_client, + mock_tapis_client, + "tapis", + "tapis", + "src.system", + "dest.system", + "/src/path", + "/dest/path", + "testdir", + ) + + mock_operations.mkdir.assert_has_calls( + [ + call(mock_tapis_client, "dest.system", "/dest/path", "testdir"), + call(mock_tapis_client, "dest.system", "/new/dir/path", "mockdir"), + ] + ) + + mock_operations.download_bytes.assert_called_with(mock_tapis_client, "src.system", "/path/to/res2") + mock_operations.upload.assert_called_with(mock_tapis_client, "dest.system", "/new/dir/path", mock_bytes) diff --git a/server/portal/middleware.py b/server/portal/middleware.py index c7afb269b4..eb3e2864e0 100644 --- a/server/portal/middleware.py +++ b/server/portal/middleware.py @@ -1,8 +1,7 @@ from django.contrib import messages from django.conf import settings from termsandconditions.models import TermsAndConditions -from termsandconditions.middleware import (TermsAndConditionsRedirectMiddleware, - is_path_protected) +from termsandconditions.middleware import TermsAndConditionsRedirectMiddleware, is_path_protected import logging logger = logging.getLogger(__name__) @@ -18,19 +17,20 @@ class PortalTermsMiddleware(TermsAndConditionsRedirectMiddleware): def process_request(self, request): """Process each request to app to ensure terms have been accepted""" - current_path = request.META['PATH_INFO'] + current_path = request.META["PATH_INFO"] protected_path = is_path_protected(current_path) if request.user.is_authenticated and protected_path: for term in TermsAndConditions.get_active_list(): if not TermsAndConditions.agreed_to_latest(request.user, term): - accept_url = getattr(settings, 'ACCEPT_TERMS_PATH', - '/terms/accept/') + term + accept_url = getattr(settings, "ACCEPT_TERMS_PATH", "/terms/accept/") + term messages.warning( - request, '

Please Accept the Terms of Use

' - 'You have not yet agreed to the current Terms of Use. ' - 'Please CLICK HERE to review and ' - 'accept the Terms of Use.
Acceptance of the Terms of ' - 'Use is required for continued use of the portal ' - 'resources.' % accept_url) + request, + "

Please Accept the Terms of Use

" + "You have not yet agreed to the current Terms of Use. " + 'Please CLICK HERE to review and ' + "accept the Terms of Use.
Acceptance of the Terms of " + "Use is required for continued use of the portal " + "resources." % accept_url, + ) return None diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index fbdce990e4..c607f6919e 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -23,12 +23,12 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if os.path.isfile(os.path.join(BASE_DIR, 'settings', 'settings_custom.py')): +if os.path.isfile(os.path.join(BASE_DIR, "settings", "settings_custom.py")): from portal.settings import settings_custom else: from portal.settings import settings_default as settings_custom -if os.path.isfile(os.path.join(BASE_DIR, 'settings', 'settings_forms.py')): +if os.path.isfile(os.path.join(BASE_DIR, "settings", "settings_forms.py")): from portal.settings import settings_forms else: settings_forms = None @@ -37,185 +37,169 @@ DEBUG = settings_custom._DEBUG FIXTURE_DIRS = [ - os.path.join(BASE_DIR, 'fixtures'), + os.path.join(BASE_DIR, "fixtures"), ] # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = settings_secret._SECRET_KEY # SECURITY WARNING: don't run with debug turned on in production! # Cookie name. this can be whatever you want -SESSION_COOKIE_NAME = 'coresessionid' # use the sessionid in your views code +SESSION_COOKIE_NAME = "coresessionid" # use the sessionid in your views code # the module to store sessions data -SESSION_ENGINE = 'django.contrib.sessions.backends.db' +SESSION_ENGINE = "django.contrib.sessions.backends.db" # age of cookie in seconds (default: 2 weeks) -SESSION_COOKIE_AGE = 24*60*60*7 # the number of seconds for only 7 for example +SESSION_COOKIE_AGE = 24 * 60 * 60 * 7 # the number of seconds for only 7 for example # whether a user's session cookie expires when the web browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = False # whether the session cookie should be secure (https:// only) SESSION_COOKIE_SECURE = True # whether the csrf token cookie should be secure (https:// only) CSRF_COOKIE_SECURE = True -CSRF_COOKIE_NAME = 'csrfcookie' +CSRF_COOKIE_NAME = "csrfcookie" # -CSRF_COOKIE_SAMESITE = 'Strict' +CSRF_COOKIE_SAMESITE = "Strict" # for local testing -CSRF_TRUSTED_ORIGINS = getattr(settings_custom, '_CSRF_TRUSTED_ORIGINS', []) +CSRF_TRUSTED_ORIGINS = getattr(settings_custom, "_CSRF_TRUSTED_ORIGINS", []) -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -ALLOWED_HOSTS = ['*'] +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +ALLOWED_HOSTS = ["*"] # https://docs.djangoproject.com/en/3.2/releases/3.0/#security -X_FRAME_OPTIONS = 'SAMEORIGIN' +X_FRAME_OPTIONS = "SAMEORIGIN" # https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Custom Portal Template Assets PORTAL_ICON_FILENAME = settings_custom._PORTAL_ICON_FILENAME -PORTAL_CSS_FILENAMES = getattr(settings_custom, '_PORTAL_CSS_FILENAMES', []) +PORTAL_CSS_FILENAMES = getattr(settings_custom, "_PORTAL_CSS_FILENAMES", []) -ROOT_URLCONF = 'portal.urls' +ROOT_URLCONF = "portal.urls" # Application definition INSTALLED_APPS = [ - # Django Channels - 'channels', - 'daphne', - + "channels", + "daphne", # Core Django. - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - - 'django.contrib.sitemaps', - 'django.contrib.sessions.middleware', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sitemaps", + "django.contrib.sessions.middleware", # Pipeline. - 'termsandconditions', - 'impersonate', - + "termsandconditions", + "impersonate", # Custom apps. - 'portal.apps.accounts', - 'portal.apps.auth', - 'portal.apps.tickets', - 'portal.apps.licenses', - 'portal.apps.notifications', - 'portal.apps.news', - 'portal.apps.onboarding', - 'portal.apps.search', - 'portal.apps.signals', - 'portal.apps.webhooks', - 'portal.apps.workbench', - 'portal.apps.workspace', - 'portal.apps.datafiles', - 'portal.apps.system_monitor', - 'portal.apps.googledrive_integration', - 'portal.apps.projects', - 'portal.apps.public_data', - 'portal.apps.request_access', - 'portal.apps.site_search', - 'portal.apps.jupyter_mounts', - 'portal.apps.portal_messages', - 'portal.apps.publications', + "portal.apps.accounts", + "portal.apps.auth", + "portal.apps.tickets", + "portal.apps.licenses", + "portal.apps.notifications", + "portal.apps.news", + "portal.apps.onboarding", + "portal.apps.search", + "portal.apps.signals", + "portal.apps.webhooks", + "portal.apps.workbench", + "portal.apps.workspace", + "portal.apps.datafiles", + "portal.apps.system_monitor", + "portal.apps.googledrive_integration", + "portal.apps.projects", + "portal.apps.public_data", + "portal.apps.request_access", + "portal.apps.site_search", + "portal.apps.jupyter_mounts", + "portal.apps.portal_messages", + "portal.apps.publications", ] MIDDLEWARE = [ # Django core middleware. - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'portal.apps.auth.middleware.TapisTokenRefreshMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'impersonate.middleware.ImpersonateMiddleware', # must be AFTER django.contrib.auth - + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "portal.apps.auth.middleware.TapisTokenRefreshMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "impersonate.middleware.ImpersonateMiddleware", # must be AFTER django.contrib.auth # Throws an Error. # 'portal.middleware.PortalTermsMiddleware', ] TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates'), - os.path.join(BASE_DIR, '../../client/dist')], + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates"), os.path.join(BASE_DIR, "../../client/dist")], # 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ + "OPTIONS": { + "context_processors": [ # Django core processors - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", # what are these for? - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.csrf', # Needed? - 'django.template.context_processors.tz', - 'django.template.context_processors.static', - 'django_settings_export.settings_export', - 'portal.utils.contextprocessors.analytics', - 'portal.utils.contextprocessors.debug', - 'portal.utils.contextprocessors.messages', - + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.csrf", # Needed? + "django.template.context_processors.tz", + "django.template.context_processors.static", + "django_settings_export.settings_export", + "portal.utils.contextprocessors.analytics", + "portal.utils.contextprocessors.debug", + "portal.utils.contextprocessors.messages", ], - 'loaders': [ - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', + "loaders": [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", ], }, }, ] -WSGI_APPLICATION = 'portal.wsgi.application' +WSGI_APPLICATION = "portal.wsgi.application" -AUTHENTICATION_BACKENDS = ['portal.apps.auth.backends.TapisOAuthBackend', - 'django.contrib.auth.backends.ModelBackend'] +AUTHENTICATION_BACKENDS = ["portal.apps.auth.backends.TapisOAuthBackend", "django.contrib.auth.backends.ModelBackend"] # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': ('django.contrib.auth.password_validation.' - 'UserAttributeSimilarityValidator'), + "NAME": ("django.contrib.auth.password_validation.UserAttributeSimilarityValidator"), }, { - 'NAME': ('django.contrib.auth.password_validation.' - 'MinimumLengthValidator'), + "NAME": ("django.contrib.auth.password_validation.MinimumLengthValidator"), }, { - 'NAME': ('django.contrib.auth.password_validation.' - 'CommonPasswordValidator'), + "NAME": ("django.contrib.auth.password_validation.CommonPasswordValidator"), }, { - 'NAME': ('django.contrib.auth.password_validation.' - 'NumericPasswordValidator'), + "NAME": ("django.contrib.auth.password_validation.NumericPasswordValidator"), }, ] -IMPERSONATE = { - 'REQUIRE_SUPERUSER': True -} +IMPERSONATE = {"REQUIRE_SUPERUSER": True} # this can be set to just '/' if we're not using core portal to create cms sessions -LOGOUT_REDIRECT_URL = getattr(settings_custom, '_LOGOUT_REDIRECT_URL', '/') -LOGIN_REDIRECT_URL = getattr(settings_custom, '_LOGIN_REDIRECT_URL', '/') -LOGIN_URL = '/auth/tapis/' +LOGOUT_REDIRECT_URL = getattr(settings_custom, "_LOGOUT_REDIRECT_URL", "/") +LOGIN_REDIRECT_URL = getattr(settings_custom, "_LOGIN_REDIRECT_URL", "/") +LOGIN_URL = "/auth/tapis/" # Internationalization # https://docs.djangoproject.com/en/2.2/topics/i18n/ -LANGUAGE_CODE = 'en' -TIME_ZONE = 'UTC' +LANGUAGE_CODE = "en" +TIME_ZONE = "UTC" USE_I18N = True USE_TZ = True @@ -225,31 +209,31 @@ LANGUAGES = ( # Customize this - ('en', 'English'), + ("en", "English"), ) # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.2/howto/static-files/ -STATIC_URL = '/core/static/' -MEDIA_URL = '/core/media/' +STATIC_URL = "/core/static/" +MEDIA_URL = "/core/media/" -STATIC_ROOT = os.path.join(BASE_DIR, '../static') -MEDIA_ROOT = os.path.join(BASE_DIR, '../media') +STATIC_ROOT = os.path.join(BASE_DIR, "../static") +MEDIA_ROOT = os.path.join(BASE_DIR, "../media") STATICFILES_DIRS = [ - os.path.join(BASE_DIR, '../../client/dist'), + os.path.join(BASE_DIR, "../../client/dist"), # Serve fonts using the cep.dev hostname in debug mode - ('src/fonts', os.path.join(BASE_DIR, '../../client/src/fonts')) + ("src/fonts", os.path.join(BASE_DIR, "../../client/src/fonts")), ] STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] FIXTURE_DIRS = [ - os.path.join(BASE_DIR, 'fixtures'), + os.path.join(BASE_DIR, "fixtures"), ] """ @@ -260,13 +244,13 @@ # https://docs.djangoproject.com/en/2.2/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': settings_secret._DJANGO_DB_ENGINE, - 'NAME': settings_secret._DJANGO_DB_NAME, - 'USER': settings_secret._DJANGO_DB_USER, - 'PASSWORD': settings_secret._DJANGO_DB_PASSWORD, - 'HOST': settings_secret._DJANGO_DB_HOST, - 'PORT': settings_secret._DJANGO_DB_PORT + "default": { + "ENGINE": settings_secret._DJANGO_DB_ENGINE, + "NAME": settings_secret._DJANGO_DB_NAME, + "USER": settings_secret._DJANGO_DB_USER, + "PASSWORD": settings_secret._DJANGO_DB_PASSWORD, + "HOST": settings_secret._DJANGO_DB_HOST, + "PORT": settings_secret._DJANGO_DB_PORT, } } @@ -274,13 +258,13 @@ # https://docs.djangoproject.com/en/3.2/topics/cache/ CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', + "default": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "LOCATION": "memcached:11211", } } -WEBSOCKET_URL = '/ws/' +WEBSOCKET_URL = "/ws/" # TAS Authentication. TAS_URL = settings_secret._TAS_URL @@ -292,7 +276,7 @@ RT_UN = settings_secret._RT_UN RT_PW = settings_secret._RT_PW RT_QUEUE = settings_custom._RT_QUEUE -RT_TAG = getattr(settings_custom, '_RT_TAG', "") +RT_TAG = getattr(settings_custom, "_RT_TAG", "") # Google Analytics. @@ -316,79 +300,74 @@ def portal_filter(record): LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, + "version": 1, + "disable_existing_loggers": False, "filters": { "portalFilter": { "()": "django.utils.log.CallbackFilter", "callback": portal_filter, }, }, - 'formatters': { - 'default': { - 'format': '[DJANGO] %(levelname)s %(asctime)s UTC %(module)s ' - '%(name)s.%(funcName)s:%(lineno)s: %(message)s' + "formatters": { + "default": { + "format": "[DJANGO] %(levelname)s %(asctime)s UTC %(module)s %(name)s.%(funcName)s:%(lineno)s: %(message)s" }, - 'tapis': { - 'format': '[TAPIS] %(levelname)s %(asctime)s UTC %(module)s ' - '%(name)s.%(funcName)s:%(lineno)s: %(message)s' + "tapis": { + "format": "[TAPIS] %(levelname)s %(asctime)s UTC %(module)s %(name)s.%(funcName)s:%(lineno)s: %(message)s" }, - 'metrics': { - 'format': '[METRICS] %(levelname)s %(module)s %(name)s.%(funcName)s:%(lineno)s:' - ' %(message)s user=%(user)s ip=%(ip)s agent=%(agent)s sessionId=%(sessionId)s op=%(operation)s' - ' info=%(info)s timestamp=%(asctime)s trackingId=portals.%(sessionId)s guid=%(logGuid)s portal=%(portal)s tenant=%(tenant)s' + "metrics": { + "format": "[METRICS] %(levelname)s %(module)s %(name)s.%(funcName)s:%(lineno)s:" + " %(message)s user=%(user)s ip=%(ip)s agent=%(agent)s sessionId=%(sessionId)s op=%(operation)s" + " info=%(info)s timestamp=%(asctime)s trackingId=portals.%(sessionId)s guid=%(logGuid)s portal=%(portal)s tenant=%(tenant)s" }, }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'default', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "default", }, - 'file': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': '/var/log/portal/portal.log', - 'maxBytes': 1024*1024*5, # 5 MB - 'backupCount': 5, - 'formatter': 'default', + "file": { + "level": "DEBUG", + "class": "logging.handlers.RotatingFileHandler", + "filename": "/var/log/portal/portal.log", + "maxBytes": 1024 * 1024 * 5, # 5 MB + "backupCount": 5, + "formatter": "default", }, - 'metrics': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'metrics', - 'filters': ['portalFilter'] + "metrics": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "metrics", + "filters": ["portalFilter"], }, }, - 'loggers': { - 'django': { - 'handlers': ['console', 'file'], - 'level': 'INFO', - 'propagate': True, + "loggers": { + "django": { + "handlers": ["console", "file"], + "level": "INFO", + "propagate": True, }, - 'portal': { - 'handlers': ['console', 'file'], - 'level': 'DEBUG', + "portal": { + "handlers": ["console", "file"], + "level": "DEBUG", }, - 'metrics': { - 'handlers': ['metrics'], - 'filters': ['portalFilter'], - 'level': 'INFO', + "metrics": { + "handlers": ["metrics"], + "filters": ["portalFilter"], + "level": "INFO", }, - 'paramiko': { - 'handlers': ['console'], - 'level': 'DEBUG' + "paramiko": {"handlers": ["console"], "level": "DEBUG"}, + "celery": { + "handlers": ["console", "file"], + "level": "INFO", }, - 'celery': { - 'handlers': ['console', 'file'], - 'level': 'INFO', - }, - 'daphne': { - 'handlers': [ - 'console', + "daphne": { + "handlers": [ + "console", ], - 'level': 'INFO' - } + "level": "INFO", + }, }, } @@ -396,7 +375,7 @@ def portal_filter(record): SETTINGS: TACC """ -IS_TACC_PORTAL = getattr(settings_custom, '_IS_TACC_PORTAL', True) +IS_TACC_PORTAL = getattr(settings_custom, "_IS_TACC_PORTAL", True) """ SETTINGS: TAPIS @@ -410,289 +389,216 @@ def portal_filter(record): TAPIS_CLIENT_KEY = settings_secret._TAPIS_CLIENT_KEY # Long-live portal admin access token -TAPIS_ADMIN_JWT = getattr(settings_secret, '_TAPIS_ADMIN_JWT', '') +TAPIS_ADMIN_JWT = getattr(settings_secret, "_TAPIS_ADMIN_JWT", "") PORTAL_ADMIN_USERNAME = settings_secret._PORTAL_ADMIN_USERNAME AGAVE_JWT_PUBKEY = ( - 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCUp/oV1vWc8/TkQSiAvTousMzO\n' - 'M4asB2iltr2QKozni5aVFu818MpOLZIr8LMnTzWllJvvaA5RAAdpbECb+48FjbBe\n' - '0hseUdN5HpwvnH/DW8ZccGvk53I6Orq7hLCv1ZHtuOCokghz/ATrhyPq+QktMfXn\n' - 'RS4HrKGJTzxaCcU7OQIDAQAB' + "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCUp/oV1vWc8/TkQSiAvTousMzO\n" + "M4asB2iltr2QKozni5aVFu818MpOLZIr8LMnTzWllJvvaA5RAAdpbECb+48FjbBe\n" + "0hseUdN5HpwvnH/DW8ZccGvk53I6Orq7hLCv1ZHtuOCokghz/ATrhyPq+QktMfXn\n" + "RS4HrKGJTzxaCcU7OQIDAQAB" ) AGAVE_JWT_HEADER = settings_custom._AGAVE_JWT_HEADER -AGAVE_JWT_ISSUER = 'wso2.org/products/am' -AGAVE_JWT_USER_CLAIM_FIELD = 'http://wso2.org/claims/fullname' +AGAVE_JWT_ISSUER = "wso2.org/products/am" +AGAVE_JWT_USER_CLAIM_FIELD = "http://wso2.org/claims/fullname" """ SETTINGS: CELERY """ -_BROKER_URL_PROTOCOL = 'amqp://' +_BROKER_URL_PROTOCOL = "amqp://" _BROKER_URL_USERNAME = settings_secret._BROKER_URL_USERNAME _BROKER_URL_PWD = settings_secret._BROKER_URL_PWD _BROKER_URL_HOST = settings_secret._BROKER_URL_HOST _BROKER_URL_PORT = settings_secret._BROKER_URL_PORT _BROKER_URL_VHOST = settings_secret._BROKER_URL_VHOST -CELERY_BROKER_URL = ''.join( +CELERY_BROKER_URL = "".join( [ - _BROKER_URL_PROTOCOL, _BROKER_URL_USERNAME, ':', - _BROKER_URL_PWD, '@', _BROKER_URL_HOST, ':', - _BROKER_URL_PORT, '/', _BROKER_URL_VHOST + _BROKER_URL_PROTOCOL, + _BROKER_URL_USERNAME, + ":", + _BROKER_URL_PWD, + "@", + _BROKER_URL_HOST, + ":", + _BROKER_URL_PORT, + "/", + _BROKER_URL_VHOST, ] ) -_RESULT_BACKEND_PROTOCOL = 'redis://' +_RESULT_BACKEND_PROTOCOL = "redis://" _RESULT_BACKEND_HOST = settings_secret._RESULT_BACKEND_HOST _RESULT_BACKEND_PORT = settings_secret._RESULT_BACKEND_PORT _RESULT_BACKEND_DB = settings_secret._RESULT_BACKEND_DB -CELERY_RESULT_BACKEND = ''.join( - [ - _RESULT_BACKEND_PROTOCOL, - _RESULT_BACKEND_HOST, ':', _RESULT_BACKEND_PORT, - '/', _RESULT_BACKEND_DB - ] +CELERY_RESULT_BACKEND = "".join( + [_RESULT_BACKEND_PROTOCOL, _RESULT_BACKEND_HOST, ":", _RESULT_BACKEND_PORT, "/", _RESULT_BACKEND_DB] ) -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_SERIALIZER = 'json' +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" CELERYD_HIJACK_ROOT_LOGGER = False -CELERYD_LOG_FORMAT = ('[DJANGO] $(processName)s %(levelname)s %(asctime)s ' - '%(module)s %(name)s.%(funcName)s:%(lineno)s: ' - '%(message)s') +CELERYD_LOG_FORMAT = ( + "[DJANGO] $(processName)s %(levelname)s %(asctime)s %(module)s %(name)s.%(funcName)s:%(lineno)s: %(message)s" +) -CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' +CELERY_DEFAULT_EXCHANGE_TYPE = "direct" CELERY_QUEUES = ( - Queue( - 'default', - Exchange('default'), - routing_key='default', - queue_arguments={ - 'x-max-priority': 10 - } - ), + Queue("default", Exchange("default"), routing_key="default", queue_arguments={"x-max-priority": 10}), # Use to queue indexing tasks - Queue( - 'indexing', - Exchange('indexing'), - routing_key='indexing', - queue_arguments={ - 'x-max-priority': 10 - } - ), + Queue("indexing", Exchange("indexing"), routing_key="indexing", queue_arguments={"x-max-priority": 10}), # Use to queue tasks which handle files - Queue( - 'files', - Exchange('files'), - routing_key='files', - queue_arguments={ - 'x-max-priority': 10 - } - ), + Queue("files", Exchange("files"), routing_key="files", queue_arguments={"x-max-priority": 10}), # Use to queue tasks which mainly call external APIs - Queue( - 'api', - Exchange('api'), - routing_key='api', - queue_arguments={ - 'x-max-priority': 10 - } - ), + Queue("api", Exchange("api"), routing_key="api", queue_arguments={"x-max-priority": 10}), # Use to queue tasks handling onboarding - Queue( - 'onboard', - Exchange('onboard'), - routing_key='onboard', - queue_arguments={ - 'x-max-priority': 10 - } - ), + Queue("onboard", Exchange("onboard"), routing_key="onboard", queue_arguments={"x-max-priority": 10}), ) -CELERY_TASK_DEFAULT_QUEUE = 'default' -CELERY_TASK_DEFAULT_EXCHANGE = 'default' -CELERY_TASK_DEFAULT_ROUTING_KEY = 'default' +CELERY_TASK_DEFAULT_QUEUE = "default" +CELERY_TASK_DEFAULT_EXCHANGE = "default" +CELERY_TASK_DEFAULT_ROUTING_KEY = "default" """ SETTINGS: TACC EXECUTION SYSTEMS. """ TACC_EXEC_SYSTEMS = { - 'corral': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/work2/{}', - 'home_dir': '/home/{}' - }, - 'stampede2': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, - 'stampede3': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, - 'frontera': { - 'work_dir': '/work2/{}', - 'scratch_dir': 'HOST_EVAL(SCRATCH)', - 'home_dir': '/home1/{}' - }, - 'ls6': { - 'work_dir': '/work/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, - 'vista': { - 'work_dir': '/work/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, + "corral": {"work_dir": "/work2/{}", "scratch_dir": "/work2/{}", "home_dir": "/home/{}"}, + "stampede2": {"work_dir": "/work2/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, + "stampede3": {"work_dir": "/work2/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, + "frontera": {"work_dir": "/work2/{}", "scratch_dir": "HOST_EVAL(SCRATCH)", "home_dir": "/home1/{}"}, + "ls6": {"work_dir": "/work/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, + "vista": {"work_dir": "/work/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, } """ SETTINGS: DATA DEPOT """ -PORTAL_DATAFILES_STORAGE_SYSTEMS = getattr( - settings_custom, '_PORTAL_DATAFILES_STORAGE_SYSTEMS', [] +PORTAL_DATAFILES_STORAGE_SYSTEMS = getattr(settings_custom, "_PORTAL_DATAFILES_STORAGE_SYSTEMS", []) +PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next( + (sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys.get("default")), None ) -PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next((sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys.get('default')), None) PORTAL_SEARCH_MANAGERS = { - 'my-data': 'portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager', - 'shared': 'portal.apps.search.api.managers.shared_search.SharedSearchManager', - 'cms': 'portal.apps.search.api.managers.cms_search.CMSSearchManager', - 'my-projects': 'portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager', - 'public': 'portal.apps.search.api.managers.public_search.PublicSearchManager' + "my-data": "portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager", + "shared": "portal.apps.search.api.managers.shared_search.SharedSearchManager", + "cms": "portal.apps.search.api.managers.cms_search.CMSSearchManager", + "my-projects": "portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager", + "public": "portal.apps.search.api.managers.public_search.PublicSearchManager", } PORTAL_DATA_DEPOT_PAGE_SIZE = 100 -FORMS = getattr(settings_forms, '_FORMS', {}) +FORMS = getattr(settings_forms, "_FORMS", {}) """ SETTINGS: EXTERNAL DATA RESOURCES """ -EXTERNAL_RESOURCE_SECRETS = getattr(settings_secret, '_EXTERNAL_RESOURCE_SECRETS', {}) +EXTERNAL_RESOURCE_SECRETS = getattr(settings_secret, "_EXTERNAL_RESOURCE_SECRETS", {}) PORTAL_WORKSPACE_MANAGERS = { - 'private': 'portal.apps.workspace.managers.private.FileManager', - 'shared': 'portal.apps.workspace.managers.shared.FileManager', + "private": "portal.apps.workspace.managers.private.FileManager", + "shared": "portal.apps.workspace.managers.shared.FileManager", } PORTAL_WORKSPACE_PAGE_SIZE = 100 -TAPIS_DEFAULT_TRASH_NAME = getattr(settings_custom, '_TAPIS_DEFAULT_TRASH_NAME', '.Trash') +TAPIS_DEFAULT_TRASH_NAME = getattr(settings_custom, "_TAPIS_DEFAULT_TRASH_NAME", ".Trash") -PORTAL_PROJECTS_SYSTEM_PREFIX = settings_custom.\ - _PORTAL_PROJECTS_SYSTEM_PREFIX +PORTAL_PROJECTS_SYSTEM_PREFIX = settings_custom._PORTAL_PROJECTS_SYSTEM_PREFIX -PORTAL_PROJECTS_ID_PREFIX = settings_custom.\ - _PORTAL_PROJECTS_ID_PREFIX +PORTAL_PROJECTS_ID_PREFIX = settings_custom._PORTAL_PROJECTS_ID_PREFIX -PORTAL_PROJECTS_ROOT_DIR = settings_custom.\ - _PORTAL_PROJECTS_ROOT_DIR +PORTAL_PROJECTS_ROOT_DIR = settings_custom._PORTAL_PROJECTS_ROOT_DIR -PORTAL_PROJECTS_ROOT_SYSTEM_NAME = settings_custom.\ - _PORTAL_PROJECTS_ROOT_SYSTEM_NAME +PORTAL_PROJECTS_ROOT_SYSTEM_NAME = settings_custom._PORTAL_PROJECTS_ROOT_SYSTEM_NAME -PORTAL_PROJECTS_ROOT_HOST = settings_custom.\ - _PORTAL_PROJECTS_ROOT_HOST +PORTAL_PROJECTS_ROOT_HOST = settings_custom._PORTAL_PROJECTS_ROOT_HOST -PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX = getattr( - settings_custom, '_PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX', None) +PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX = getattr(settings_custom, "_PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX", None) -PORTAL_PROJECTS_REVIEW_ROOT_DIR = getattr( - settings_custom, '_PORTAL_PROJECTS_REVIEW_ROOT_DIR', None) +PORTAL_PROJECTS_REVIEW_ROOT_DIR = getattr(settings_custom, "_PORTAL_PROJECTS_REVIEW_ROOT_DIR", None) -PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = getattr( - settings_custom, '_PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME', None) +PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = getattr(settings_custom, "_PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME", None) -PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = getattr( - settings_custom, '_PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX', None) +PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = getattr(settings_custom, "_PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX", None) -PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = getattr( - settings_custom, '_PORTAL_PROJECTS_PUBLISHED_ROOT_DIR', None) +PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = getattr(settings_custom, "_PORTAL_PROJECTS_PUBLISHED_ROOT_DIR", None) PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = getattr( - settings_custom, '_PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME', None) + settings_custom, "_PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME", None +) -PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = getattr( - settings_custom, '_PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME', None) +PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = getattr(settings_custom, "_PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME", None) -PROJECT_ADMIN_GROUP = getattr(settings_custom, '_PROJECT_ADMIN_GROUP', 'Project Admin') +PROJECT_ADMIN_GROUP = getattr(settings_custom, "_PROJECT_ADMIN_GROUP", "Project Admin") -PORTAL_PUBLICATION_DATACITE_SHOULDER = getattr( - settings_custom, '_PORTAL_PUBLICATION_DATACITE_SHOULDER', None) +PORTAL_PUBLICATION_DATACITE_SHOULDER = getattr(settings_custom, "_PORTAL_PUBLICATION_DATACITE_SHOULDER", None) -PORTAL_PUBLICATION_DATACITE_URL_PREFIX = getattr( - settings_custom, '_PORTAL_PUBLICATION_DATACITE_URL_PREFIX', None) +PORTAL_PUBLICATION_DATACITE_URL_PREFIX = getattr(settings_custom, "_PORTAL_PUBLICATION_DATACITE_URL_PREFIX", None) -DATACITE_URL = getattr( - settings_custom, '_DATACITE_URL', None) +DATACITE_URL = getattr(settings_custom, "_DATACITE_URL", None) -DATACITE_USER = getattr( - settings_secret, '_DATACITE_USER', None) +DATACITE_USER = getattr(settings_secret, "_DATACITE_USER", None) -DATACITE_PASS = getattr( - settings_secret, '_DATACITE_PASS', None) +DATACITE_PASS = getattr(settings_secret, "_DATACITE_PASS", None) -PORTAL_PROJECTS_PRIVATE_KEY = settings_secret.\ - _PORTAL_PROJECTS_PRIVATE_KEY +PORTAL_PROJECTS_PRIVATE_KEY = settings_secret._PORTAL_PROJECTS_PRIVATE_KEY -PORTAL_PROJECTS_PUBLIC_KEY = settings_secret.\ - _PORTAL_PROJECTS_PUBLIC_KEY +PORTAL_PROJECTS_PUBLIC_KEY = settings_secret._PORTAL_PROJECTS_PUBLIC_KEY -COMMUNITY_INDEX_SCHEDULE = settings_custom.\ - _COMMUNITY_INDEX_SCHEDULE +COMMUNITY_INDEX_SCHEDULE = settings_custom._COMMUNITY_INDEX_SCHEDULE -PORTAL_PROJECTS_PEMS_APP_ID = settings_custom.\ - _PORTAL_PROJECTS_PEMS_APP_ID +PORTAL_PROJECTS_PEMS_APP_ID = settings_custom._PORTAL_PROJECTS_PEMS_APP_ID -PORTAL_KEYS_MANAGER = settings_custom.\ - _PORTAL_KEYS_MANAGER +PORTAL_KEYS_MANAGER = settings_custom._PORTAL_KEYS_MANAGER -PORTAL_USER_ACCOUNT_SETUP_STEPS = getattr(settings_custom, '_PORTAL_USER_ACCOUNT_SETUP_STEPS', []) +PORTAL_USER_ACCOUNT_SETUP_STEPS = getattr(settings_custom, "_PORTAL_USER_ACCOUNT_SETUP_STEPS", []) -PORTAL_NAMESPACE = settings_custom.\ - _PORTAL_NAMESPACE +PORTAL_NAMESPACE = settings_custom._PORTAL_NAMESPACE -PORTAL_PROJECTS_SYSTEM_PORT = getattr(settings_custom, '_PORTAL_PROJECTS_SYSTEM_PORT', 22) +PORTAL_PROJECTS_SYSTEM_PORT = getattr(settings_custom, "_PORTAL_PROJECTS_SYSTEM_PORT", 22) PORTAL_APPS_NAMES_SEARCH = settings_custom._PORTAL_APPS_NAMES_SEARCH -PORTAL_APPS_DEFAULT_TAB = getattr(settings_custom, '_PORTAL_APPS_DEFAULT_TAB', '') +PORTAL_APPS_DEFAULT_TAB = getattr(settings_custom, "_PORTAL_APPS_DEFAULT_TAB", "") -PORTAL_PUBLICATION_PUBLISHER = getattr( - settings_custom, '_PORTAL_PUBLICATION_PUBLISHER', PORTAL_NAMESPACE) +PORTAL_PUBLICATION_PUBLISHER = getattr(settings_custom, "_PORTAL_PUBLICATION_PUBLISHER", PORTAL_NAMESPACE) -PORTAL_PUBLICATION_ARCHIVE_APP_ID = getattr( - settings_custom, '_PORTAL_PUBLICATION_ARCHIVE_APP_ID', None) +PORTAL_PUBLICATION_ARCHIVE_APP_ID = getattr(settings_custom, "_PORTAL_PUBLICATION_ARCHIVE_APP_ID", None) -PORTAL_PUBLICATION_ARCHIVE_APP_VERSION = getattr( - settings_custom, '_PORTAL_PUBLICATION_ARCHIVE_APP_VERSION', None) +PORTAL_PUBLICATION_ARCHIVE_APP_VERSION = getattr(settings_custom, "_PORTAL_PUBLICATION_ARCHIVE_APP_VERSION", None) -PORTAL_PUBLICATION_RANCH_SYSTEM_ID = getattr( - settings_custom, '_PORTAL_PUBLICATION_RANCH_SYSTEM_ID', None) +PORTAL_PUBLICATION_RANCH_SYSTEM_ID = getattr(settings_custom, "_PORTAL_PUBLICATION_RANCH_SYSTEM_ID", None) -ALLOCATIONS_TO_EXCLUDE = ( - getattr(settings_custom, "_ALLOCATIONS_TO_EXCLUDE", ["DesignSafe-DCV", "DesignSafe-Corral"]) -) +ALLOCATIONS_TO_EXCLUDE = getattr(settings_custom, "_ALLOCATIONS_TO_EXCLUDE", ["DesignSafe-DCV", "DesignSafe-Corral"]) -PORTAL_JOB_NOTIFICATION_STATES = ["PENDING", "STAGING_INPUTS", "RUNNING", "ARCHIVING", "BLOCKED", "PAUSED", "FINISHED", "CANCELLED", "FAILED"] +PORTAL_JOB_NOTIFICATION_STATES = [ + "PENDING", + "STAGING_INPUTS", + "RUNNING", + "ARCHIVING", + "BLOCKED", + "PAUSED", + "FINISHED", + "CANCELLED", + "FAILED", +] -NGROK_DOMAIN = os.environ.get('NGROK_DOMAIN', '') +NGROK_DOMAIN = os.environ.get("NGROK_DOMAIN", "") -PORTAL_ALLOCATION = getattr(settings_custom, '_PORTAL_ALLOCATION', '') +PORTAL_ALLOCATION = getattr(settings_custom, "_PORTAL_ALLOCATION", "") -PORTAL_PROJECTS_USE_SET_FACL_JOB = getattr(settings_custom, '_PORTAL_PROJECTS_USE_SET_FACL_JOB', True) +PORTAL_PROJECTS_USE_SET_FACL_JOB = getattr(settings_custom, "_PORTAL_PROJECTS_USE_SET_FACL_JOB", True) # When True, project creation builds the metadata graph and file listings -PORTAL_PROJECTS_ENABLE_METADATA = getattr(settings_custom, '_PORTAL_PROJECTS_ENABLE_METADATA', False) +PORTAL_PROJECTS_ENABLE_METADATA = getattr(settings_custom, "_PORTAL_PROJECTS_ENABLE_METADATA", False) # Vanity URL for the portal. Backwards compatibility with old _WH_BASE_URL setting. # Also include support for NGINX_SERVER_NAME environment variable if no settings are set. @@ -702,11 +608,7 @@ def portal_filter(record): getattr( settings_custom, "_WH_BASE_URL", - ( - f"https://{os.environ.get('NGINX_SERVER_NAME')}" - if os.environ.get("NGINX_SERVER_NAME") - else "" - ), + (f"https://{os.environ.get('NGINX_SERVER_NAME')}" if os.environ.get("NGINX_SERVER_NAME") else ""), ), ) @@ -720,37 +622,39 @@ def portal_filter(record): ES_INDEX_PREFIX = settings_secret._ES_INDEX_PREFIX HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': ES_HOSTS, - 'INDEX_NAME': ES_INDEX_PREFIX.format('cms'), - 'KWARGS': {'http_auth': ES_AUTH} + "default": { + "ENGINE": "haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine", + "URL": ES_HOSTS, + "INDEX_NAME": ES_INDEX_PREFIX.format("cms"), + "KWARGS": {"http_auth": ES_AUTH}, } } -HAYSTACK_ROUTERS = ['aldryn_search.router.LanguageRouter', ] +HAYSTACK_ROUTERS = [ + "aldryn_search.router.LanguageRouter", +] -ALDRYN_SEARCH_DEFAULT_LANGUAGE = 'en' +ALDRYN_SEARCH_DEFAULT_LANGUAGE = "en" ALDRYN_SEARCH_REGISTER_APPHOOK = True -SYSTEM_MONITOR_DISPLAY_LIST = getattr(settings_custom, '_SYSTEM_MONITOR_DISPLAY_LIST', []) +SYSTEM_MONITOR_DISPLAY_LIST = getattr(settings_custom, "_SYSTEM_MONITOR_DISPLAY_LIST", []) -SYSTEM_MONITOR_URL = getattr(settings_custom, '_SYSTEM_MONITOR_URL', 'https://tap.tacc.utexas.edu/status/') +SYSTEM_MONITOR_URL = getattr(settings_custom, "_SYSTEM_MONITOR_URL", "https://tap.tacc.utexas.edu/status/") -DOCS_CHATBOT_URL = getattr(settings_custom, '_DOCS_CHATBOT_URL', None) +DOCS_CHATBOT_URL = getattr(settings_custom, "_DOCS_CHATBOT_URL", None) """ SETTINGS: EXPORTS """ SETTINGS_EXPORT = [ - 'PORTAL_ICON_FILENAME', - 'PORTAL_CSS_FILENAMES', - 'DEBUG', - 'GOOGLE_ANALYTICS_PROPERTY_ID', - 'PORTAL_NAMESPACE', - 'WORKBENCH_SETTINGS', - 'DOCS_CHATBOT_URL', - 'PORTAL_USER_ACCOUNT_SETUP_STEPS', + "PORTAL_ICON_FILENAME", + "PORTAL_CSS_FILENAMES", + "DEBUG", + "GOOGLE_ANALYTICS_PROPERTY_ID", + "PORTAL_NAMESPACE", + "WORKBENCH_SETTINGS", + "DOCS_CHATBOT_URL", + "PORTAL_USER_ACCOUNT_SETUP_STEPS", ] """ @@ -758,72 +662,130 @@ def portal_filter(record): """ SUPPORTED_MS_WORD = [ - '.doc', '.dot', '.docx', '.docm', '.dotx', '.dotm', '.docb', + ".doc", + ".dot", + ".docx", + ".docm", + ".dotx", + ".dotm", + ".docb", ] SUPPORTED_MS_EXCEL = [ - '.xls', '.xlt', '.xlm', '.xlsx', '.xlsm', '.xltx', '.xltm', + ".xls", + ".xlt", + ".xlm", + ".xlsx", + ".xlsm", + ".xltx", + ".xltm", ] SUPPORTED_MS_POWERPOINT = [ - '.ppt', '.pot', '.pps', '.pptx', '.pptm', - '.potx', '.ppsx', '.ppsm', '.sldx', '.sldm', + ".ppt", + ".pot", + ".pps", + ".pptx", + ".pptm", + ".potx", + ".ppsx", + ".ppsm", + ".sldx", + ".sldm", ] -SUPPORTED_MS_OFFICE = ( - SUPPORTED_MS_WORD + - SUPPORTED_MS_POWERPOINT + - SUPPORTED_MS_EXCEL -) +SUPPORTED_MS_OFFICE = SUPPORTED_MS_WORD + SUPPORTED_MS_POWERPOINT + SUPPORTED_MS_EXCEL SUPPORTED_IMAGE_PREVIEW_EXTS = [ - '.png', '.gif', '.jpg', '.jpeg', + ".png", + ".gif", + ".jpg", + ".jpeg", ] SUPPORTED_TEXT_PREVIEW_EXTS = [ - '.as', '.as3', '.asm', '.bat', '.c', '.cc', '.cmake', '.cpp', - '.cs', '.css', '.csv', '.cxx', '.diff', '.groovy', '.h', '.haml', - '.hh', '.java', '.js', '.less', '.m', '.make', '.md', - '.ml', '.mm', '.msg', '.php', '.pl', '.properties', '.py', '.rb', - '.sass', '.scala', '.script', '.sh', '.sml', '.sql', '.txt', '.vi', - '.vim', '.xml', '.xsd', '.xsl', '.yaml', '.yml', '.tcl', '.json', - '.out', '.err', '.f', + ".as", + ".as3", + ".asm", + ".bat", + ".c", + ".cc", + ".cmake", + ".cpp", + ".cs", + ".css", + ".csv", + ".cxx", + ".diff", + ".groovy", + ".h", + ".haml", + ".hh", + ".java", + ".js", + ".less", + ".m", + ".make", + ".md", + ".ml", + ".mm", + ".msg", + ".php", + ".pl", + ".properties", + ".py", + ".rb", + ".sass", + ".scala", + ".script", + ".sh", + ".sml", + ".sql", + ".txt", + ".vi", + ".vim", + ".xml", + ".xsd", + ".xsl", + ".yaml", + ".yml", + ".tcl", + ".json", + ".out", + ".err", + ".f", ] SUPPORTED_OBJECT_PREVIEW_EXTS = [ - '.pdf', + ".pdf", ] -SUPPORTED_IPYNB_PREVIEW_EXTS = [ - '.ipynb' -] +SUPPORTED_IPYNB_PREVIEW_EXTS = [".ipynb"] -SUPPORTED_NEW_WINDOW_PREVIEW_EXTS = [ - '.htm', '.html' -] +SUPPORTED_NEW_WINDOW_PREVIEW_EXTS = [".htm", ".html"] -SUPPORTED_BRAINMAP_PREVIEW_EXTS = [ - '.nii', '.nii.gz' -] +SUPPORTED_BRAINMAP_PREVIEW_EXTS = [".nii", ".nii.gz"] -SUPPORTED_PREVIEW_EXTENSIONS = (SUPPORTED_IMAGE_PREVIEW_EXTS + - SUPPORTED_TEXT_PREVIEW_EXTS + - SUPPORTED_OBJECT_PREVIEW_EXTS + - SUPPORTED_MS_OFFICE + - SUPPORTED_IPYNB_PREVIEW_EXTS + - SUPPORTED_BRAINMAP_PREVIEW_EXTS) +SUPPORTED_PREVIEW_EXTENSIONS = ( + SUPPORTED_IMAGE_PREVIEW_EXTS + + SUPPORTED_TEXT_PREVIEW_EXTS + + SUPPORTED_OBJECT_PREVIEW_EXTS + + SUPPORTED_MS_OFFICE + + SUPPORTED_IPYNB_PREVIEW_EXTS + + SUPPORTED_BRAINMAP_PREVIEW_EXTS +) # Channels -ASGI_APPLICATION = 'portal.asgi.application' +ASGI_APPLICATION = "portal.asgi.application" CHANNEL_LAYERS = { - 'default': { - 'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { + "default": { + "BACKEND": "channels_redis.core.RedisChannelLayer", + "CONFIG": { "hosts": [(_RESULT_BACKEND_HOST, _RESULT_BACKEND_PORT)], }, }, - 'short-lived': { - 'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { + "short-lived": { + "BACKEND": "channels_redis.core.RedisChannelLayer", + "CONFIG": { "hosts": [(_RESULT_BACKEND_HOST, _RESULT_BACKEND_PORT)], }, }, @@ -832,42 +794,42 @@ def portal_filter(record): """ SETTINGS: WORKBENCH SETTINGS """ -WORKBENCH_SETTINGS = getattr(settings_custom, '_WORKBENCH_SETTINGS', {}) -WORKBENCH_SETTINGS.update({'trashPath': TAPIS_DEFAULT_TRASH_NAME}) -WORKBENCH_SETTINGS.setdefault('showUserNews', False) +WORKBENCH_SETTINGS = getattr(settings_custom, "_WORKBENCH_SETTINGS", {}) +WORKBENCH_SETTINGS.update({"trashPath": TAPIS_DEFAULT_TRASH_NAME}) +WORKBENCH_SETTINGS.setdefault("showUserNews", False) """ SETTINGS: RECAPTCHA """ -RECAPTCHA_SECRET_KEY = getattr(settings_secret, '_RECAPTCHA_SECRET_KEY', None) -RECAPTCHA_SITE_KEY = getattr(settings_secret, '_RECAPTCHA_SITE_KEY', None) +RECAPTCHA_SECRET_KEY = getattr(settings_secret, "_RECAPTCHA_SECRET_KEY", None) +RECAPTCHA_SITE_KEY = getattr(settings_secret, "_RECAPTCHA_SITE_KEY", None) -PORTAL_ELEVATED_ROLES = getattr(settings_custom, '_PORTAL_ELEVATED_ROLES', {}) +PORTAL_ELEVATED_ROLES = getattr(settings_custom, "_PORTAL_ELEVATED_ROLES", {}) """ SETTINGS: EMAIL """ -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -EMAIL_HOST = getattr(settings_custom, '_SMTP_HOST', 'localhost') -EMAIL_PORT = getattr(settings_custom, '_SMTP_PORT', 25) -EMAIL_HOST_USER = getattr(settings_custom, '_SMTP_USER', '') -EMAIL_HOST_PASSWORD = getattr(settings_custom, '_SMTP_PASSWORD', '') -DEFAULT_FROM_EMAIL = getattr(settings_custom, '_DEFAULT_FROM_EMAIL', '') +EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" +EMAIL_HOST = getattr(settings_custom, "_SMTP_HOST", "localhost") +EMAIL_PORT = getattr(settings_custom, "_SMTP_PORT", 25) +EMAIL_HOST_USER = getattr(settings_custom, "_SMTP_USER", "") +EMAIL_HOST_PASSWORD = getattr(settings_custom, "_SMTP_PASSWORD", "") +DEFAULT_FROM_EMAIL = getattr(settings_custom, "_DEFAULT_FROM_EMAIL", "") """ SETTINGS: INTERNAL DOCS """ -INTERNAL_DOCS_ROOT = getattr(settings_custom, '_INTERNAL_DOCS_ROOT', '') -INTERNAL_DOCS_URL = getattr(settings_custom, '_INTERNAL_DOCS_URL', '') +INTERNAL_DOCS_ROOT = getattr(settings_custom, "_INTERNAL_DOCS_ROOT", "") +INTERNAL_DOCS_URL = getattr(settings_custom, "_INTERNAL_DOCS_URL", "") """ SETTINGS: LOCAL OVERRIDES """ -if os.path.isfile(os.path.join(BASE_DIR, 'settings', 'settings_local.py')): +if os.path.isfile(os.path.join(BASE_DIR, "settings", "settings_local.py")): from .settings_local import * # noqa: F403, F401 """ SETTINGS: PUBLICATIONS """ -PUBLICATION_REVIEWERS = getattr(settings_secret, '_PUBLICATION_REVIEWERS', []) +PUBLICATION_REVIEWERS = getattr(settings_secret, "_PUBLICATION_REVIEWERS", []) diff --git a/server/portal/settings/settings_secret.example.py b/server/portal/settings/settings_secret.example.py index 22ec4ed8e5..7b9d52c7a4 100644 --- a/server/portal/settings/settings_secret.example.py +++ b/server/portal/settings/settings_secret.example.py @@ -6,36 +6,36 @@ # DJANGO SETTINGS COMMON ######################## -_SECRET_KEY = 'CHANGE ME !' +_SECRET_KEY = "CHANGE ME !" ######################## # DJANGO SETTINGS LOCAL ######################## # Database. -_DJANGO_DB_ENGINE = 'django.db.backends.postgresql' -_DJANGO_DB_HOST = 'core_portal_postgres' -_DJANGO_DB_PORT = '5432' -_DJANGO_DB_NAME = 'dev' -_DJANGO_DB_USER = 'dev' -_DJANGO_DB_PASSWORD = 'dev' +_DJANGO_DB_ENGINE = "django.db.backends.postgresql" +_DJANGO_DB_HOST = "core_portal_postgres" +_DJANGO_DB_PORT = "5432" +_DJANGO_DB_NAME = "dev" +_DJANGO_DB_USER = "dev" +_DJANGO_DB_PASSWORD = "dev" # TAS Authentication. -_TAS_URL = 'https://tas-dev.tacc.utexas.edu/api' -_TAS_CLIENT_KEY = 'key' -_TAS_CLIENT_SECRET = 'secret' +_TAS_URL = "https://tas-dev.tacc.utexas.edu/api" +_TAS_CLIENT_KEY = "key" +_TAS_CLIENT_SECRET = "secret" # Redmine Tracker Authentication. -_RT_HOST = 'https://consult.tacc.utexas.edu/REST/1.0' -_RT_UN = 'username' -_RT_PW = 'password' +_RT_HOST = "https://consult.tacc.utexas.edu/REST/1.0" +_RT_UN = "username" +_RT_PW = "password" ######################## # TAPIS v2 SETTINGS ######################## # Admin account -_PORTAL_ADMIN_USERNAME = 'portal_admin' +_PORTAL_ADMIN_USERNAME = "portal_admin" ######################## # TAPIS v3 SETTINGS @@ -44,53 +44,53 @@ ######################## # Tapis Tenant. -_TAPIS_TENANT_BASEURL = 'https://example.tapis.io' +_TAPIS_TENANT_BASEURL = "https://example.tapis.io" # Tapis Client Configuration -_TAPIS_CLIENT_ID = '' -_TAPIS_CLIENT_KEY = '' +_TAPIS_CLIENT_ID = "" +_TAPIS_CLIENT_KEY = "" # Long-live portal admin access token -_TAPIS_ADMIN_JWT = '' +_TAPIS_ADMIN_JWT = "" ######################## # RABBITMQ SETTINGS ######################## -_BROKER_URL_USERNAME = 'dev' -_BROKER_URL_PWD = 'dev' -_BROKER_URL_HOST = 'core_portal_rabbitmq' -_BROKER_URL_PORT = '5672' -_BROKER_URL_VHOST = 'dev' +_BROKER_URL_USERNAME = "dev" +_BROKER_URL_PWD = "dev" +_BROKER_URL_HOST = "core_portal_rabbitmq" +_BROKER_URL_PORT = "5672" +_BROKER_URL_VHOST = "dev" ######################## # ELASTICSEARCH SETTINGS ######################## -_ES_HOSTS = 'core_portal_elasticsearch:9200' -_ES_AUTH = 'username:password' -_ES_INDEX_PREFIX = 'cep-dev-{}' +_ES_HOSTS = "core_portal_elasticsearch:9200" +_ES_AUTH = "username:password" +_ES_INDEX_PREFIX = "cep-dev-{}" ######################## # CELERY SETTINGS ######################## -_RESULT_BACKEND_HOST = 'core_portal_redis' -_RESULT_BACKEND_PORT = '6379' -_RESULT_BACKEND_DB = '0' +_RESULT_BACKEND_HOST = "core_portal_redis" +_RESULT_BACKEND_PORT = "6379" +_RESULT_BACKEND_DB = "0" ####################### # PROJECTS SETTINGS ####################### -_PORTAL_PROJECTS_PRIVATE_KEY = '' -_PORTAL_PROJECTS_PUBLIC_KEY = '' +_PORTAL_PROJECTS_PRIVATE_KEY = "" +_PORTAL_PROJECTS_PUBLIC_KEY = "" """ SETTINGS: RECAPTCHA """ -RECAPTCHA_SECRET_KEY = 'key' -RECAPTCHA_SITE_KEY = 'secret' +RECAPTCHA_SECRET_KEY = "key" +RECAPTCHA_SITE_KEY = "secret" ######################## # EXTERNAL DATA RESOURCES SETTINGS @@ -110,7 +110,7 @@ "client_secret": "S3CR3T_K3Y", "client_id": "XXXXXXX.apps.googleusercontent.com", "name": "Google Drive", - "directory": "external-resources" + "directory": "external-resources", } } diff --git a/server/portal/settings/unit_test_settings.py b/server/portal/settings/unit_test_settings.py index 406d542545..3e01e885ee 100644 --- a/server/portal/settings/unit_test_settings.py +++ b/server/portal/settings/unit_test_settings.py @@ -21,168 +21,153 @@ SITE_ID = 1 # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '__CHANGE_ME!__' +SECRET_KEY = "__CHANGE_ME!__" # SECURITY WARNING: don't run with debug turned on in production! # Cookie name. this can be whatever you want -SESSION_COOKIE_NAME = 'sessionid' # use the sessionid in your views code +SESSION_COOKIE_NAME = "sessionid" # use the sessionid in your views code # the module to store sessions data -SESSION_ENGINE = 'django.contrib.sessions.backends.db' +SESSION_ENGINE = "django.contrib.sessions.backends.db" # age of cookie in seconds (default: 2 weeks) -SESSION_COOKIE_AGE = 24*60*60*7 # the number of seconds for only 7 for example +SESSION_COOKIE_AGE = 24 * 60 * 60 * 7 # the number of seconds for only 7 for example # whether a user's session cookie expires when the web browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = False # whether the session cookie should be secure (https:// only) SESSION_COOKIE_SECURE = False -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] # Custom Portal Template Assets -PORTAL_ICON_FILENAME = 'path/to/icon.ico' -PORTAL_ADMIN_USERNAME = 'wma_prtl' +PORTAL_ICON_FILENAME = "path/to/icon.ico" +PORTAL_ADMIN_USERNAME = "wma_prtl" # Application definition -ROOT_URLCONF = 'portal.urls' +ROOT_URLCONF = "portal.urls" INSTALLED_APPS = [ - # Django Channels - 'channels', - 'daphne', - + "channels", + "daphne", # Core Django. - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - - 'django.contrib.sitemaps', - 'django.contrib.sessions.middleware', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sitemaps", + "django.contrib.sessions.middleware", # Pipeline. - 'termsandconditions', - 'impersonate', - + "termsandconditions", + "impersonate", # Custom apps. - 'portal.apps.accounts', - 'portal.apps.auth', - 'portal.apps.tickets', - 'portal.apps.licenses', - 'portal.apps.notifications', - 'portal.apps.news', - 'portal.apps.onboarding', - 'portal.apps.search', - 'portal.apps.webhooks', - 'portal.apps.workbench', - 'portal.apps.workspace', - 'portal.apps.system_monitor', - 'portal.apps.googledrive_integration', - 'portal.apps.datafiles', - 'portal.apps.projects', - 'portal.apps.portal_messages', - 'portal.apps.publications', - + "portal.apps.accounts", + "portal.apps.auth", + "portal.apps.tickets", + "portal.apps.licenses", + "portal.apps.notifications", + "portal.apps.news", + "portal.apps.onboarding", + "portal.apps.search", + "portal.apps.webhooks", + "portal.apps.workbench", + "portal.apps.workspace", + "portal.apps.system_monitor", + "portal.apps.googledrive_integration", + "portal.apps.datafiles", + "portal.apps.projects", + "portal.apps.portal_messages", + "portal.apps.publications", ] MIDDLEWARE = [ # Django core middleware. - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - - 'impersonate.middleware.ImpersonateMiddleware', # must be AFTER django.contrib.auth + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "impersonate.middleware.ImpersonateMiddleware", # must be AFTER django.contrib.auth ] TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - - 'portal.utils.contextprocessors.analytics', - 'portal.utils.contextprocessors.debug', - 'portal.utils.contextprocessors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "portal.utils.contextprocessors.analytics", + "portal.utils.contextprocessors.debug", + "portal.utils.contextprocessors.messages", ], }, }, ] -WSGI_APPLICATION = 'portal.wsgi.application' +WSGI_APPLICATION = "portal.wsgi.application" -AUTHENTICATION_BACKENDS = ['django.contrib.auth.backends.ModelBackend'] +AUTHENTICATION_BACKENDS = ["django.contrib.auth.backends.ModelBackend"] # Password validation # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] IMPERSONATE_REQUIRE_SUPERUSER = True -LOGIN_REDIRECT_URL = '/index/' +LOGIN_REDIRECT_URL = "/index/" LOGOUT_REDIRECT_URL = "/cms/logout/" # Internationalization # https://docs.djangoproject.com/en/1.10/topics/i18n/ -LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +LANGUAGE_CODE = "en-us" +TIME_ZONE = "UTC" USE_I18N = True USE_TZ = True -LANGUAGES = [ - ('en-us', 'US English') -] +LANGUAGES = [("en-us", "US English")] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ -STATIC_URL = '/static/' -MEDIA_URL = '/media/' +STATIC_URL = "/static/" +MEDIA_URL = "/media/" STATICFILES_DIRS = [ - os.path.join(BASE_DIR, 'static'), + os.path.join(BASE_DIR, "static"), ] FIXTURE_DIRS = [ - os.path.join(BASE_DIR, 'fixtures'), + os.path.join(BASE_DIR, "fixtures"), ] STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'test' - } -} +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "test"}} # DATABASES = { # 'default': { @@ -197,122 +182,126 @@ ALLOCATION_SYSTEMS = [] -PORTAL_NAMESPACE = 'test' -PORTAL_ALLOCATION = 'test' +PORTAL_NAMESPACE = "test" +PORTAL_ALLOCATION = "test" PORTAL_PROJECTS_USE_SET_FACL_JOB = False -PROJECT_ADMIN_GROUP = 'Project Admin' +PROJECT_ADMIN_GROUP = "Project Admin" -PORTAL_KEYS_MANAGER = 'portal.apps.accounts.managers.ssh_keys.KeysManager' -PORTAL_PROJECTS_PEMS_APP_ID = 'pems.app-test' +PORTAL_KEYS_MANAGER = "portal.apps.accounts.managers.ssh_keys.KeysManager" +PORTAL_PROJECTS_PEMS_APP_ID = "pems.app-test" -PORTAL_PROJECTS_SYSTEM_PREFIX = 'test.project' +PORTAL_PROJECTS_SYSTEM_PREFIX = "test.project" PORTAL_PROJECTS_ENABLE_METADATA = False -PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = 'test.project.published' +PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = "test.project.published" PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = None PORTAL_PUBLICATION_PUBLISHER = None -PORTAL_PROJECTS_ID_PREFIX = 'test.project' +PORTAL_PROJECTS_ID_PREFIX = "test.project" -PORTAL_PROJECTS_ROOT_DIR = '/path/to/root' +PORTAL_PROJECTS_ROOT_DIR = "/path/to/root" -PORTAL_PROJECTS_ROOT_SYSTEM_NAME = 'projects.system.name' +PORTAL_PROJECTS_ROOT_SYSTEM_NAME = "projects.system.name" -PORTAL_PROJECTS_ROOT_HOST = 'host.for.projects.tacc.utexas.edu' +PORTAL_PROJECTS_ROOT_HOST = "host.for.projects.tacc.utexas.edu" PORTAL_PROJECTS_SYSTEM_PORT = 22 -PORTAL_PROJECTS_PRIVATE_KEY = ('-----BEGIN RSA PRIVATE KEY-----' - 'change this' - '-----END RSA PRIVATE KEY-----') -PORTAL_PROJECTS_PUBLIC_KEY = 'ssh-rsa change this' +PORTAL_PROJECTS_PRIVATE_KEY = "-----BEGIN RSA PRIVATE KEY-----change this-----END RSA PRIVATE KEY-----" +PORTAL_PROJECTS_PUBLIC_KEY = "ssh-rsa change this" PORTAL_USER_ACCOUNT_SETUP_STEPS = [ - { - 'step': 'portal.apps.onboarding.steps.test_steps.MockStep', - 'settings': { - 'key': 'value' - } - } + {"step": "portal.apps.onboarding.steps.test_steps.MockStep", "settings": {"key": "value"}} ] -PORTAL_USER_ACCOUNT_SETUP_WEBHOOK_PWD = 'dev' +PORTAL_USER_ACCOUNT_SETUP_WEBHOOK_PWD = "dev" PORTAL_DATA_DEPOT_MANAGERS = { - 'my-data': 'portal.apps.data_depot.managers.private_data.FileManager', - 'shared': 'portal.apps.data_depot.managers.shared.FileManager', - 'my-projects': 'portal.apps.data_depot.managers.projects.FileManager', - 'google-drive': 'portal.apps.data_depot.managers.google_drive.FileManager' + "my-data": "portal.apps.data_depot.managers.private_data.FileManager", + "shared": "portal.apps.data_depot.managers.shared.FileManager", + "my-projects": "portal.apps.data_depot.managers.projects.FileManager", + "google-drive": "portal.apps.data_depot.managers.google_drive.FileManager", } PORTAL_SEARCH_MANAGERS = { - 'my-data': 'portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager', - 'shared': 'portal.apps.search.api.managers.shared_search.SharedSearchManager', - 'cms': 'portal.apps.search.api.managers.cms_search.CMSSearchManager', + "my-data": "portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager", + "shared": "portal.apps.search.api.managers.shared_search.SharedSearchManager", + "cms": "portal.apps.search.api.managers.cms_search.CMSSearchManager", # 'my-projects': 'portal.apps.data_depot.managers.projects.FileManager' } -PORTAL_JOB_NOTIFICATION_STATES = ["PENDING", "STAGING_INPUTS", "RUNNING", "ARCHIVING", "BLOCKED", "PAUSED", "FINISHED", "CANCELLED", "FAILED"] +PORTAL_JOB_NOTIFICATION_STATES = [ + "PENDING", + "STAGING_INPUTS", + "RUNNING", + "ARCHIVING", + "BLOCKED", + "PAUSED", + "FINISHED", + "CANCELLED", + "FAILED", +] EXTERNAL_RESOURCE_SECRETS = { "google-drive": { "client_secret": "test", "client_id": "test", "name": "Google Drive", - "directory": "external-resources" + "directory": "external-resources", } } PORTAL_DATA_DEPOT_PAGE_SIZE = 100 PORTAL_WORKSPACE_MANAGERS = { - 'private': 'portal.apps.workspace.managers.private.FileManager', - 'shared': 'portal.apps.workspace.managers.shared.FileManager', + "private": "portal.apps.workspace.managers.private.FileManager", + "shared": "portal.apps.workspace.managers.shared.FileManager", } PORTAL_WORKSPACE_PAGE_SIZE = 100 # TAS Authentication. -TAS_URL = 'https://test.com' -TAS_CLIENT_KEY = 'test' -TAS_CLIENT_SECRET = 'test' +TAS_URL = "https://test.com" +TAS_CLIENT_KEY = "test" +TAS_CLIENT_SECRET = "test" # Redmine Tracker Authentication. -RT_URL = 'test' -RT_HOST = 'https://test.com' -RT_UN = 'test' -RT_PW = 'test' -RT_QUEUE = 'test' -RT_TAG = 'test_tag' +RT_URL = "test" +RT_HOST = "https://test.com" +RT_UN = "test" +RT_PW = "test" +RT_QUEUE = "test" +RT_TAG = "test_tag" # Tapis Tenant. -TAPIS_TENANT_BASEURL = 'https://example.tapis.io' +TAPIS_TENANT_BASEURL = "https://example.tapis.io" # Tapis Client Configuration -TAPIS_CLIENT_ID = 'test' -TAPIS_CLIENT_KEY = 'test' -TAPIS_ADMIN_JWT = 'test' -TAPIS_DEFAULT_TRASH_NAME = 'test' +TAPIS_CLIENT_ID = "test" +TAPIS_CLIENT_KEY = "test" +TAPIS_ADMIN_JWT = "test" +TAPIS_DEFAULT_TRASH_NAME = "test" -AGAVE_JWT_HEADER = 'HTTP_X_AGAVE_HEADER' -AGAVE_JWT_ISSUER = 'wso2.org/products/am' -AGAVE_JWT_USER_CLAIM_FIELD = 'http://wso2.org/claims/fullname' +AGAVE_JWT_HEADER = "HTTP_X_AGAVE_HEADER" +AGAVE_JWT_ISSUER = "wso2.org/products/am" +AGAVE_JWT_USER_CLAIM_FIELD = "http://wso2.org/claims/fullname" -ES_HOSTS = ['test.com'] +ES_HOSTS = ["test.com"] ES_AUTH = "user:password" ES_INDEX_PREFIX = "test-staging-{}" SYSTEM_MONITOR_URL = "https://sysmon.example.com/foo.json" HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': ('haystack.backends.elasticsearch_backend.' - 'ElasticsearchSearchEngine'), - 'URL': 'test:9200/', - 'INDEX_NAME': 'cms', + "default": { + "ENGINE": ("haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine"), + "URL": "test:9200/", + "INDEX_NAME": "cms", } } -HAYSTACK_ROUTERS = ['aldryn_search.router.LanguageRouter', ] +HAYSTACK_ROUTERS = [ + "aldryn_search.router.LanguageRouter", +] """ SETTINGS: RECAPTCHA TESTING KEY @@ -326,229 +315,251 @@ """ LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '[DJANGO-TEST] %(levelname)s %(asctime)s %(module)s ' - '%(name)s.%(funcName)s:%(lineno)s: %(message)s' + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "[DJANGO-TEST] %(levelname)s %(asctime)s %(module)s %(name)s.%(funcName)s:%(lineno)s: %(message)s" }, - 'metrics': { - 'format': '[METRICS-TEST] %(levelname)s %(module)s %(name)s.' - '%(funcName)s:%(lineno)s: %(message)s ' - 'user=%(user)s sessionId=%(sessionId)s ' - 'op=%(operation)s info=%(info)s' + "metrics": { + "format": "[METRICS-TEST] %(levelname)s %(module)s %(name)s." + "%(funcName)s:%(lineno)s: %(message)s " + "user=%(user)s sessionId=%(sessionId)s " + "op=%(operation)s info=%(info)s" }, }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'default', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "default", + }, + "metrics_console": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "metrics", }, - 'metrics_console': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'metrics', - } }, - 'loggers': { - 'django': { - 'handlers': ['console'], - 'level': 'INFO', - 'propagate': True, + "loggers": { + "django": { + "handlers": ["console"], + "level": "INFO", + "propagate": True, }, - 'portal': { - 'handlers': ['console'], - 'level': 'DEBUG', + "portal": { + "handlers": ["console"], + "level": "DEBUG", }, - 'metrics': { - 'handlers': ['metrics_console'], - 'level': 'INFO', + "metrics": { + "handlers": ["metrics_console"], + "level": "INFO", }, - 'paramiko': { - 'handlers': ['console'], - 'level': 'DEBUG' - } + "paramiko": {"handlers": ["console"], "level": "DEBUG"}, }, } MIGRATION_MODULES = { - 'auth': None, - 'contenttypes': None, - - - 'default': None, - 'core': None, - 'profiles': None, + "auth": None, + "contenttypes": None, + "default": None, + "core": None, + "profiles": None, } -COMMUNITY_INDEX_SCHEDULE = {'hour': 0, 'minute': 0, 'day_of_week': 0} +COMMUNITY_INDEX_SCHEDULE = {"hour": 0, "minute": 0, "day_of_week": 0} """ SETTINGS: SUPPORTED FILE PREVIEW TYPES """ SUPPORTED_MS_WORD = [ - '.doc', '.dot', '.docx', '.docm', '.dotx', '.dotm', '.docb', + ".doc", + ".dot", + ".docx", + ".docm", + ".dotx", + ".dotm", + ".docb", ] SUPPORTED_MS_EXCEL = [ - '.xls', '.xlt', '.xlm', '.xlsx', '.xlsm', '.xltx', '.xltm', + ".xls", + ".xlt", + ".xlm", + ".xlsx", + ".xlsm", + ".xltx", + ".xltm", ] SUPPORTED_MS_POWERPOINT = [ - '.ppt', '.pot', '.pps', '.pptx', '.pptm', - '.potx', '.ppsx', '.ppsm', '.sldx', '.sldm', + ".ppt", + ".pot", + ".pps", + ".pptx", + ".pptm", + ".potx", + ".ppsx", + ".ppsm", + ".sldx", + ".sldm", ] -SUPPORTED_MS_OFFICE = ( - SUPPORTED_MS_WORD + - SUPPORTED_MS_POWERPOINT + - SUPPORTED_MS_EXCEL -) +SUPPORTED_MS_OFFICE = SUPPORTED_MS_WORD + SUPPORTED_MS_POWERPOINT + SUPPORTED_MS_EXCEL SUPPORTED_IMAGE_PREVIEW_EXTS = [ - '.png', '.gif', '.jpg', '.jpeg', + ".png", + ".gif", + ".jpg", + ".jpeg", ] SUPPORTED_TEXT_PREVIEW_EXTS = [ - '.as', '.as3', '.asm', '.bat', '.c', '.cc', '.cmake', '.cpp', - '.cs', '.css', '.csv', '.cxx', '.diff', '.groovy', '.h', '.haml', - '.hh', '.java', '.js', '.less', '.m', '.make', '.md', - '.ml', '.mm', '.msg', '.php', '.pl', '.properties', '.py', '.rb', - '.sass', '.scala', '.script', '.sh', '.sml', '.sql', '.txt', '.vi', - '.vim', '.xml', '.xsd', '.xsl', '.yaml', '.yml', '.tcl', '.json', - '.out', '.err', '.f', + ".as", + ".as3", + ".asm", + ".bat", + ".c", + ".cc", + ".cmake", + ".cpp", + ".cs", + ".css", + ".csv", + ".cxx", + ".diff", + ".groovy", + ".h", + ".haml", + ".hh", + ".java", + ".js", + ".less", + ".m", + ".make", + ".md", + ".ml", + ".mm", + ".msg", + ".php", + ".pl", + ".properties", + ".py", + ".rb", + ".sass", + ".scala", + ".script", + ".sh", + ".sml", + ".sql", + ".txt", + ".vi", + ".vim", + ".xml", + ".xsd", + ".xsl", + ".yaml", + ".yml", + ".tcl", + ".json", + ".out", + ".err", + ".f", ] SUPPORTED_OBJECT_PREVIEW_EXTS = [ - '.pdf', + ".pdf", ] -SUPPORTED_IPYNB_PREVIEW_EXTS = [ - '.ipynb' -] +SUPPORTED_IPYNB_PREVIEW_EXTS = [".ipynb"] -SUPPORTED_NEW_WINDOW_PREVIEW_EXTS = [ - '.htm', '.html' -] +SUPPORTED_NEW_WINDOW_PREVIEW_EXTS = [".htm", ".html"] -SUPPORTED_BRAINMAP_PREVIEW_EXTS = [ - '.nii', '.nii.gz' -] +SUPPORTED_BRAINMAP_PREVIEW_EXTS = [".nii", ".nii.gz"] -SUPPORTED_PREVIEW_EXTENSIONS = (SUPPORTED_IMAGE_PREVIEW_EXTS + - SUPPORTED_TEXT_PREVIEW_EXTS + - SUPPORTED_OBJECT_PREVIEW_EXTS + - SUPPORTED_MS_OFFICE + - SUPPORTED_IPYNB_PREVIEW_EXTS + - SUPPORTED_BRAINMAP_PREVIEW_EXTS) +SUPPORTED_PREVIEW_EXTENSIONS = ( + SUPPORTED_IMAGE_PREVIEW_EXTS + + SUPPORTED_TEXT_PREVIEW_EXTS + + SUPPORTED_OBJECT_PREVIEW_EXTS + + SUPPORTED_MS_OFFICE + + SUPPORTED_IPYNB_PREVIEW_EXTS + + SUPPORTED_BRAINMAP_PREVIEW_EXTS +) # Channels -ASGI_APPLICATION = 'portal.asgi.application' +ASGI_APPLICATION = "portal.asgi.application" CHANNEL_LAYERS = { - 'default': { - 'BACKEND': 'channels.layers.InMemoryChannelLayer', + "default": { + "BACKEND": "channels.layers.InMemoryChannelLayer", }, } PORTAL_DATAFILES_STORAGE_SYSTEMS = [ { - 'name': 'My Data (Work)', - 'system': 'cloud.data', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/home/{username}', - 'icon': None, - 'default': True + "name": "My Data (Work)", + "system": "cloud.data", + "scheme": "private", + "api": "tapis", + "homeDir": "/home/{username}", + "icon": None, + "default": True, }, { - 'name': 'My Data (Frontera)', - 'system': 'frontera', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/home1/{tasdir}', - 'icon': None, + "name": "My Data (Frontera)", + "system": "frontera", + "scheme": "private", + "api": "tapis", + "homeDir": "/home1/{tasdir}", + "icon": None, }, { - 'name': 'Community Data', - 'system': 'cloud.data', - 'scheme': 'community', - 'api': 'tapis', - 'homeDir': '/path/to/community', - 'icon': None, - 'siteSearchPriority': 1 + "name": "Community Data", + "system": "cloud.data", + "scheme": "community", + "api": "tapis", + "homeDir": "/path/to/community", + "icon": None, + "siteSearchPriority": 1, }, { - 'name': 'Public Data', - 'system': 'cloud.data', - 'scheme': 'public', - 'api': 'tapis', - 'homeDir': '/path/to/public', - 'icon': 'publications', - 'siteSearchPriority': 0 + "name": "Public Data", + "system": "cloud.data", + "scheme": "public", + "api": "tapis", + "homeDir": "/path/to/public", + "icon": "publications", + "siteSearchPriority": 0, }, + {"name": "Shared Workspaces", "scheme": "projects", "api": "tapis", "icon": "publications"}, { - 'name': 'Shared Workspaces', - 'scheme': 'projects', - 'api': 'tapis', - 'icon': 'publications' + "name": "Google Drive", + "system": "googledrive", + "scheme": "private", + "api": "googledrive", + "icon": None, + "integration": "portal.apps.googledrive_integration", }, - { - 'name': 'Google Drive', - 'system': 'googledrive', - 'scheme': 'private', - 'api': 'googledrive', - 'icon': None, - 'integration': 'portal.apps.googledrive_integration' - } ] -PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next((sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['default'] is True), None) +PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next( + (sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys["default"] is True), None +) """ SETTINGS: TACC EXECUTION SYSTEMS """ TACC_EXEC_SYSTEMS = { - 'corral': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/work2/{}', - 'home_dir': '/home/{}' - }, - 'stampede2': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, - 'frontera': { - 'work_dir': '/work2/{}', - 'scratch_dir': '/scratch1/{}', - 'home_dir': '/home1/{}' - }, - 'ls6': { - 'work_dir': '/work/{}', - 'scratch_dir': '/scratch/{}', - 'home_dir': '/home1/{}' - }, + "corral": {"work_dir": "/work2/{}", "scratch_dir": "/work2/{}", "home_dir": "/home/{}"}, + "stampede2": {"work_dir": "/work2/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, + "frontera": {"work_dir": "/work2/{}", "scratch_dir": "/scratch1/{}", "home_dir": "/home1/{}"}, + "ls6": {"work_dir": "/work/{}", "scratch_dir": "/scratch/{}", "home_dir": "/home1/{}"}, } VANITY_BASE_URL = "https://testserver" -WORKBENCH_SETTINGS = { - "debug": False -} +WORKBENCH_SETTINGS = {"debug": False} -PORTAL_ELEVATED_ROLES = { - "is_staff": { - "groups": [], - "usernames": [] - }, - "is_superuser": { - "groups": [], - "usernames": [] - } -} +PORTAL_ELEVATED_ROLES = {"is_staff": {"groups": [], "usernames": []}, "is_superuser": {"groups": [], "usernames": []}} -INTERNAL_DOCS_URL = 'core/internal-docs/' -INTERNAL_DOCS_ROOT = '' +INTERNAL_DOCS_URL = "core/internal-docs/" +INTERNAL_DOCS_ROOT = "" IS_TACC_PORTAL = True diff --git a/server/portal/urls.py b/server/portal/urls.py index 712cdc28b2..cf0d26d509 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -17,7 +17,6 @@ :synopsis: Main URLs """ - from django.conf import settings from django.contrib import admin from django.conf.urls.static import static @@ -30,107 +29,98 @@ from impersonate import views as impersonate_views from portal.views.views import health_check from portal.views.views import serve_docs + admin.autodiscover() urlpatterns = [ - # django-impersonate path( - 'core/admin/impersonate/stop/', + "core/admin/impersonate/stop/", impersonate_views.stop_impersonate, - name='impersonate-stop', + name="impersonate-stop", ), path( - 'core/admin/impersonate/list/', + "core/admin/impersonate/list/", impersonate_views.list_users, - {'template': 'impersonate/list_users.html'}, - name='impersonate-list', + {"template": "impersonate/list_users.html"}, + name="impersonate-list", ), path( - 'core/admin/impersonate/search/', + "core/admin/impersonate/search/", impersonate_views.search_users, - {'template': 'impersonate/search_users.html'}, - name='impersonate-search', + {"template": "impersonate/search_users.html"}, + name="impersonate-search", ), path( - 'core/admin/impersonate//', + "core/admin/impersonate//", impersonate_views.impersonate, - name='impersonate-start', + name="impersonate-start", ), - # admin. - path('core/admin/', admin.site.urls), - + path("core/admin/", admin.site.urls), # terms-and-conditions - path('terms/', include('termsandconditions.urls')), - + path("terms/", include("termsandconditions.urls")), # accounts. - path('accounts/', include('portal.apps.accounts.urls', namespace='portal_accounts')), - path('api/accounts/', include('portal.apps.accounts.api.urls', namespace='portal_accounts_api')), - - path('api/onboarding/', include('portal.apps.onboarding.api.urls', namespace='portal_onboarding_api')), - path('register/', RedirectView.as_view(pattern_name='portal_accounts:register', permanent=True), name='register'), - + path("accounts/", include("portal.apps.accounts.urls", namespace="portal_accounts")), + path("api/accounts/", include("portal.apps.accounts.api.urls", namespace="portal_accounts_api")), + path("api/onboarding/", include("portal.apps.onboarding.api.urls", namespace="portal_onboarding_api")), + path("register/", RedirectView.as_view(pattern_name="portal_accounts:register", permanent=True), name="register"), # auth. - path('auth/', include('portal.apps.auth.urls', namespace='portal_auth')), - re_path('login/$', login, name='login'), - + path("auth/", include("portal.apps.auth.urls", namespace="portal_auth")), + re_path("login/$", login, name="login"), # markup - re_path('core/markup/nav', TemplateView.as_view(template_name='includes/nav_portal.raw.html'), name='portal_nav_markup'), - + re_path( + "core/markup/nav", TemplateView.as_view(template_name="includes/nav_portal.raw.html"), name="portal_nav_markup" + ), # api - path('api/auth/', include('portal.apps.auth.api.urls', namespace='auth_api')), - path('api/users/', include('portal.apps.users.urls', namespace='users')), - path('api/workbench/', include('portal.apps.workbench.api.urls', namespace='workbench_api')), - path('api/workspace/', include('portal.apps.workspace.api.urls', namespace='workspace_api')), - path('api/tickets/', include('portal.apps.tickets.api.urls', namespace='portal_tickets_api')), - path('api/request-access/', include('portal.apps.request_access.api.urls', namespace='request_access_api')), - path('api/datafiles/', include('portal.apps.datafiles.urls', namespace='datafiles')), - path('api/system-monitor/', include('portal.apps.system_monitor.urls', namespace='system_monitor')), - path('api/notifications/', include('portal.apps.notifications.urls', namespace='notifications')), - path('api/news/', include('portal.apps.news.api.urls', namespace='news_api')), - path('api/jupyter_mounts/', include('portal.apps.jupyter_mounts.api.urls', namespace='jupyter_mounts_api')), - path('api/projects/', include('portal.apps.projects.urls', namespace='projects')), - path('api/site-search/', include('portal.apps.site_search.api.urls', namespace='site_search_api')), - path('api/forms/', include('portal.apps.forms.urls', namespace='forms')), - path('api/publications/', include('portal.apps.publications.urls', namespace='publications_api')), - + path("api/auth/", include("portal.apps.auth.api.urls", namespace="auth_api")), + path("api/users/", include("portal.apps.users.urls", namespace="users")), + path("api/workbench/", include("portal.apps.workbench.api.urls", namespace="workbench_api")), + path("api/workspace/", include("portal.apps.workspace.api.urls", namespace="workspace_api")), + path("api/tickets/", include("portal.apps.tickets.api.urls", namespace="portal_tickets_api")), + path("api/request-access/", include("portal.apps.request_access.api.urls", namespace="request_access_api")), + path("api/datafiles/", include("portal.apps.datafiles.urls", namespace="datafiles")), + path("api/system-monitor/", include("portal.apps.system_monitor.urls", namespace="system_monitor")), + path("api/notifications/", include("portal.apps.notifications.urls", namespace="notifications")), + path("api/news/", include("portal.apps.news.api.urls", namespace="news_api")), + path("api/jupyter_mounts/", include("portal.apps.jupyter_mounts.api.urls", namespace="jupyter_mounts_api")), + path("api/projects/", include("portal.apps.projects.urls", namespace="projects")), + path("api/site-search/", include("portal.apps.site_search.api.urls", namespace="site_search_api")), + path("api/forms/", include("portal.apps.forms.urls", namespace="forms")), + path("api/publications/", include("portal.apps.publications.urls", namespace="publications_api")), # webhooks - path('webhooks/', include('portal.apps.webhooks.urls', namespace='webhooks')), - + path("webhooks/", include("portal.apps.webhooks.urls", namespace="webhooks")), # views - path('tickets/', include('portal.apps.tickets.urls', namespace='tickets')), - path('googledrive-privacy-policy/', - include('portal.apps.googledrive_integration.urls', - namespace='googledrive-privacy-policy')), - path('workbench/', include('portal.apps.workbench.urls', namespace='workbench')), - path('public-data/', include('portal.apps.public_data.urls', namespace='public')), - path('published-datasets/', include('portal.apps.public_data.urls', namespace='publications')), - path('request-access/', include('portal.apps.request_access.urls', namespace='request_access')), - path('user-news/', include('portal.apps.news.urls', namespace='news')), - path('search/', include('portal.apps.site_search.urls', namespace='site_search')), - + path("tickets/", include("portal.apps.tickets.urls", namespace="tickets")), + path( + "googledrive-privacy-policy/", + include("portal.apps.googledrive_integration.urls", namespace="googledrive-privacy-policy"), + ), + path("workbench/", include("portal.apps.workbench.urls", namespace="workbench")), + path("public-data/", include("portal.apps.public_data.urls", namespace="public")), + path("published-datasets/", include("portal.apps.public_data.urls", namespace="publications")), + path("request-access/", include("portal.apps.request_access.urls", namespace="request_access")), + path("user-news/", include("portal.apps.news.urls", namespace="news")), + path("search/", include("portal.apps.site_search.urls", namespace="site_search")), # portal_messages - path('api/portal_messages/', include('portal.apps.portal_messages.urls', namespace='portal_messages')), - - + path("api/portal_messages/", include("portal.apps.portal_messages.urls", namespace="portal_messages")), # integrations - path('accounts/applications/googledrive/', include('portal.apps.googledrive_integration.urls', namespace='googledrive_integration')), - + path( + "accounts/applications/googledrive/", + include("portal.apps.googledrive_integration.urls", namespace="googledrive_integration"), + ), # version check. - path('version/', portal_version), - + path("version/", portal_version), # health check - path('core/health-check', health_check), - + path("core/health-check", health_check), ] # custom endpoint -if settings.WORKBENCH_SETTINGS.get('hasCustomEndpoints'): +if settings.WORKBENCH_SETTINGS.get("hasCustomEndpoints"): urlpatterns.append( path( - f'api/{settings.PORTAL_NAMESPACE.lower()}/', - include(f'portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.urls', namespace='custom') + f"api/{settings.PORTAL_NAMESPACE.lower()}/", + include(f"portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.urls", namespace="custom"), ) ) # internal docs diff --git a/server/portal/utils/contextprocessors.py b/server/portal/utils/contextprocessors.py index 5026e0927d..fdfe3b6da2 100644 --- a/server/portal/utils/contextprocessors.py +++ b/server/portal/utils/contextprocessors.py @@ -9,9 +9,9 @@ def analytics(request): render your Google Analytics tracking code template. """ context = {} - ga_prop_id = getattr(settings, 'GOOGLE_ANALYTICS_PROPERTY_ID', False) + ga_prop_id = getattr(settings, "GOOGLE_ANALYTICS_PROPERTY_ID", False) if not settings.DEBUG and ga_prop_id: - context['GOOGLE_ANALYTICS_PROPERTY_ID'] = ga_prop_id + context["GOOGLE_ANALYTICS_PROPERTY_ID"] = ga_prop_id return context @@ -34,15 +34,13 @@ def messages(request): unique_msgs.append(m) return { - 'messages': unique_msgs, - 'DEFAULT_MESSAGE_LEVELS': DEFAULT_LEVELS, + "messages": unique_msgs, + "DEFAULT_MESSAGE_LEVELS": DEFAULT_LEVELS, } def debug(request): context = {} if settings.DEBUG: - context = { - 'debug': True - } + context = {"debug": True} return context diff --git a/server/portal/utils/decorators.py b/server/portal/utils/decorators.py index ae3ee5e183..f5246c1d51 100644 --- a/server/portal/utils/decorators.py +++ b/server/portal/utils/decorators.py @@ -38,6 +38,7 @@ def post(self, request, **kwargs): pass ``` """ + @wraps(func) def decorated_function(request, *args, **kwargs): """Decorated function.""" @@ -56,6 +57,7 @@ def handle_uncaught_exceptions(message): :param str message: Error message for the json repsonse """ + def _decorator(fn): @wraps(fn) @@ -64,8 +66,10 @@ def wrapper(self, *args, **kw): return fn(self, *args, **kw) except Exception: logger.exception("Handling uncaught exception") - return JsonResponse({'message': message}, status=500) + return JsonResponse({"message": message}, status=500) + return wrapper + return _decorator diff --git a/server/portal/utils/encryption.py b/server/portal/utils/encryption.py index a536912395..6e61b5f70e 100644 --- a/server/portal/utils/encryption.py +++ b/server/portal/utils/encryption.py @@ -18,9 +18,9 @@ def createKeyPair(): private_key = create_private_key() - priv_key_str = export_key(private_key, 'PEM') + priv_key_str = export_key(private_key, "PEM") public_key = create_public_key(private_key) - publ_key_str = export_key(public_key, 'OpenSSH') + publ_key_str = export_key(public_key, "OpenSSH") return priv_key_str, publ_key_str @@ -43,7 +43,7 @@ def create_public_key(key): return pub_key -def export_key(key, format='PEM'): # pylint: disable=redefined-builtin +def export_key(key, format="PEM"): # pylint: disable=redefined-builtin """Exports private key :param key: RSA key @@ -53,7 +53,7 @@ def export_key(key, format='PEM'): # pylint: disable=redefined-builtin Use `format='PEM'` for exporting private keys and `format='OpenSSH' for exporting public keys """ - return key.exportKey(format).decode('utf-8') + return key.exportKey(format).decode("utf-8") def encrypt(raw): @@ -65,7 +65,7 @@ def encrypt(raw): Shamelessly copied from: https://stackoverflow.com/questions/42568262/how-to-encrypt-text-with-a-password-in-python/44212550#44212550 """ - source = raw.encode('utf-8') + source = raw.encode("utf-8") # Use hash to make sure size is appropiate key = SHA256.new(str.encode(settings.SECRET_KEY)).digest() # pylint: disable=invalid-name @@ -90,14 +90,14 @@ def decrypt(raw): key = SHA256.new(str.encode(settings.SECRET_KEY)).digest() # extract the IV from the beginning # pylint: disable=invalid-name - IV = source[:AES.block_size] + IV = source[: AES.block_size] # pylint: enable=invalid-name decryptor = AES.new(key, AES.MODE_CBC, IV) # decrypt - data = decryptor.decrypt(source[AES.block_size:]) + data = decryptor.decrypt(source[AES.block_size :]) # pick the padding value from the end; padding = data[-1] if data[-padding:] != bytes([padding]) * padding: raise ValueError("Invalid padding...") # remove the padding - return data[:-padding].decode('utf-8') + return data[:-padding].decode("utf-8") diff --git a/server/portal/utils/exceptions.py b/server/portal/utils/exceptions.py index 4fd301e84d..fb73a3b0a9 100644 --- a/server/portal/utils/exceptions.py +++ b/server/portal/utils/exceptions.py @@ -2,6 +2,7 @@ .. module:: portal.utils.exceptions :synopsis: Exceptions used across the portal """ + from requests.exceptions import RequestException from requests.models import Response @@ -30,8 +31,8 @@ class PortalException(RequestException): >>> raise PortalException("New Exception message", request = e.request, response = e.response) """ - def __init__(self, message=None, status=None, - extra=None, *args, **kwargs): + + def __init__(self, message=None, status=None, extra=None, *args, **kwargs): super(PortalException, self).__init__(*args, **kwargs) response = self.response or Response() response.status_code = status or response.status_code @@ -43,9 +44,6 @@ def __init__(self, message=None, status=None, class ApiMethodNotAllowed(PortalException): """Custom 405 Method Not Allowed Exception""" + def __init__(self, extra=None, *args, **kwargs): - super(ApiMethodNotAllowed, self).__init__( - message='Method Not Allowed', - status=405, - extra=extra - ) + super(ApiMethodNotAllowed, self).__init__(message="Method Not Allowed", status=405, extra=extra) diff --git a/server/portal/utils/fields.py b/server/portal/utils/fields.py index c84049ed6f..4645da92c3 100644 --- a/server/portal/utils/fields.py +++ b/server/portal/utils/fields.py @@ -1,9 +1,7 @@ import json from django.conf import settings -from django.contrib.postgres.fields import ( - JSONField as DjangoJSONField -) +from django.contrib.postgres.fields import JSONField as DjangoJSONField from django.db.models import Field @@ -14,10 +12,11 @@ # from https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1 -if 'sqlite' in settings.DATABASES['default']['ENGINE']: +if "sqlite" in settings.DATABASES["default"]["ENGINE"]: + class JSONField(Field): def db_type(self, connection): - return 'text' + return "text" def from_db_value(self, value, expression, connection): if value is not None: @@ -40,5 +39,6 @@ def get_prep_value(self, value): def value_to_string(self, obj): return self.value_from_object(obj) else: + class JSONField(DjangoJSONField): pass diff --git a/server/portal/utils/jwt_auth.py b/server/portal/utils/jwt_auth.py index 929320b155..32b35802e0 100644 --- a/server/portal/utils/jwt_auth.py +++ b/server/portal/utils/jwt_auth.py @@ -31,13 +31,13 @@ def _decode_jwt(jwt): key_der = b64decode(pubkey) key = load_der_public_key(key_der) except (TypeError, ValueError, UnsupportedAlgorithm): - LOGGER.exception('Could not load public key.') + LOGGER.exception("Could not load public key.") return {} try: decoded = pyjwt.decode(jwt, key, issuer=settings.AGAVE_JWT_ISSUER) except pyjwt.exceptions.DecodeError as exc: - LOGGER.exception('Could not decode JWT. %s', exc) + LOGGER.exception("Could not decode JWT. %s", exc) return {} return decoded @@ -49,10 +49,10 @@ def _get_jwt_payload(request): :return: JWT payload :rtype: str """ - payload = request.META.get(getattr(settings, 'AGAVE_JWT_HEADER', '')) + payload = request.META.get(getattr(settings, "AGAVE_JWT_HEADER", "")) if payload and isinstance(payload, text_type): # Header encoding (see RFC5987) - payload = payload.encode('iso-8859-1') + payload = payload.encode("iso-8859-1") return payload @@ -74,18 +74,15 @@ def login_user_agave_jwt(request): if not jwt_payload: return None - username = jwt_payload.get( - getattr(settings, 'AGAVE_JWT_USER_CLAIM_FIELD', ''), - '' - ) + username = jwt_payload.get(getattr(settings, "AGAVE_JWT_USER_CLAIM_FIELD", ""), "") try: user = get_user_model().objects.get(username=username) except ObjectDoesNotExist: - LOGGER.exception('Could not find JWT user: %s', username) + LOGGER.exception("Could not find JWT user: %s", username) user = None if user is not None: - user.backend = 'django.contrib.auth.backends.ModelBackend' + user.backend = "django.contrib.auth.backends.ModelBackend" login(request, user) # Refresh tapis oauth token diff --git a/server/portal/utils/translations.py b/server/portal/utils/translations.py index fbf4461bfb..57f8be959a 100644 --- a/server/portal/utils/translations.py +++ b/server/portal/utils/translations.py @@ -14,22 +14,20 @@ def url_parse_inputs(job): Translates the inputs of an Agave job to be URL encoded """ job = copy.deepcopy(job) - for key, value in job['inputs'].items(): + for key, value in job["inputs"].items(): # this could either be an array, or a string... if isinstance(value, str): parsed = urlparse(value) if parsed.scheme: - job['inputs'][key] = '{}://{}{}'.format( - parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) + job["inputs"][key] = "{}://{}{}".format(parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) else: - job['inputs'][key] = urllib.parse.quote(parsed.path) + job["inputs"][key] = urllib.parse.quote(parsed.path) else: # If array, replace it with new array where each element was parsed parsed_values = [] for input in value: parsed = urlparse(input) - input = '{}://{}{}'.format( - parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) + input = "{}://{}{}".format(parsed.scheme, parsed.netloc, urllib.parse.quote(parsed.path)) parsed_values.append(input) - job['inputs'][key] = parsed_values + job["inputs"][key] = parsed_values return job diff --git a/server/portal/utils/unit_test.py b/server/portal/utils/unit_test.py index e76c6f6768..5747537030 100644 --- a/server/portal/utils/unit_test.py +++ b/server/portal/utils/unit_test.py @@ -10,42 +10,28 @@ class TestTranslations(TestCase): """Test Translations.""" - fixtures = ['users', 'auth'] + + fixtures = ["users", "auth"] def setUp(self): """Setup.""" super(TestTranslations, self).setUp() - self.user = get_user_model().objects.get(username='username') + self.user = get_user_model().objects.get(username="username") self.job = { "inputs": { "inputFile": "agave://test.system/test file.txt", - "inputFiles": [ - "agave://test.system/test file 1.txt", - "agave://test.system/test file 2.txt" - ] + "inputFiles": ["agave://test.system/test file 1.txt", "agave://test.system/test file 2.txt"], } } def test_url_parse_inputs(self): result = url_parse_inputs(self.job) - self.assertEqual( - result["inputs"]["inputFile"], - "agave://test.system/test%20file.txt" - ) - self.assertEqual( - result["inputs"]["inputFiles"][0], - "agave://test.system/test%20file%201.txt" - ) - self.assertEqual( - result["inputs"]["inputFiles"][1], - "agave://test.system/test%20file%202.txt" - ) + self.assertEqual(result["inputs"]["inputFile"], "agave://test.system/test%20file.txt") + self.assertEqual(result["inputs"]["inputFiles"][0], "agave://test.system/test%20file%201.txt") + self.assertEqual(result["inputs"]["inputFiles"][1], "agave://test.system/test%20file%202.txt") # Assert original object has not mutated - self.assertEqual( - self.job["inputs"]["inputFile"], - "agave://test.system/test file.txt" - ) + self.assertEqual(self.job["inputs"]["inputFile"], "agave://test.system/test file.txt") self.assertNotEqual(self.job, result) @@ -53,71 +39,61 @@ class TestGroupMembership(TestCase): """Test group membership helper.""" def test_check_group_membership(self): - user = get_user_model().objects.create_user(username='group_user') - group = Group.objects.create(name='Project Admin') + user = get_user_model().objects.create_user(username="group_user") + group = Group.objects.create(name="Project Admin") user.groups.add(group) - self.assertTrue(check_group_membership(user, 'Project Admin')) - self.assertFalse(check_group_membership(user, 'Other Group')) + self.assertTrue(check_group_membership(user, "Project Admin")) + self.assertFalse(check_group_membership(user, "Other Group")) class TestAgaveJWTAuth(TestCase): """Test Agave JWT Auth.""" - fixtures = ['users', 'auth'] + fixtures = ["users", "auth"] - @patch('portal.utils.jwt_auth.login') - @patch('portal.utils.jwt_auth._get_jwt_payload', return_value='payload') + @patch("portal.utils.jwt_auth.login") + @patch("portal.utils.jwt_auth._get_jwt_payload", return_value="payload") @patch( - 'portal.utils.jwt_auth._decode_jwt', + "portal.utils.jwt_auth._decode_jwt", return_value={ - 'http://wso2.org/claims/usertype': 'APPLICATION_USER', - 'http://wso2.org/claims/tier': 'Unlimited', - 'iss': 'wso2.org/products/am', - 'http://wso2.org/claims/lastname': 'Portal', - 'http://wso2.org/claims/applicationtier': 'Unlimited', - 'http://wso2.org/claims/applicationid': '89', - 'http://wso2.org/claims/subscriber': 'PORTALS/wma_prtl', - 'http://wso2.org/claims/enduserTenantId': '-1234', - 'http://wso2.org/claims/emailaddress': 'aci-wma@tacc.utexas.edu', - 'http://wso2.org/claims/version': 'v2', - 'http://wso2.org/claims/keytype': 'PRODUCTION', - 'exp': 1554836586702, - 'http://wso2.org/claims/applicationname': 'josuebc.cli', - 'http://wso2.org/claims/role': ( - 'Internal/PORTALS_wma_prtl_cep.dev_PRODUCTION,' - ), - 'http://wso2.org/claims/givenname': 'WMA', - 'http://wso2.org/claims/apicontext': '/projects-cep/v2', - 'http://wso2.org/claims/fullname': 'wma_prtl', - 'http://wso2.org/claims/enduser': 'wma_prtl@carbon.super' - } + "http://wso2.org/claims/usertype": "APPLICATION_USER", + "http://wso2.org/claims/tier": "Unlimited", + "iss": "wso2.org/products/am", + "http://wso2.org/claims/lastname": "Portal", + "http://wso2.org/claims/applicationtier": "Unlimited", + "http://wso2.org/claims/applicationid": "89", + "http://wso2.org/claims/subscriber": "PORTALS/wma_prtl", + "http://wso2.org/claims/enduserTenantId": "-1234", + "http://wso2.org/claims/emailaddress": "aci-wma@tacc.utexas.edu", + "http://wso2.org/claims/version": "v2", + "http://wso2.org/claims/keytype": "PRODUCTION", + "exp": 1554836586702, + "http://wso2.org/claims/applicationname": "josuebc.cli", + "http://wso2.org/claims/role": ("Internal/PORTALS_wma_prtl_cep.dev_PRODUCTION,"), + "http://wso2.org/claims/givenname": "WMA", + "http://wso2.org/claims/apicontext": "/projects-cep/v2", + "http://wso2.org/claims/fullname": "wma_prtl", + "http://wso2.org/claims/enduser": "wma_prtl@carbon.super", + }, ) - def test_login_user_agave_jwt( - self, - mock_decode_jwt, - mock_get_jwt_payload, - mock_login - ): + def test_login_user_agave_jwt(self, mock_decode_jwt, mock_get_jwt_payload, mock_login): """Test login_user_agave_jwt. If everything goes well, the request is logged in and passed to the view. """ mock_request = Mock() login_user_agave_jwt(mock_request) - user = get_user_model().objects.get(username='wma_prtl') + user = get_user_model().objects.get(username="wma_prtl") mock_get_jwt_payload.assert_called_with(mock_request) mock_decode_jwt.assert_called_with(mock_get_jwt_payload()) self.assertEqual(len(mock_login.mock_calls), 1) mock_login.assert_called_with(mock_request, user) - @override_settings( - AGAVE_JWT_PUBKEY='pub-key==', - AGAVE_JWT_HEADER='x_agave_header' - ) - @patch('portal.utils.jwt_auth.login') - @patch('portal.utils.jwt_auth._get_jwt_payload', return_value=None) + @override_settings(AGAVE_JWT_PUBKEY="pub-key==", AGAVE_JWT_HEADER="x_agave_header") + @patch("portal.utils.jwt_auth.login") + @patch("portal.utils.jwt_auth._get_jwt_payload", return_value=None) def test_agave_jwt_no_payload(self, mock_get_jwt_payload, mock_login): """Test Agave jwt with no payload. @@ -127,12 +103,9 @@ def test_agave_jwt_no_payload(self, mock_get_jwt_payload, mock_login): login_user_agave_jwt(mock_request) self.assertEqual(len(mock_login.mock_calls), 0) - @override_settings( - AGAVE_JWT_PUBKEY='pub-key==', - AGAVE_JWT_HEADER='x_agave_header' - ) - @patch('portal.utils.jwt_auth.login') - @patch('portal.utils.jwt_auth._get_jwt_payload', return_value='payload') + @override_settings(AGAVE_JWT_PUBKEY="pub-key==", AGAVE_JWT_HEADER="x_agave_header") + @patch("portal.utils.jwt_auth.login") + @patch("portal.utils.jwt_auth._get_jwt_payload", return_value="payload") def test_agave_jwt_no_decode(self, mock_get_jwt_payload, mock_login): """Test Agave jwt with nothing decoded. @@ -142,26 +115,22 @@ def test_agave_jwt_no_decode(self, mock_get_jwt_payload, mock_login): login_user_agave_jwt(mock_request) self.assertEqual(len(mock_login.mock_calls), 0) - @patch('portal.utils.jwt_auth.login') - @patch('portal.utils.jwt_auth._get_jwt_payload', return_value='payload') - @patch('portal.utils.jwt_auth._decode_jwt', return_value={'http://wso2.org/claims/fullname': 'wma_prtl'}) - @patch('portal.apps.auth.models.TapisOAuthToken.client', autospec=True) - def test_agave_jwt_expired_token(self, - mock_client, - mock_decode_jwt, - mock_get_jwt_payload, - mock_login): - - user = get_user_model().objects.get(username='wma_prtl') + @patch("portal.utils.jwt_auth.login") + @patch("portal.utils.jwt_auth._get_jwt_payload", return_value="payload") + @patch("portal.utils.jwt_auth._decode_jwt", return_value={"http://wso2.org/claims/fullname": "wma_prtl"}) + @patch("portal.apps.auth.models.TapisOAuthToken.client", autospec=True) + def test_agave_jwt_expired_token(self, mock_client, mock_decode_jwt, mock_get_jwt_payload, mock_login): + + user = get_user_model().objects.get(username="wma_prtl") user.tapis_oauth.expires_in = 0 user.tapis_oauth.save() self.assertTrue(user.tapis_oauth.expired) - mock_client.access_token.access_token = "XYZXYZXYZ", + mock_client.access_token.access_token = ("XYZXYZXYZ",) mock_client.access_token.expires_in.return_value = timedelta(seconds=2000) mock_request = Mock() login_user_agave_jwt(mock_request) - user = get_user_model().objects.get(username='wma_prtl') + user = get_user_model().objects.get(username="wma_prtl") mock_client.refresh_tokens.assert_called_once_with() self.assertFalse(user.tapis_oauth.expired) diff --git a/server/portal/views/base.py b/server/portal/views/base.py index a6cdb7cf33..30d849c3e1 100644 --- a/server/portal/views/base.py +++ b/server/portal/views/base.py @@ -36,16 +36,10 @@ def dispatch(self, request, *args, **kwargs): message = e.response.reason extra = e.extra if status != 404: - logger.error( - '%s: %s', - message, - e.response.text, - exc_info=True, - extra=extra - ) + logger.error("%s: %s", message, e.response.text, exc_info=True, extra=extra) else: - logger.info('Error %s', message, exc_info=True, extra=extra) - return JsonResponse({'message': message}, status=400) + logger.info("Error %s", message, exc_info=True, extra=extra) + return JsonResponse({"message": message}, status=400) except (ConnectionError, HTTPError, BaseTapyException) as e: # status code and json content from ConnectionError/HTTPError exceptions # are used in the returned response. Note: the handling of these two exceptions @@ -61,39 +55,28 @@ def dispatch(self, request, *args, **kwargs): message = "Unknown Error" if status in [404, 403]: logger.warning( - '%s: %s', + "%s: %s", message, e.response.text, exc_info=True, - extra={ - 'username': request.user.username, - 'session_key': request.session.session_key - } + extra={"username": request.user.username, "session_key": request.session.session_key}, ) else: logger.error( - '%s: %s', + "%s: %s", message, e.response.text, exc_info=True, - extra={ - 'username': request.user.username, - 'session_key': request.session.session_key - } + extra={"username": request.user.username, "session_key": request.session.session_key}, ) else: logger.error( e, exc_info=True, - extra={ - 'username': request.user.username, - 'session_key': request.session.session_key - } + extra={"username": request.user.username, "session_key": request.session.session_key}, ) message = str(e) - return JsonResponse({'message': message}, status=status) + return JsonResponse({"message": message}, status=status) except Exception as e: # pylint: disable=broad-except logger.error(e, exc_info=True) - return JsonResponse( - {'message': "Something went wrong here..."}, - status=500) + return JsonResponse({"message": "Something went wrong here..."}, status=500) diff --git a/server/portal/views/unit_test.py b/server/portal/views/unit_test.py index f4ffbbf69a..a9b120704a 100644 --- a/server/portal/views/unit_test.py +++ b/server/portal/views/unit_test.py @@ -8,7 +8,7 @@ # route to be used for testing purposes -API_ROUTE = '/api/system-monitor/' +API_ROUTE = "/api/system-monitor/" # arbitrary status code that is not 403 or 404 for testing purposes NON_403_404 = 401 @@ -16,10 +16,10 @@ @pytest.fixture def api_method_mock(mocker): - ''' + """ Mock of an method in our API_ROUTE to allow us to test error handling and responses - ''' - workbench_state = mocker.patch('portal.apps.system_monitor.views.SysmonDataView.get') + """ + workbench_state = mocker.patch("portal.apps.system_monitor.views.SysmonDataView.get") yield workbench_state @@ -44,7 +44,7 @@ def test_custom_api_exception(client, api_method_mock): response = client.get(API_ROUTE) assert response.status_code == 400 result = json.loads(response.content) - assert result == {'message': 'problem'} + assert result == {"message": "problem"} api_method_mock.side_effect = ApiException(status=404, message="problem") response = client.get(API_ROUTE) @@ -58,7 +58,7 @@ def test_connectionerror_httperror_no_response_in_exception(ExceptionClass, clie api_method_mock.side_effect = ExceptionClass response = client.get(API_ROUTE) - assert json.loads(response.content) == {'message': ''} + assert json.loads(response.content) == {"message": ""} assert response.status_code == 500 @@ -70,12 +70,12 @@ def test_connectionerror_httperror_with_response(ExceptionClass, status_code, cl # NOTE: this is important as our client code uses these status codes in reacting to tapis behavior! test_response = requests.Response() - test_response._content = json.dumps({"message": "Custom error message"}).encode('utf-8') + test_response._content = json.dumps({"message": "Custom error message"}).encode("utf-8") test_response.status_code = status_code api_method_mock.side_effect = ExceptionClass(response=test_response) response = client.get(API_ROUTE) - assert json.loads(response.content) == {'message': 'Custom error message'} + assert json.loads(response.content) == {"message": "Custom error message"} assert response.status_code == status_code @@ -83,12 +83,12 @@ def test_connectionerror_httperror_with_response(ExceptionClass, status_code, cl @pytest.mark.parametrize("status_code", [403, 404, NON_403_404]) def test_connectionerror_httperror_non_json_content(ExceptionClass, status_code, client, api_method_mock): test_response = requests.Response() - test_response._content = "Non json error content".encode('utf-8') + test_response._content = "Non json error content".encode("utf-8") test_response.status_code = status_code api_method_mock.side_effect = requests.exceptions.HTTPError(response=test_response) response = client.get(API_ROUTE) - assert json.loads(response.content) == {'message': 'Unknown Error'} + assert json.loads(response.content) == {"message": "Unknown Error"} assert response.status_code == status_code @@ -96,24 +96,24 @@ def test_portal_lib_exception(client, api_method_mock): api_method_mock.side_effect = PortalLibException response = client.get(API_ROUTE) assert response.status_code == 500 - assert json.loads(response.content) == {'message': 'Something went wrong here...'} + assert json.loads(response.content) == {"message": "Something went wrong here..."} def test_django_exceptions_that_squash(client, api_method_mock): api_method_mock.side_effect = ObjectDoesNotExist response = client.get(API_ROUTE) assert response.status_code == 500 - assert json.loads(response.content) == {'message': 'Something went wrong here...'} + assert json.loads(response.content) == {"message": "Something went wrong here..."} def test_exception(client, api_method_mock): api_method_mock.side_effect = Exception response = client.get(API_ROUTE) assert response.status_code == 500 - assert json.loads(response.content) == {'message': 'Something went wrong here...'} + assert json.loads(response.content) == {"message": "Something went wrong here..."} def test_health_check(client): - response = client.get('/core/health-check') + response = client.get("/core/health-check") assert response.status_code == 200 - assert json.loads(response.content) == {'status': 'healthy'} + assert json.loads(response.content) == {"status": "healthy"} diff --git a/server/portal/views/views.py b/server/portal/views/views.py index 795a481887..c5643b08eb 100644 --- a/server/portal/views/views.py +++ b/server/portal/views/views.py @@ -10,27 +10,27 @@ def project_version(request): try: - with open('.git/HEAD') as f: + with open(".git/HEAD") as f: head = f.readline() - if 'ref:' in head: + if "ref:" in head: # we're on a branch - branch = head.split(':')[1].strip() - with open('.git/{0}'.format(branch)) as f: - version = '{}:{}'.format(branch, f.readline()) + branch = head.split(":")[1].strip() + with open(".git/{0}".format(branch)) as f: + version = "{}:{}".format(branch, f.readline()) else: # we're in a detached head, e.g., a tag. would be nice to show tag name... version = head except IOError: - logger.warning('Unable to read project version from git HEAD') - version = 'UNKNOWN' + logger.warning("Unable to read project version from git HEAD") + version = "UNKNOWN" - return HttpResponse(version, content_type='text/plain') + return HttpResponse(version, content_type="text/plain") def health_check(request): - health_status = {'status': 'healthy'} + health_status = {"status": "healthy"} return JsonResponse(health_status) @@ -39,9 +39,9 @@ def serve_docs(request, path): file_path = os.path.join(settings.INTERNAL_DOCS_ROOT, path) if os.path.isdir(file_path): # For mkdocs directories, append index.html - index_file = os.path.join(file_path, 'index.html') + index_file = os.path.join(file_path, "index.html") if os.path.isfile(index_file): - path = os.path.join(path, 'index.html') + path = os.path.join(path, "index.html") else: raise Http404("Directory index not found") diff --git a/server/portal/wsgi.py b/server/portal/wsgi.py index 64a34eae08..6d1f9a340c 100644 --- a/server/portal/wsgi.py +++ b/server/portal/wsgi.py @@ -10,5 +10,5 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portal.settings.settings") application = get_wsgi_application() diff --git a/server/pyproject.toml b/server/pyproject.toml index 270d98b7a8..c5fa1ff5ed 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -6,7 +6,7 @@ authors = [ { name = "TACC-WMA", email = "wma-portals@tacc.utexas.edu" }, ] readme = "README.md" -requires-python = "^3.12" +requires-python = "==3.12.*" dynamic = [ "dependencies" ] [tool.poetry] @@ -64,9 +64,19 @@ mock = "^5.0.2" pytest-cov = "^4.0.0" pytest-django = "^4.5.2" pytest-asyncio = "^0.21.1" -flake8 = "^6.0.0" coverage = "^7.2.5" requests-mock = "^1.10.0" +ruff = "^0.16.5" + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP"] + +[tool.ruff.lint.per-file-ignores] +"**/migrations/*.py" = ["E501", "RUF012"] [build-system] requires = ["poetry-core"]