diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d769e45b..10c0a1ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,4 +82,4 @@ jobs: run: pytest -vs workflow/tests/ - name: Test workflow with script shell: bash --login {0} - run: bash workflow.sh --config workflow/config/default_config.yaml --execute all + run: bash workflow.sh --config workflow/config/config.yaml --execute all diff --git a/containers/cytoscape/src/connect_cytoscape.py b/containers/cytoscape/src/connect_cytoscape.py index f0fca780..8addc7fa 100644 --- a/containers/cytoscape/src/connect_cytoscape.py +++ b/containers/cytoscape/src/connect_cytoscape.py @@ -9,17 +9,18 @@ ''' import argparse -from py2cytoscape import cyrest -from requests.exceptions import ConnectionError -import time +import os +import subprocess import sys -import requests +import time from json import JSONDecodeError -import subprocess -import os -from py2cytoscape.data.cyrest_client import CyRestClient from pathlib import Path +import requests +from py2cytoscape import cyrest +from py2cytoscape.data.cyrest_client import CyRestClient +from requests.exceptions import ConnectionError + ########################################################################################################## ########################################################################################################## ''' diff --git a/tps_workflow.py b/tps_workflow.py index 4c50a80b..b867fec0 100644 --- a/tps_workflow.py +++ b/tps_workflow.py @@ -1,3 +1,18 @@ +''' +The TPS Workflow is an encapsulated python-centric wrapper for the TPS core +software. The TPS input settings are defined in a config file that is +passed into the /workflow.sh driver script to start the workflow run. +Once complete, all tps output files are placed in the specified output directory + +usage: bash workflow.sh --config [config.yml] --execute [all|annotations|tps] + +To run full workflow : + bash workflow.sh --config workflow/config/config.yaml --execute all + +@author : Adam Shedivy + +''' + import yaml import argparse import workflow diff --git a/workflow/README.md b/workflow/README.md index 25d14276..b669f5dc 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -2,7 +2,7 @@ **TPS workflow overview** -![Overview](../workflow/images/TPS_workflow.png) +![Overview](../workflow/images/TPS_diagram2.svg) The TPS/workflow module utilizes the TPS repository structure in order to run the TPS software end-to-end in order to produce Cytoscape sessions. @@ -57,4 +57,4 @@ University of California, San Diego Trey Ideker Lab. Code in `docker_util.py` is derived from https://github.com/Reed-CompBio/spras available under the MIT license. -The Cytoscape `Dockerfile` is derived from https://github.com/cytoscape/docker-cytoscape-desktop \ No newline at end of file +The Cytoscape `Dockerfile` is derived from https://github.com/cytoscape/docker-cytoscape-desktop diff --git a/workflow/__init__.py b/workflow/__init__.py index 10b02131..5cc4a5e2 100644 --- a/workflow/__init__.py +++ b/workflow/__init__.py @@ -3,11 +3,12 @@ https://github.com/Murali-group/Beeline/blob/master/BLRun/__init__.py ''' -import yaml import os -from typing import Dict, Any, Union, List -from pathlib import Path import sys +from pathlib import Path +from typing import Any, Dict, List, Union + +import yaml # define defaults to be used during initialization TPS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -108,7 +109,7 @@ def __make_out_folder(self, folder: Union[str, Path]) -> None: if not os.path.exists(folder): os.makedirs(folder) - def __get_out_folder(self) -> None: + def __get_out_folder(self) -> Union[Path, str]: """ if output folder provided in config: return folder diff --git a/workflow/config/default_config.yaml b/workflow/config/config.yaml similarity index 97% rename from workflow/config/default_config.yaml rename to workflow/config/config.yaml index 863f3157..a408726a 100644 --- a/workflow/config/default_config.yaml +++ b/workflow/config/config.yaml @@ -1,42 +1,42 @@ -Docker: - TPS: True - Cytoscape: True - -TPS: - required: - network: data/networks/input-network.tsv - timeseries: data/timeseries/median-time-series.tsv - firstscores: data/timeseries/p-values-first.tsv - prevscores: data/timeseries/p-values-prev.tsv - source: EGF_HUMAN - threshold: 0.01 - outlabel: "test-runner-full" - outfolder: output - optional: - partialmodel: data/resources/kinase-substrate-interactions.sif - peptidemap: data/timeseries/peptide-mapping.tsv - solver: None - slack: None - bitvect: None - flags: #[--no-connectivity, --no-temporality, --no-monotonicity] -Annotations: - peptideMapFile: data/timeseries/peptide-mapping.tsv - timeSeriesFile: data/timeseries/log2-fold-change.txt - peptideFirstScores: data/timeseries/p-values-first.tsv - peptidePrevScoreFile: data/timeseries/p-values-prev.tsv - windowsFile: "output_placeholder" - networkFile: "output_placeholder" - goldStandardFile: data/resources/eight-egfr-reference-all.txt - pvalThresh: 0.01 - logTransform: False - styleTemplateFile: data/templates/tps_style_template.xml - outAnnotFile: "test-annot-full.txt" - outStyleFile: "style_output.xml" - addZero: True # default - logDefault: -1.0 # default - repairMissing: False # default -# These settings are ignored when Cytoscape is run through Docker -Cytoscape: - path: ~\Documents\Comp_bio\Cytoscape_v3.7.1 - session: "tps_session_full.cys" +Docker: + TPS: True + Cytoscape: True + +TPS: + required: + network: data/networks/input-network.tsv + timeseries: data/timeseries/median-time-series.tsv + firstscores: data/timeseries/p-values-first.tsv + prevscores: data/timeseries/p-values-prev.tsv + source: EGF_HUMAN + threshold: 0.01 + outlabel: "test-runner-full" + outfolder: output + optional: + partialmodel: data/resources/kinase-substrate-interactions.sif + peptidemap: data/timeseries/peptide-mapping.tsv + solver: None + slack: None + bitvect: None + flags: #[--no-connectivity, --no-temporality, --no-monotonicity] +Annotations: + peptideMapFile: data/timeseries/peptide-mapping.tsv + timeSeriesFile: data/timeseries/log2-fold-change.txt + peptideFirstScores: data/timeseries/p-values-first.tsv + peptidePrevScoreFile: data/timeseries/p-values-prev.tsv + windowsFile: "output_placeholder" + networkFile: "output_placeholder" + goldStandardFile: data/resources/eight-egfr-reference-all.txt + pvalThresh: 0.01 + logTransform: False + styleTemplateFile: data/templates/tps_style_template.xml + outAnnotFile: "test-annot-full.txt" + outStyleFile: "style_output.xml" + addZero: True # default + logDefault: -1.0 # default + repairMissing: False # default +# These settings are ignored when Cytoscape is run through Docker +Cytoscape: + path: ~\Documents\Comp_bio\Cytoscape_v3.7.1 + session: "tps_session_full.cys" name: "Cytoscape.exe" \ No newline at end of file diff --git a/workflow/docker_util.py b/workflow/docker_util.py index d2fd70ba..f429f240 100644 --- a/workflow/docker_util.py +++ b/workflow/docker_util.py @@ -1,11 +1,19 @@ -import docker -import re +''' +This file defines the utility functions and classes used for interation +with Docker + +@author : Adam Shedivy + +''' + import functools -from pathlib import PurePath, Path -from typing import Dict, List, Optional, Any +import re +from pathlib import Path, PurePath +from typing import Any, Dict, List, Optional -from workflow.workflow import * +import docker +from workflow.workflow import * CLIENTS = { 'TPS' : 'ajshedivy/tps', @@ -39,13 +47,17 @@ def prepare_path_docker(orig_path: PurePath) -> str: def is_container_running(container_name: str) -> bool: - """check if docker container is running + """check if a container is running - Args: - container_name (str): name of container + Parameters + ---------- + container_name : str + name of container assigned after calling docker run - Returns: - bool + Returns + ------- + bool + True if running, else False """ client = docker.from_env() @@ -72,6 +84,39 @@ def is_container_running(container_name: str) -> bool: class DockerClient(object): + """ + A class used to prepare docker containers for TPS workflow. Used by workflow/workflow.py + + ... + + Attributes + ---------- + client_name : str + string name of container to prepare + client : str + name of container tag + worflow_runner : Optional[WorkflowRun] + workflow object with input parameters and settings + outfolder : Union[str, Pathlike] + ouput folder location + cytoscape : Optional[Visualize] + cytoscape input and parameter settings + + Methods + ------- + prepare_tps(self) -> Dict[str, Any] + prepare TPS docker input settings + + prepare_cytoscape(self) -> Dict[str, Any] + prepare Cytoscape docker input settings + + _build_cytoscape_command(self) -> List[str] + helper function for generating cytoscape bash run command + + run_docker(self) -> None + run specified docker client + + """ def __init__(self, client: str, workflow_settings: Optional[WorkflowRun] = None, cytoscape: Optional[Visualize] = None) -> None: @@ -87,10 +132,12 @@ def __init__(self, client: str, } def prepare_tps(self) -> Dict[str, Any]: - """Helper function for preparing TPS Container Settings + """helper function for generating tps docker input settings. Used when client_name = 'TPS' - Returns: - Dict[str, Any]: collection of Docker Settings + Returns + ------- + Dict[str, Any] + dictionary of docker input settings used by run_docker """ container_settings = { 'work_dir':None, @@ -123,8 +170,9 @@ def prepare_cytoscape(self) -> Dict[str, Any]: - style file - annotations file - network file - - Returns: + Returns + ------- + Dict[str, Any] Dict[str, Any]: collection of Docker Settings """ container_settings = { @@ -139,7 +187,7 @@ def prepare_cytoscape(self) -> Dict[str, Any]: output_path = prepare_path_docker(work_dir)+'/'+self.workflow_runner.out_folder container_settings['work_dir'] = work_dir - container_settings['command'] = self.build_cytoscape_command() + container_settings['command'] = self._build_cytoscape_command() container_settings['volumes'] = { output_path : {'bind': '/cytoscape/'+self.workflow_runner.out_folder, 'mode': 'rw'} } @@ -152,7 +200,14 @@ def prepare_cytoscape(self) -> Dict[str, Any]: container_settings['container_wdir'] = '/cytoscape' return container_settings - def build_cytoscape_command(self) -> List[str]: + def _build_cytoscape_command(self) -> List[str]: + """private helper function for build cytoscape run command used in Cytoscape Docker container + + Returns + ------- + List[str] + list of bash args + """ command = ['bash', 'connect.sh'] args = { '--save': self.workflow_runner.out_folder + SEP + os.path.basename(self.cytoscape.save_file), @@ -165,6 +220,8 @@ def build_cytoscape_command(self) -> List[str]: return command def run_docker(self) -> None: + """driver function for running specified docker client""" + if is_container_running(self.client_name): print(f"container: {self.client} is running, please stop container and run workflow again") sys.exit() diff --git a/workflow/images/TPS_diagram2.png b/workflow/images/TPS_diagram2.png new file mode 100644 index 00000000..c1605228 Binary files /dev/null and b/workflow/images/TPS_diagram2.png differ diff --git a/workflow/images/TPS_diagram2.svg b/workflow/images/TPS_diagram2.svg new file mode 100644 index 00000000..bc8e0f15 --- /dev/null +++ b/workflow/images/TPS_diagram2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/workflow/tests/test_workflow.py b/workflow/tests/test_workflow.py index 9dacb2e4..2f944666 100644 --- a/workflow/tests/test_workflow.py +++ b/workflow/tests/test_workflow.py @@ -52,7 +52,7 @@ def test_annotations(self, config_type): assert os.path.isfile(file) os.remove(file) - @pytest.mark.parametrize('config_type', ['default', 'tps-no-container']) + @pytest.mark.parametrize('config_type', ['default']) def test_full(self, config_type): config = os.path.join(self.tps_root,'workflow', 'tests', 'standards', CONFIGS[config_type]) call = ['python', self.tps_main, diff --git a/workflow/workflow.py b/workflow/workflow.py index b882e341..cab745c5 100644 --- a/workflow/workflow.py +++ b/workflow/workflow.py @@ -1,30 +1,55 @@ +''' +This file defines the helper classes and functions used during the exectuion of the +TPS workflow + +@author : Adam Shedivy + +''' -from pathlib import Path -import subprocess import glob import os -import time +import subprocess import sys +import time +from json.decoder import JSONDecodeError +from pathlib import Path +from typing import Any, Dict, Union + import psutil +from containers.cytoscape.src.connect_cytoscape import load_tps_outputs +from py2cytoscape import cyrest +from requests.exceptions import ConnectionError, HTTPError + from workflow import WorkflowRun from workflow.utilities import PrepTemporalCytoscapeTPS -from requests.exceptions import ConnectionError -from requests.exceptions import HTTPError -from json.decoder import JSONDecodeError -from py2cytoscape import cyrest -from typing import Dict, Any -from containers.cytoscape.src.connect_cytoscape import load_tps_outputs class Runner(object): - ''' + """ the Runner object encapsulates the WorkflowRun object in order to execute the TPS software - ''' + + ... + + Attributes + ---------- + worflow_runner : WorflowRun + object that encapsulates TPS workflow settings + + Methods + ------- + run_tps(self) -> 'Runner' + function to execute TPS software locally or with docker client + + __check_outputs(self) -> Dict[str, Path] + private method for checking files generated by running TPS + """ + def __init__(self, workflow_runner: WorkflowRun): self.workflow_runner = workflow_runner def run_tps(self) -> 'Runner': + if self.workflow_runner.docker_settings.use_tps: print('using TPS Docker Container') # only import Docker_client when needed @@ -64,16 +89,45 @@ def __check_outputs(self) -> Dict[str, Path]: print('error has occurred after run') return files - - -class Annotations: ''' the Annotations object generates node annotations from defined annotations input settings and a Runner object (in order to pass outputs of TPS) ''' - def __init__(self, annot_input_settings: Dict[str, Any], - runner: Runner): +class Annotations: + """ + the Annotations object generates node annotations from defined annotations + input settings and a Runner object (in order to pass outputs of TPS) + + ... + + Attributes + ---------- + annot_input_settings : Dict[str, Any] + dictionary of anntation settings specified in config file + runner : Runner + Runner object used to run TPS + output_files : Dict[str, Path] + list of output files + out_folder : Union[Path, str] + ouput folder location + out_annot_file : Path + annotation file + out_style_file : Path + style file + + Methods + ------- + generate_annotations(self) -> 'Annotations' + generation node annoations from TPS run + + get_datatypes(self) -> str + getter + + + """ + + def __init__(self, annot_input_settings: Dict[str, Any], runner: Runner): self.annot_input_settings = annot_input_settings self.runner = runner self.output_files = self.runner.output_files @@ -82,6 +136,11 @@ def __init__(self, annot_input_settings: Dict[str, Any], self.out_style_file = os.path.join(self.out_folder, self.annot_input_settings['outStyleFile']) def generate_annotations(self) -> 'Annotations': + """_summary_ + + Returns: + Annotations: _description_ + """ print(f'output folder: {self.out_folder}') _, self.cytoscape_datatypes_ = PrepTemporalCytoscapeTPS( self.annot_input_settings['peptideMapFile'], @@ -104,13 +163,39 @@ def generate_annotations(self) -> 'Annotations': def get_datatypes(self) -> str: return self.cytoscape_datatypes_ - -class Visualize: ''' the Visualize object is a set of methods to import TPS output files and generated node annotations into a local Cytoscape session ''' +class Visualize: + """ + the Visualize object is a set of methods to import TPS output files + and generated node annotations into a local Cytoscape session + + ... + + Attributes (__init__ input) + ---------- + cytoscape_input_settings : Dict[str, Any] + cytoscape input settings parsed from config file + runner : Runner + Runner object from TPS + annotations : Annotations + object with information about annotations files generated by workflow + + Methods + ------- + load_local_cytoscape(self) -> 'Visualize' + function for loading outputs from workflow into local Cytoscape session + + load_docker_cytoscape(self) -> 'Visualize' + function for loading outputs from workflow into remote cytpscape session using docker + + load_cytoscape(self) -> None + driver function + """ + def __init__(self, cytoscape_input_settings: Dict[str, Any], runner: Runner, annotations: Annotations): self.cytoscape_input_settings = cytoscape_input_settings @@ -126,11 +211,37 @@ def __init__(self, cytoscape_input_settings: Dict[str, Any], self.style_file = os.path.abspath(self.annotations.out_style_file) def __process_exists(self, process_name) -> bool: + """check if Cytoscape is running locally + + Parameters + ---------- + process_name : str + Cytoscape.exe + + Returns + ------- + bool + True if running, else False + """ if process_name in (p.name() for p in psutil.process_iter()): return True return False - def __find_path(self, name, path) -> Path: + def __find_path(self, name: str, path: Union[str, os.PathLike]) -> Path: + """find file at given starting path + + Parameters + ---------- + name : str + name of file (Cytoscape.exe) + path : Union[str, os.PathLike] + starting directory for search + + Returns + ------- + Path + absolute path for filename + """ result = [] for root, dirs, files in os.walk(path): if name in files: @@ -141,7 +252,9 @@ def __find_path(self, name, path) -> Path: return None return result[0] - def check_cytpscape(self) -> None: + def _check_cytpscape(self) -> None: + """check local Cytoscape process, start if not running""" + check = self.__process_exists(self.cytoscape_input_settings.name) if not check: @@ -156,8 +269,8 @@ def check_cytpscape(self) -> None: else: print('Cytoscape already running') - def load_local_cytoscape(self) -> None: - self.check_cytpscape() + def load_local_cytoscape(self) -> 'Visualize': + self._check_cytpscape() connection_count = 0 json_count = 0 http_count = 0 @@ -197,6 +310,7 @@ def load_docker_cytoscape(self) -> 'Visualize': return self def load_cytoscape(self) -> None: + """driver function for generating Cytoscape session""" if self.runner.workflow_runner.docker_settings.use_cytoscape: # docker side step self.load_docker_cytoscape()