diff --git a/docs/user_guide/cli.rst b/docs/user_guide/cli.rst index a75658155..4733a4223 100644 --- a/docs/user_guide/cli.rst +++ b/docs/user_guide/cli.rst @@ -143,4 +143,9 @@ Configures the directory at which PipelineWise expects to find installed tap and `PIPELINEWISE_CONFIG_DIRECTORY` """"""""""""""""""""""""""""""" -Overrides the default directory at which PipelineWise expects to find configuration files generated by `pipelinewise import`. +Overrides the default directory at which PipelineWise expects to find configuration files generated by ``pipelinewise import``. + +`PIPELINEWISE_TEMP_DIRECTORY` +""""""""""""""""""""""""""""" + +Overrides the default directory which PipelineWise uses for temporary files. diff --git a/pipelinewise/cli/__init__.py b/pipelinewise/cli/__init__.py index 9f42b2d42..4b6173b9e 100644 --- a/pipelinewise/cli/__init__.py +++ b/pipelinewise/cli/__init__.py @@ -26,6 +26,7 @@ PIPELINEWISE_HOME = os.path.abspath( os.environ.setdefault('PIPELINEWISE_HOME', PIPELINEWISE_DEFAULT_HOME) ) +TEMP_DIR = os.environ.get('PIPELINEWISE_TEMP_DIRECTORY') VENV_DIR = os.path.join(PIPELINEWISE_HOME, '.virtualenvs') COMMANDS = [ 'init', @@ -248,7 +249,9 @@ def main(): profiler, profiling_dir = __init_profiler(args.profiler, logger) - ppw_instance = PipelineWise(args, CONFIG_DIR, VENV_DIR, profiling_dir) + ppw_instance = PipelineWise( + args, CONFIG_DIR, VENV_DIR, temp_dir=TEMP_DIR, profiling_dir=profiling_dir + ) try: getattr(ppw_instance, args.command)() diff --git a/pipelinewise/cli/config.py b/pipelinewise/cli/config.py index 5e31ce259..20b725372 100644 --- a/pipelinewise/cli/config.py +++ b/pipelinewise/cli/config.py @@ -15,7 +15,7 @@ class Config: """PipelineWise Configuration Class""" - def __init__(self, config_dir): + def __init__(self, config_dir, temp_dir=None): """ Class Constructor @@ -24,12 +24,15 @@ def __init__(self, config_dir): self.logger = logging.getLogger(__name__) self.config_dir = config_dir self.config_path = os.path.join(self.config_dir, 'config.json') + self.temp_dir = temp_dir + if self.temp_dir is None: + self.temp_dir = os.path.join(self.config_dir, 'tmp') self.global_config = {} self.targets = {} @classmethod # pylint: disable=too-many-locals - def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None): + def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None, temp_dir=None): """ Class Constructor @@ -38,7 +41,7 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None): Pipelinewise can import and generate singer configurations files from human friendly easy to understand YAML files. """ - config = cls(config_dir) + config = cls(config_dir, temp_dir=temp_dir) targets = {} taps = {} @@ -124,12 +127,6 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None): return config - def get_temp_dir(self): - """ - Returns the tap specific temp directory - """ - return os.path.join(self.config_dir, 'tmp') - def get_target_dir(self, target_id): """ Returns the absolute path of a target configuration directory @@ -177,7 +174,7 @@ def save(self): # Save every tap JSON files for tap in target['taps']: extra_config_keys = utils.get_tap_extra_config_keys( - tap, self.get_temp_dir() + tap, self.temp_dir ) self.save_tap_jsons(target, tap, extra_config_keys) @@ -404,7 +401,7 @@ def generate_inheritable_config(self, tap: Dict) -> Dict: # Generate tap inheritable_config dict tap_inheritable_config = utils.delete_empty_keys( { - 'temp_dir': self.get_temp_dir(), + 'temp_dir': self.temp_dir, 'tap_id': tap.get('id'), 'query_tag': json.dumps( { diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py index 5e7ae625e..2359ce32f 100644 --- a/pipelinewise/cli/pipelinewise.py +++ b/pipelinewise/cli/pipelinewise.py @@ -64,7 +64,7 @@ class PipelineWise: STATUS_FAILED = 'FAILED' TRANSFORM_FIELD_CONNECTOR_NAME = 'transform-field' - def __init__(self, args, config_dir, venv_dir, profiling_dir=None): + def __init__(self, args, config_dir, venv_dir, profiling_dir=None, temp_dir=None): self.profiling_mode = args.profiler self.profiling_dir = profiling_dir @@ -73,6 +73,9 @@ def __init__(self, args, config_dir, venv_dir, profiling_dir=None): self.logger = logging.getLogger(__name__) self.config_dir = config_dir self.venv_dir = venv_dir + self.temp_dir = temp_dir + if self.temp_dir is None: + self.temp_dir = os.path.join(self.config_dir, 'tmp') self.extra_log = args.extra_log self.pipelinewise_bin = os.path.join( self.venv_dir, 'cli', 'bin', 'pipelinewise' @@ -141,7 +144,7 @@ def create_consumable_target_config(self, target_config, tap_inheritable_config) # Save the new dict as JSON into a temp file tempfile_path = utils.create_temp_file( - dir=self.get_temp_dir(), prefix='target_config_', suffix='.json' + dir=self.temp_dir, prefix='target_config_', suffix='.json' )[1] utils.save_json(dict_a, tempfile_path) @@ -324,12 +327,12 @@ def create_filtered_tap_properties( if create_fallback: # Save to files: filtered and fallback properties temp_properties_path = utils.create_temp_file( - dir=self.get_temp_dir(), prefix='properties_', suffix='.json' + dir=self.temp_dir, prefix='properties_', suffix='.json' )[1] utils.save_json(properties, temp_properties_path) temp_fallback_properties_path = utils.create_temp_file( - dir=self.get_temp_dir(), prefix='properties_', suffix='.json' + dir=self.temp_dir, prefix='properties_', suffix='.json' )[1] utils.save_json(fallback_properties, temp_fallback_properties_path) @@ -342,7 +345,7 @@ def create_filtered_tap_properties( # Fallback not required: Save only the filtered properties JSON temp_properties_path = utils.create_temp_file( - dir=self.get_temp_dir(), prefix='properties_', suffix='.json' + dir=self.temp_dir, prefix='properties_', suffix='.json' )[1] utils.save_json(properties, temp_properties_path) @@ -363,12 +366,6 @@ def load_config(self): else: self.config = {} - def get_temp_dir(self): - """ - Returns the tap specific temp directory - """ - return os.path.join(self.config_dir, 'tmp') - def get_tap_dir(self, target_id, tap_id): """ Get absolute path of a tap directory @@ -1083,7 +1080,7 @@ def run_tap_fastsync( target=target, transform=transform, venv_dir=self.venv_dir, - temp_dir=self.get_temp_dir(), + temp_dir=self.temp_dir, tables=self.args.tables, profiling_mode=self.profiling_mode, profiling_dir=self.profiling_dir, @@ -1565,7 +1562,9 @@ def import_project(self): """ # Read the YAML config files and transform/save into singer compatible # JSON files in a common directory structure - config = Config.from_yamls(self.config_dir, self.args.dir, self.args.secret) + config = Config.from_yamls( + self.config_dir, self.args.dir, self.args.secret, self.temp_dir + ) config.save() # Activating tap stream selections @@ -1801,7 +1800,7 @@ def __validate_transformations( # create a temp file with the content being the given catalog object # we need this file to execute the validation cli command temp_catalog_file = utils.create_temp_file( - dir=self.get_temp_dir(), prefix='properties_', suffix='.json' + dir=self.temp_dir, prefix='properties_', suffix='.json' )[1] utils.save_json(catalog, temp_catalog_file) diff --git a/tests/units/cli/test_config.py b/tests/units/cli/test_config.py index 23f27ac21..b9c5a3a82 100644 --- a/tests/units/cli/test_config.py +++ b/tests/units/cli/test_config.py @@ -7,6 +7,7 @@ from pipelinewise.cli.errors import InvalidConfigException PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise' +PIPELINEWISE_TEST_TEMP_DIR = '/tmp/pipelinewise_tmp' # Todo: Inherit from unittest.TestCase @@ -208,10 +209,10 @@ def test_from_invalid_yamls_fails(self): def test_getters(self): """Test Config getter functions""" - config = Config(PIPELINEWISE_TEST_HOME) + config = Config(PIPELINEWISE_TEST_HOME, PIPELINEWISE_TEST_TEMP_DIR) # Target and tap directory should be g - assert config.get_temp_dir() == '{}/tmp'.format(PIPELINEWISE_TEST_HOME) + assert config.temp_dir == PIPELINEWISE_TEST_TEMP_DIR assert config.get_target_dir('test-target-id') == '{}/test-target-id'.format( PIPELINEWISE_TEST_HOME )