Add user configurable tempory directory

Author: judahrand

pipelinewise/cli/__init__.py

@@ -26,6 +26,7 @@
 PIPELINEWISE_HOME = os.path.abspath(
     os.environ.setdefault('PIPELINEWISE_HOME', PIPELINEWISE_DEFAULT_HOME)
 )
+TEMP_DIR = os.environ.get('PIPELINEWISE_TEMP_DIRECTORY')
 VENV_DIR = os.path.join(PIPELINEWISE_HOME, '.virtualenvs')
 COMMANDS = [
     'init',
@@ -248,7 +249,9 @@ def main():
 
     profiler, profiling_dir = __init_profiler(args.profiler, logger)
 
-    ppw_instance = PipelineWise(args, CONFIG_DIR, VENV_DIR, profiling_dir)
+    ppw_instance = PipelineWise(
+        args, CONFIG_DIR, VENV_DIR, temp_dir=TEMP_DIR, profiling_dir=profiling_dir
+    )
 
     try:
         getattr(ppw_instance, args.command)()

pipelinewise/cli/config.py

@@ -15,7 +15,7 @@
 class Config:
     """PipelineWise Configuration Class"""
 
-    def __init__(self, config_dir):
+    def __init__(self, config_dir, temp_dir=None):
         """
         Class Constructor
 
@@ -24,12 +24,15 @@ def __init__(self, config_dir):
         self.logger = logging.getLogger(__name__)
         self.config_dir = config_dir
         self.config_path = os.path.join(self.config_dir, 'config.json')
+        self.temp_dir = temp_dir
+        if self.temp_dir is None:
+            self.temp_dir = os.path.join(self.config_dir, 'tmp')
         self.global_config = {}
         self.targets = {}
 
     @classmethod
     # pylint: disable=too-many-locals
-    def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
+    def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None, temp_dir=None):
         """
         Class Constructor
 
@@ -38,7 +41,7 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
         Pipelinewise can import and generate singer configurations files
         from human friendly easy to understand YAML files.
         """
-        config = cls(config_dir)
+        config = cls(config_dir, temp_dir=temp_dir)
         targets = {}
         taps = {}
 
@@ -124,12 +127,6 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
 
         return config
 
-    def get_temp_dir(self):
-        """
-        Returns the tap specific temp directory
-        """
-        return os.path.join(self.config_dir, 'tmp')
-
     def get_target_dir(self, target_id):
         """
         Returns the absolute path of a target configuration directory
@@ -177,7 +174,7 @@ def save(self):
             # Save every tap JSON files
             for tap in target['taps']:
                 extra_config_keys = utils.get_tap_extra_config_keys(
-                    tap, self.get_temp_dir()
+                    tap, self.temp_dir
                 )
                 self.save_tap_jsons(target, tap, extra_config_keys)
 
@@ -404,7 +401,7 @@ def generate_inheritable_config(self, tap: Dict) -> Dict:
         # Generate tap inheritable_config dict
         tap_inheritable_config = utils.delete_empty_keys(
             {
-                'temp_dir': self.get_temp_dir(),
+                'temp_dir': self.temp_dir,
                 'tap_id': tap.get('id'),
                 'query_tag': json.dumps(
                     {

pipelinewise/cli/pipelinewise.py

@@ -64,7 +64,7 @@ class PipelineWise:
     STATUS_FAILED = 'FAILED'
     TRANSFORM_FIELD_CONNECTOR_NAME = 'transform-field'
 
-    def __init__(self, args, config_dir, venv_dir, profiling_dir=None):
+    def __init__(self, args, config_dir, venv_dir, profiling_dir=None, temp_dir=None):
 
         self.profiling_mode = args.profiler
         self.profiling_dir = profiling_dir
@@ -73,6 +73,9 @@ def __init__(self, args, config_dir, venv_dir, profiling_dir=None):
         self.logger = logging.getLogger(__name__)
         self.config_dir = config_dir
         self.venv_dir = venv_dir
+        self.temp_dir = temp_dir
+        if self.temp_dir is None:
+            self.temp_dir = os.path.join(self.config_dir, 'tmp')
         self.extra_log = args.extra_log
         self.pipelinewise_bin = os.path.join(
             self.venv_dir, 'cli', 'bin', 'pipelinewise'
@@ -141,7 +144,7 @@ def create_consumable_target_config(self, target_config, tap_inheritable_config)
 
             # Save the new dict as JSON into a temp file
             tempfile_path = utils.create_temp_file(
-                dir=self.get_temp_dir(), prefix='target_config_', suffix='.json'
+                dir=self.temp_dir, prefix='target_config_', suffix='.json'
             )[1]
             utils.save_json(dict_a, tempfile_path)
 
@@ -324,12 +327,12 @@ def create_filtered_tap_properties(
             if create_fallback:
                 # Save to files: filtered and fallback properties
                 temp_properties_path = utils.create_temp_file(
-                    dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                    dir=self.temp_dir, prefix='properties_', suffix='.json'
                 )[1]
                 utils.save_json(properties, temp_properties_path)
 
                 temp_fallback_properties_path = utils.create_temp_file(
-                    dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                    dir=self.temp_dir, prefix='properties_', suffix='.json'
                 )[1]
                 utils.save_json(fallback_properties, temp_fallback_properties_path)
 
@@ -342,7 +345,7 @@ def create_filtered_tap_properties(
 
             # Fallback not required: Save only the filtered properties JSON
             temp_properties_path = utils.create_temp_file(
-                dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                dir=self.temp_dir, prefix='properties_', suffix='.json'
             )[1]
             utils.save_json(properties, temp_properties_path)
 
@@ -363,12 +366,6 @@ def load_config(self):
         else:
             self.config = {}
 
-    def get_temp_dir(self):
-        """
-        Returns the tap specific temp directory
-        """
-        return os.path.join(self.config_dir, 'tmp')
-
     def get_tap_dir(self, target_id, tap_id):
         """
         Get absolute path of a tap directory
@@ -1083,7 +1080,7 @@ def run_tap_fastsync(
             target=target,
             transform=transform,
             venv_dir=self.venv_dir,
-            temp_dir=self.get_temp_dir(),
+            temp_dir=self.temp_dir,
             tables=self.args.tables,
             profiling_mode=self.profiling_mode,
             profiling_dir=self.profiling_dir,
@@ -1565,7 +1562,9 @@ def import_project(self):
         """
         # Read the YAML config files and transform/save into singer compatible
         # JSON files in a common directory structure
-        config = Config.from_yamls(self.config_dir, self.args.dir, self.args.secret)
+        config = Config.from_yamls(
+            self.config_dir, self.args.dir, self.args.secret, self.temp_dir
+        )
         config.save()
 
         # Activating tap stream selections
@@ -1801,7 +1800,7 @@ def __validate_transformations(
             # create a temp file with the content being the given catalog object
             # we need this file to execute the validation cli command
             temp_catalog_file = utils.create_temp_file(
-                dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                dir=self.temp_dir, prefix='properties_', suffix='.json'
             )[1]
 
             utils.save_json(catalog, temp_catalog_file)

tests/units/cli/test_config.py

@@ -7,6 +7,7 @@
 from pipelinewise.cli.errors import InvalidConfigException
 
 PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise'
+PIPELINEWISE_TEST_TEMP_DIR = '/tmp/pipelinewise_tmp'
 
 
 # Todo: Inherit from unittest.TestCase
@@ -208,10 +209,10 @@ def test_from_invalid_yamls_fails(self):
 
     def test_getters(self):
         """Test Config getter functions"""
-        config = Config(PIPELINEWISE_TEST_HOME)
+        config = Config(PIPELINEWISE_TEST_HOME, PIPELINEWISE_TEST_TEMP_DIR)
 
         # Target and tap directory should be g
-        assert config.get_temp_dir() == '{}/tmp'.format(PIPELINEWISE_TEST_HOME)
+        assert config.temp_dir == PIPELINEWISE_TEST_TEMP_DIR
         assert config.get_target_dir('test-target-id') == '{}/test-target-id'.format(
             PIPELINEWISE_TEST_HOME
         )