From a68ab787e750f810f0c8166aa0e11cfd2826700e Mon Sep 17 00:00:00 2001 From: Romil Date: Thu, 10 Mar 2022 00:15:16 -0800 Subject: [PATCH 1/2] mounting mvp --- examples/storage_mount_demo.yaml | 33 ++++++++++++++++++++++++++++++++ sky/data/storage.py | 13 +++++++++++-- sky/task.py | 18 +++++++++++++---- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 examples/storage_mount_demo.yaml diff --git a/examples/storage_mount_demo.yaml b/examples/storage_mount_demo.yaml new file mode 100644 index 00000000000..055205f09d9 --- /dev/null +++ b/examples/storage_mount_demo.yaml @@ -0,0 +1,33 @@ +name: storage-demo + +resources: + cloud: aws + instance_type: m5.2xlarge + +num_nodes: 1 + +file_mounts: + /sharedfs: + name: romil-fs + source: ~/tmp # Empty dir for MVP, this will not be required if mode==MOUNT + persistent: True + mode: 'MOUNT' + +setup: | + # Installing fio for benchmarking. Killing apt process to avoid lock issues + sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1`; + sudo pkill -9 apt-get; + sudo pkill -9 dpkg; + sudo apt-get install fio -y + +run: | + echo This is hostname: $(hostname) + + # Run read benchmark + fio --name=sequential-read --rw=read --refill_buffers --bs=4M --size=4G --filename=/sharedfs/$(hostname)_read.fio 2>&1 | tee $(hostname)_readbench.txt + + # Run example write + echo My hostname: $(hostname) > /sharedfs/$(hostname).txt + + # ls and exit + ls -la /sharedfs diff --git a/sky/data/storage.py b/sky/data/storage.py index ddedc2a1db4..c1184dff8be 100644 --- a/sky/data/storage.py +++ b/sky/data/storage.py @@ -25,6 +25,11 @@ class StorageType(enum.Enum): AZURE = 'AZURE' +class StorageMode(enum.Enum): + MOUNT = 'MOUNT' + COPY = 'COPY' + + class AbstractStore: """AbstractStore abstracts away the different storage types exposed by different clouds. @@ -148,7 +153,8 @@ def __init__(self, name: str, source: Path, stores: Optional[Dict[StorageType, AbstractStore]] = None, - persistent: bool = True): + persistent: bool = True, + mode: StorageMode = StorageMode.MOUNT): """Initializes a Storage object. Three fields are required: the name of the storage, the source @@ -163,10 +169,13 @@ def __init__(self, need to be absolute. stores: Optional; Specify pre-initialized stores (S3Store, GcsStore). persistent: bool; Whether to persist across sky launches. + mode: StorageMode; Specify how the storage object is manifested on + the remote VM. Can be either MOUNT or COPY. """ self.name = name self.source = source self.persistent = persistent + self.mode = mode scheme = urllib.parse.urlsplit(self.source).scheme is_bucket_url = False @@ -368,7 +377,7 @@ def sync_local_dir(self) -> None: increase parallelism, modify max_concurrent_requests in your aws config file (Default path: ~/.aws/config). """ - sync_command = f'aws s3 sync {self.source} s3://{self.name}/ --delete' + sync_command = f'aws s3 sync {self.source} s3://{self.name}/ --delete' # TODO(romilb): Delete is problematic if task writes to s3 and the user runs this again... logger.info(f'Executing: {sync_command}') with subprocess.Popen(sync_command.split(' '), stderr=subprocess.PIPE) as process: diff --git a/sky/task.py b/sky/task.py index 3965f315d5d..ffeac2b0268 100644 --- a/sky/task.py +++ b/sky/task.py @@ -10,6 +10,7 @@ from sky import clouds from sky import resources as resources_lib from sky.data import storage as storage_lib +from sky.data.storage import StorageMode Resources = resources_lib.Resources # A lambda generating commands (node rank_i, node addrs -> cmd_i). @@ -247,13 +248,15 @@ def from_yaml(yaml_path): name = storage.get('name') source = storage.get('source') force_stores = storage.get('force_stores') + mode = storage.get('mode') assert name and source, \ 'Storage Object needs name and source path specified.' persistent = True if storage.get( 'persistent') is None else storage['persistent'] storage_obj = storage_lib.Storage(name=name, source=source, - persistent=persistent) + persistent=persistent, + mode=StorageMode(mode)) if force_stores is not None: assert set(force_stores) <= {'s3', 'gcs', 'azure_blob'} for cloud_type in force_stores: @@ -432,9 +435,16 @@ def add_storage_mounts(self) -> None: storage_type = storage_plans[store] if storage_type is storage_lib.StorageType.S3: # TODO: allow for Storage mounting of different clouds - self.update_file_mounts({ - mnt_path: 's3://' + store.name, - }) + if store.mode == StorageMode.MOUNT: + self.setup = ( + '(sudo wget https://github.com/kahing/goofys/releases/latest/download/goofys' + ' -O /usr/local/bin/goofys && sudo chmod +x /usr/local/bin/goofys && ' + f' sudo mkdir -p {mnt_path} && sudo chmod 777 {mnt_path} && goofys --stat-cache-ttl 10s --type-cache-ttl 10s {store.name} {mnt_path}' + f'); {self.setup or "true"}') + else: + self.update_file_mounts({ + mnt_path: 's3://' + store.name, + }) elif storage_type is storage_lib.StorageType.GCS: # Remember to run `gcloud auth application-default login` self.setup = ( From e63957ab0529442e7b7d43fbb2c40925b4e31e7a Mon Sep 17 00:00:00 2001 From: Romil Date: Thu, 10 Mar 2022 17:36:59 -0800 Subject: [PATCH 2/2] update example --- examples/storage_mount_demo.yaml | 22 +++++++++------------- sky/task.py | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/examples/storage_mount_demo.yaml b/examples/storage_mount_demo.yaml index 055205f09d9..9202e2a8e24 100644 --- a/examples/storage_mount_demo.yaml +++ b/examples/storage_mount_demo.yaml @@ -4,7 +4,7 @@ resources: cloud: aws instance_type: m5.2xlarge -num_nodes: 1 +num_nodes: 2 file_mounts: /sharedfs: @@ -13,21 +13,17 @@ file_mounts: persistent: True mode: 'MOUNT' -setup: | - # Installing fio for benchmarking. Killing apt process to avoid lock issues - sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1`; - sudo pkill -9 apt-get; - sudo pkill -9 dpkg; - sudo apt-get install fio -y - run: | echo This is hostname: $(hostname) - # Run read benchmark - fio --name=sequential-read --rw=read --refill_buffers --bs=4M --size=4G --filename=/sharedfs/$(hostname)_read.fio 2>&1 | tee $(hostname)_readbench.txt + for ((i=1;i<=100;i++)); + do + # Run example write + echo ${i} + echo File ${i}, My hostname: $(hostname) > /sharedfs/$(hostname)_${i}.txt + sleep 1 + done - # Run example write - echo My hostname: $(hostname) > /sharedfs/$(hostname).txt # ls and exit - ls -la /sharedfs + ls -la /sharedfs \ No newline at end of file diff --git a/sky/task.py b/sky/task.py index ffeac2b0268..8b52fa53d7e 100644 --- a/sky/task.py +++ b/sky/task.py @@ -439,7 +439,7 @@ def add_storage_mounts(self) -> None: self.setup = ( '(sudo wget https://github.com/kahing/goofys/releases/latest/download/goofys' ' -O /usr/local/bin/goofys && sudo chmod +x /usr/local/bin/goofys && ' - f' sudo mkdir -p {mnt_path} && sudo chmod 777 {mnt_path} && goofys --stat-cache-ttl 10s --type-cache-ttl 10s {store.name} {mnt_path}' + f' sudo mkdir -p {mnt_path} && sudo chmod 777 {mnt_path} && goofys --stat-cache-ttl 3s --type-cache-ttl 3s {store.name} {mnt_path}' f'); {self.setup or "true"}') else: self.update_file_mounts({